drivers/net/ethernet/intel/i40e/i40e_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
   3
   4 #include <linux/etherdevice.h>
   5 #include <linux/of_net.h>
   6 #include <linux/pci.h>
   7 #include <linux/bpf.h>
   8 #include <generated/utsrelease.h>
   9 #include <linux/crash_dump.h>
  10
  11 /* Local includes */
  12 #include "i40e.h"
  13 #include "i40e_diag.h"
  14 #include "i40e_xsk.h"
  15 #include <net/udp_tunnel.h>
  16 #include <net/xdp_sock_drv.h>
  17 /* All i40e tracepoints are defined by the include below, which
  18  * must be included exactly once across the whole kernel with
  19  * CREATE_TRACE_POINTS defined
  20  */
  21 #define CREATE_TRACE_POINTS
  22 #include "i40e_trace.h"
  23
  24 const char i40e_driver_name[] = "i40e";
  25 static const char i40e_driver_string[] =
  26                         "Intel(R) Ethernet Connection XL710 Network Driver";
  27
  28 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
  29
  30 /* a bit of forward declarations */
  31 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  32 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
  33 static int i40e_add_vsi(struct i40e_vsi *vsi);
  34 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  35 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  36 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  37 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  38 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  39 static void i40e_prep_for_reset(struct i40e_pf *pf);
  40 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
  41                                    bool lock_acquired);
  42 static int i40e_reset(struct i40e_pf *pf);
  43 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
  44 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
  45 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
  46 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
  47 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
  48 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  49 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  50 static int i40e_get_capabilities(struct i40e_pf *pf,
  51                                  enum i40e_admin_queue_opc list_type);
  52 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
  53
  54 /* i40e_pci_tbl - PCI Device ID Table
  55  *
  56  * Last entry must be all 0s
  57  *
  58  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  59  *   Class, Class Mask, private data (not used) }
  60  */
  61 static const struct pci_device_id i40e_pci_tbl[] = {
  62         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  63         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  64         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  65         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  66         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  67         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  68         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  69         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  70         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
  72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
  73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
  86         /* required last entry */
  87         {0, }
  88 };
  89 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  90
  91 #define I40E_MAX_VF_COUNT 128
  92 static int debug = -1;
  93 module_param(debug, uint, 0);
  94 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
  95
  96 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
  97 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
  98 MODULE_LICENSE("GPL v2");
  99
 100 static struct workqueue_struct *i40e_wq;
 101
 102 /**
 103  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 104  * @hw:   pointer to the HW structure
 105  * @mem:  ptr to mem struct to fill out
 106  * @size: size of memory requested
 107  * @alignment: what to align the allocation to
 108  **/
 109 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 110                             u64 size, u32 alignment)
 111 {
 112         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 113
 114         mem->size = ALIGN(size, alignment);
 115         mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
 116                                      GFP_KERNEL);
 117         if (!mem->va)
 118                 return -ENOMEM;
 119
 120         return 0;
 121 }
 122
 123 /**
 124  * i40e_free_dma_mem_d - OS specific memory free for shared code
 125  * @hw:   pointer to the HW structure
 126  * @mem:  ptr to mem struct to free
 127  **/
 128 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 129 {
 130         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 131
 132         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 133         mem->va = NULL;
 134         mem->pa = 0;
 135         mem->size = 0;
 136
 137         return 0;
 138 }
 139
 140 /**
 141  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 142  * @hw:   pointer to the HW structure
 143  * @mem:  ptr to mem struct to fill out
 144  * @size: size of memory requested
 145  **/
 146 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 147                              u32 size)
 148 {
 149         mem->size = size;
 150         mem->va = kzalloc(size, GFP_KERNEL);
 151
 152         if (!mem->va)
 153                 return -ENOMEM;
 154
 155         return 0;
 156 }
 157
 158 /**
 159  * i40e_free_virt_mem_d - OS specific memory free for shared code
 160  * @hw:   pointer to the HW structure
 161  * @mem:  ptr to mem struct to free
 162  **/
 163 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 164 {
 165         /* it's ok to kfree a NULL pointer */
 166         kfree(mem->va);
 167         mem->va = NULL;
 168         mem->size = 0;
 169
 170         return 0;
 171 }
 172
 173 /**
 174  * i40e_get_lump - find a lump of free generic resource
 175  * @pf: board private structure
 176  * @pile: the pile of resource to search
 177  * @needed: the number of items needed
 178  * @id: an owner id to stick on the items assigned
 179  *
 180  * Returns the base item index of the lump, or negative for error
 181  *
 182  * The search_hint trick and lack of advanced fit-finding only work
 183  * because we're highly likely to have all the same size lump requests.
 184  * Linear search time and any fragmentation should be minimal.
 185  **/
 186 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 187                          u16 needed, u16 id)
 188 {
 189         int ret = -ENOMEM;
 190         int i, j;
 191
 192         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 193                 dev_info(&pf->pdev->dev,
 194                          "param err: pile=%s needed=%d id=0x%04x\n",
 195                          pile ? "<valid>" : "<null>", needed, id);
 196                 return -EINVAL;
 197         }
 198
 199         /* start the linear search with an imperfect hint */
 200         i = pile->search_hint;
 201         while (i < pile->num_entries) {
 202                 /* skip already allocated entries */
 203                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 204                         i++;
 205                         continue;
 206                 }
 207
 208                 /* do we have enough in this lump? */
 209                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 210                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 211                                 break;
 212                 }
 213
 214                 if (j == needed) {
 215                         /* there was enough, so assign it to the requestor */
 216                         for (j = 0; j < needed; j++)
 217                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 218                         ret = i;
 219                         pile->search_hint = i + j;
 220                         break;
 221                 }
 222
 223                 /* not enough, so skip over it and continue looking */
 224                 i += j;
 225         }
 226
 227         return ret;
 228 }
 229
 230 /**
 231  * i40e_put_lump - return a lump of generic resource
 232  * @pile: the pile of resource to search
 233  * @index: the base item index
 234  * @id: the owner id of the items assigned
 235  *
 236  * Returns the count of items in the lump
 237  **/
 238 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 239 {
 240         int valid_id = (id | I40E_PILE_VALID_BIT);
 241         int count = 0;
 242         int i;
 243
 244         if (!pile || index >= pile->num_entries)
 245                 return -EINVAL;
 246
 247         for (i = index;
 248              i < pile->num_entries && pile->list[i] == valid_id;
 249              i++) {
 250                 pile->list[i] = 0;
 251                 count++;
 252         }
 253
 254         if (count && index < pile->search_hint)
 255                 pile->search_hint = index;
 256
 257         return count;
 258 }
 259
 260 /**
 261  * i40e_find_vsi_from_id - searches for the vsi with the given id
 262  * @pf: the pf structure to search for the vsi
 263  * @id: id of the vsi it is searching for
 264  **/
 265 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 266 {
 267         int i;
 268
 269         for (i = 0; i < pf->num_alloc_vsi; i++)
 270                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 271                         return pf->vsi[i];
 272
 273         return NULL;
 274 }
 275
 276 /**
 277  * i40e_service_event_schedule - Schedule the service task to wake up
 278  * @pf: board private structure
 279  *
 280  * If not already scheduled, this puts the task into the work queue
 281  **/
 282 void i40e_service_event_schedule(struct i40e_pf *pf)
 283 {
 284         if ((!test_bit(__I40E_DOWN, pf->state) &&
 285              !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
 286               test_bit(__I40E_RECOVERY_MODE, pf->state))
 287                 queue_work(i40e_wq, &pf->service_task);
 288 }
 289
 290 /**
 291  * i40e_tx_timeout - Respond to a Tx Hang
 292  * @netdev: network interface device structure
 293  * @txqueue: queue number timing out
 294  *
 295  * If any port has noticed a Tx timeout, it is likely that the whole
 296  * device is munged, not just the one netdev port, so go for the full
 297  * reset.
 298  **/
 299 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 300 {
 301         struct i40e_netdev_priv *np = netdev_priv(netdev);
 302         struct i40e_vsi *vsi = np->vsi;
 303         struct i40e_pf *pf = vsi->back;
 304         struct i40e_ring *tx_ring = NULL;
 305         unsigned int i;
 306         u32 head, val;
 307
 308         pf->tx_timeout_count++;
 309
 310         /* with txqueue index, find the tx_ring struct */
 311         for (i = 0; i < vsi->num_queue_pairs; i++) {
 312                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 313                         if (txqueue ==
 314                             vsi->tx_rings[i]->queue_index) {
 315                                 tx_ring = vsi->tx_rings[i];
 316                                 break;
 317                         }
 318                 }
 319         }
 320
 321         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 322                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 323         else if (time_before(jiffies,
 324                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 325                 return;   /* don't do any new action before the next timeout */
 326
 327         /* don't kick off another recovery if one is already pending */
 328         if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
 329                 return;
 330
 331         if (tx_ring) {
 332                 head = i40e_get_head(tx_ring);
 333                 /* Read interrupt register */
 334                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 335                         val = rd32(&pf->hw,
 336                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 337                                                 tx_ring->vsi->base_vector - 1));
 338                 else
 339                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 340
 341                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 342                             vsi->seid, txqueue, tx_ring->next_to_clean,
 343                             head, tx_ring->next_to_use,
 344                             readl(tx_ring->tail), val);
 345         }
 346
 347         pf->tx_timeout_last_recovery = jiffies;
 348         netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
 349                     pf->tx_timeout_recovery_level, txqueue);
 350
 351         switch (pf->tx_timeout_recovery_level) {
 352         case 1:
 353                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
 354                 break;
 355         case 2:
 356                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
 357                 break;
 358         case 3:
 359                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
 360                 break;
 361         default:
 362                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 363                 break;
 364         }
 365
 366         i40e_service_event_schedule(pf);
 367         pf->tx_timeout_recovery_level++;
 368 }
 369
 370 /**
 371  * i40e_get_vsi_stats_struct - Get System Network Statistics
 372  * @vsi: the VSI we care about
 373  *
 374  * Returns the address of the device statistics structure.
 375  * The statistics are actually updated from the service task.
 376  **/
 377 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 378 {
 379         return &vsi->net_stats;
 380 }
 381
 382 /**
 383  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
 384  * @ring: Tx ring to get statistics from
 385  * @stats: statistics entry to be updated
 386  **/
 387 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
 388                                             struct rtnl_link_stats64 *stats)
 389 {
 390         u64 bytes, packets;
 391         unsigned int start;
 392
 393         do {
 394                 start = u64_stats_fetch_begin_irq(&ring->syncp);
 395                 packets = ring->stats.packets;
 396                 bytes   = ring->stats.bytes;
 397         } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 398
 399         stats->tx_packets += packets;
 400         stats->tx_bytes   += bytes;
 401 }
 402
 403 /**
 404  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 405  * @netdev: network interface device structure
 406  * @stats: data structure to store statistics
 407  *
 408  * Returns the address of the device statistics structure.
 409  * The statistics are actually updated from the service task.
 410  **/
 411 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
 412                                   struct rtnl_link_stats64 *stats)
 413 {
 414         struct i40e_netdev_priv *np = netdev_priv(netdev);
 415         struct i40e_vsi *vsi = np->vsi;
 416         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 417         struct i40e_ring *ring;
 418         int i;
 419
 420         if (test_bit(__I40E_VSI_DOWN, vsi->state))
 421                 return;
 422
 423         if (!vsi->tx_rings)
 424                 return;
 425
 426         rcu_read_lock();
 427         for (i = 0; i < vsi->num_queue_pairs; i++) {
 428                 u64 bytes, packets;
 429                 unsigned int start;
 430
 431                 ring = READ_ONCE(vsi->tx_rings[i]);
 432                 if (!ring)
 433                         continue;
 434                 i40e_get_netdev_stats_struct_tx(ring, stats);
 435
 436                 if (i40e_enabled_xdp_vsi(vsi)) {
 437                         ring = READ_ONCE(vsi->xdp_rings[i]);
 438                         if (!ring)
 439                                 continue;
 440                         i40e_get_netdev_stats_struct_tx(ring, stats);
 441                 }
 442
 443                 ring = READ_ONCE(vsi->rx_rings[i]);
 444                 if (!ring)
 445                         continue;
 446                 do {
 447                         start   = u64_stats_fetch_begin_irq(&ring->syncp);
 448                         packets = ring->stats.packets;
 449                         bytes   = ring->stats.bytes;
 450                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
 451
 452                 stats->rx_packets += packets;
 453                 stats->rx_bytes   += bytes;
 454
 455         }
 456         rcu_read_unlock();
 457
 458         /* following stats updated by i40e_watchdog_subtask() */
 459         stats->multicast        = vsi_stats->multicast;
 460         stats->tx_errors        = vsi_stats->tx_errors;
 461         stats->tx_dropped       = vsi_stats->tx_dropped;
 462         stats->rx_errors        = vsi_stats->rx_errors;
 463         stats->rx_dropped       = vsi_stats->rx_dropped;
 464         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 465         stats->rx_length_errors = vsi_stats->rx_length_errors;
 466 }
 467
 468 /**
 469  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 470  * @vsi: the VSI to have its stats reset
 471  **/
 472 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 473 {
 474         struct rtnl_link_stats64 *ns;
 475         int i;
 476
 477         if (!vsi)
 478                 return;
 479
 480         ns = i40e_get_vsi_stats_struct(vsi);
 481         memset(ns, 0, sizeof(*ns));
 482         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 483         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 484         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 485         if (vsi->rx_rings && vsi->rx_rings[0]) {
 486                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 487                         memset(&vsi->rx_rings[i]->stats, 0,
 488                                sizeof(vsi->rx_rings[i]->stats));
 489                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 490                                sizeof(vsi->rx_rings[i]->rx_stats));
 491                         memset(&vsi->tx_rings[i]->stats, 0,
 492                                sizeof(vsi->tx_rings[i]->stats));
 493                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 494                                sizeof(vsi->tx_rings[i]->tx_stats));
 495                 }
 496         }
 497         vsi->stat_offsets_loaded = false;
 498 }
 499
 500 /**
 501  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 502  * @pf: the PF to be reset
 503  **/
 504 void i40e_pf_reset_stats(struct i40e_pf *pf)
 505 {
 506         int i;
 507
 508         memset(&pf->stats, 0, sizeof(pf->stats));
 509         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 510         pf->stat_offsets_loaded = false;
 511
 512         for (i = 0; i < I40E_MAX_VEB; i++) {
 513                 if (pf->veb[i]) {
 514                         memset(&pf->veb[i]->stats, 0,
 515                                sizeof(pf->veb[i]->stats));
 516                         memset(&pf->veb[i]->stats_offsets, 0,
 517                                sizeof(pf->veb[i]->stats_offsets));
 518                         memset(&pf->veb[i]->tc_stats, 0,
 519                                sizeof(pf->veb[i]->tc_stats));
 520                         memset(&pf->veb[i]->tc_stats_offsets, 0,
 521                                sizeof(pf->veb[i]->tc_stats_offsets));
 522                         pf->veb[i]->stat_offsets_loaded = false;
 523                 }
 524         }
 525         pf->hw_csum_rx_error = 0;
 526 }
 527
 528 /**
 529  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 530  * @hw: ptr to the hardware info
 531  * @hireg: the high 32 bit reg to read
 532  * @loreg: the low 32 bit reg to read
 533  * @offset_loaded: has the initial offset been loaded yet
 534  * @offset: ptr to current offset value
 535  * @stat: ptr to the stat
 536  *
 537  * Since the device stats are not reset at PFReset, they likely will not
 538  * be zeroed when the driver starts.  We'll save the first values read
 539  * and use them as offsets to be subtracted from the raw values in order
 540  * to report stats that count from zero.  In the process, we also manage
 541  * the potential roll-over.
 542  **/
 543 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 544                                bool offset_loaded, u64 *offset, u64 *stat)
 545 {
 546         u64 new_data;
 547
 548         if (hw->device_id == I40E_DEV_ID_QEMU) {
 549                 new_data = rd32(hw, loreg);
 550                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 551         } else {
 552                 new_data = rd64(hw, loreg);
 553         }
 554         if (!offset_loaded)
 555                 *offset = new_data;
 556         if (likely(new_data >= *offset))
 557                 *stat = new_data - *offset;
 558         else
 559                 *stat = (new_data + BIT_ULL(48)) - *offset;
 560         *stat &= 0xFFFFFFFFFFFFULL;
 561 }
 562
 563 /**
 564  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 565  * @hw: ptr to the hardware info
 566  * @reg: the hw reg to read
 567  * @offset_loaded: has the initial offset been loaded yet
 568  * @offset: ptr to current offset value
 569  * @stat: ptr to the stat
 570  **/
 571 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 572                                bool offset_loaded, u64 *offset, u64 *stat)
 573 {
 574         u32 new_data;
 575
 576         new_data = rd32(hw, reg);
 577         if (!offset_loaded)
 578                 *offset = new_data;
 579         if (likely(new_data >= *offset))
 580                 *stat = (u32)(new_data - *offset);
 581         else
 582                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 583 }
 584
 585 /**
 586  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
 587  * @hw: ptr to the hardware info
 588  * @reg: the hw reg to read and clear
 589  * @stat: ptr to the stat
 590  **/
 591 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
 592 {
 593         u32 new_data = rd32(hw, reg);
 594
 595         wr32(hw, reg, 1); /* must write a nonzero value to clear register */
 596         *stat += new_data;
 597 }
 598
 599 /**
 600  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 601  * @vsi: the VSI to be updated
 602  **/
 603 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 604 {
 605         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 606         struct i40e_pf *pf = vsi->back;
 607         struct i40e_hw *hw = &pf->hw;
 608         struct i40e_eth_stats *oes;
 609         struct i40e_eth_stats *es;     /* device's eth stats */
 610
 611         es = &vsi->eth_stats;
 612         oes = &vsi->eth_stats_offsets;
 613
 614         /* Gather up the stats that the hw collects */
 615         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 616                            vsi->stat_offsets_loaded,
 617                            &oes->tx_errors, &es->tx_errors);
 618         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 619                            vsi->stat_offsets_loaded,
 620                            &oes->rx_discards, &es->rx_discards);
 621         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 622                            vsi->stat_offsets_loaded,
 623                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 624
 625         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 626                            I40E_GLV_GORCL(stat_idx),
 627                            vsi->stat_offsets_loaded,
 628                            &oes->rx_bytes, &es->rx_bytes);
 629         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 630                            I40E_GLV_UPRCL(stat_idx),
 631                            vsi->stat_offsets_loaded,
 632                            &oes->rx_unicast, &es->rx_unicast);
 633         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 634                            I40E_GLV_MPRCL(stat_idx),
 635                            vsi->stat_offsets_loaded,
 636                            &oes->rx_multicast, &es->rx_multicast);
 637         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 638                            I40E_GLV_BPRCL(stat_idx),
 639                            vsi->stat_offsets_loaded,
 640                            &oes->rx_broadcast, &es->rx_broadcast);
 641
 642         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 643                            I40E_GLV_GOTCL(stat_idx),
 644                            vsi->stat_offsets_loaded,
 645                            &oes->tx_bytes, &es->tx_bytes);
 646         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 647                            I40E_GLV_UPTCL(stat_idx),
 648                            vsi->stat_offsets_loaded,
 649                            &oes->tx_unicast, &es->tx_unicast);
 650         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 651                            I40E_GLV_MPTCL(stat_idx),
 652                            vsi->stat_offsets_loaded,
 653                            &oes->tx_multicast, &es->tx_multicast);
 654         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 655                            I40E_GLV_BPTCL(stat_idx),
 656                            vsi->stat_offsets_loaded,
 657                            &oes->tx_broadcast, &es->tx_broadcast);
 658         vsi->stat_offsets_loaded = true;
 659 }
 660
 661 /**
 662  * i40e_update_veb_stats - Update Switch component statistics
 663  * @veb: the VEB being updated
 664  **/
 665 void i40e_update_veb_stats(struct i40e_veb *veb)
 666 {
 667         struct i40e_pf *pf = veb->pf;
 668         struct i40e_hw *hw = &pf->hw;
 669         struct i40e_eth_stats *oes;
 670         struct i40e_eth_stats *es;     /* device's eth stats */
 671         struct i40e_veb_tc_stats *veb_oes;
 672         struct i40e_veb_tc_stats *veb_es;
 673         int i, idx = 0;
 674
 675         idx = veb->stats_idx;
 676         es = &veb->stats;
 677         oes = &veb->stats_offsets;
 678         veb_es = &veb->tc_stats;
 679         veb_oes = &veb->tc_stats_offsets;
 680
 681         /* Gather up the stats that the hw collects */
 682         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 683                            veb->stat_offsets_loaded,
 684                            &oes->tx_discards, &es->tx_discards);
 685         if (hw->revision_id > 0)
 686                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 687                                    veb->stat_offsets_loaded,
 688                                    &oes->rx_unknown_protocol,
 689                                    &es->rx_unknown_protocol);
 690         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 691                            veb->stat_offsets_loaded,
 692                            &oes->rx_bytes, &es->rx_bytes);
 693         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 694                            veb->stat_offsets_loaded,
 695                            &oes->rx_unicast, &es->rx_unicast);
 696         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 697                            veb->stat_offsets_loaded,
 698                            &oes->rx_multicast, &es->rx_multicast);
 699         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 700                            veb->stat_offsets_loaded,
 701                            &oes->rx_broadcast, &es->rx_broadcast);
 702
 703         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 704                            veb->stat_offsets_loaded,
 705                            &oes->tx_bytes, &es->tx_bytes);
 706         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 707                            veb->stat_offsets_loaded,
 708                            &oes->tx_unicast, &es->tx_unicast);
 709         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 710                            veb->stat_offsets_loaded,
 711                            &oes->tx_multicast, &es->tx_multicast);
 712         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 713                            veb->stat_offsets_loaded,
 714                            &oes->tx_broadcast, &es->tx_broadcast);
 715         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 716                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 717                                    I40E_GLVEBTC_RPCL(i, idx),
 718                                    veb->stat_offsets_loaded,
 719                                    &veb_oes->tc_rx_packets[i],
 720                                    &veb_es->tc_rx_packets[i]);
 721                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 722                                    I40E_GLVEBTC_RBCL(i, idx),
 723                                    veb->stat_offsets_loaded,
 724                                    &veb_oes->tc_rx_bytes[i],
 725                                    &veb_es->tc_rx_bytes[i]);
 726                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 727                                    I40E_GLVEBTC_TPCL(i, idx),
 728                                    veb->stat_offsets_loaded,
 729                                    &veb_oes->tc_tx_packets[i],
 730                                    &veb_es->tc_tx_packets[i]);
 731                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 732                                    I40E_GLVEBTC_TBCL(i, idx),
 733                                    veb->stat_offsets_loaded,
 734                                    &veb_oes->tc_tx_bytes[i],
 735                                    &veb_es->tc_tx_bytes[i]);
 736         }
 737         veb->stat_offsets_loaded = true;
 738 }
 739
 740 /**
 741  * i40e_update_vsi_stats - Update the vsi statistics counters.
 742  * @vsi: the VSI to be updated
 743  *
 744  * There are a few instances where we store the same stat in a
 745  * couple of different structs.  This is partly because we have
 746  * the netdev stats that need to be filled out, which is slightly
 747  * different from the "eth_stats" defined by the chip and used in
 748  * VF communications.  We sort it out here.
 749  **/
 750 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 751 {
 752         struct i40e_pf *pf = vsi->back;
 753         struct rtnl_link_stats64 *ons;
 754         struct rtnl_link_stats64 *ns;   /* netdev stats */
 755         struct i40e_eth_stats *oes;
 756         struct i40e_eth_stats *es;     /* device's eth stats */
 757         u32 tx_restart, tx_busy;
 758         struct i40e_ring *p;
 759         u32 rx_page, rx_buf;
 760         u64 bytes, packets;
 761         unsigned int start;
 762         u64 tx_linearize;
 763         u64 tx_force_wb;
 764         u64 rx_p, rx_b;
 765         u64 tx_p, tx_b;
 766         u16 q;
 767
 768         if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
 769             test_bit(__I40E_CONFIG_BUSY, pf->state))
 770                 return;
 771
 772         ns = i40e_get_vsi_stats_struct(vsi);
 773         ons = &vsi->net_stats_offsets;
 774         es = &vsi->eth_stats;
 775         oes = &vsi->eth_stats_offsets;
 776
 777         /* Gather up the netdev and vsi stats that the driver collects
 778          * on the fly during packet processing
 779          */
 780         rx_b = rx_p = 0;
 781         tx_b = tx_p = 0;
 782         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 783         rx_page = 0;
 784         rx_buf = 0;
 785         rcu_read_lock();
 786         for (q = 0; q < vsi->num_queue_pairs; q++) {
 787                 /* locate Tx ring */
 788                 p = READ_ONCE(vsi->tx_rings[q]);
 789                 if (!p)
 790                         continue;
 791
 792                 do {
 793                         start = u64_stats_fetch_begin_irq(&p->syncp);
 794                         packets = p->stats.packets;
 795                         bytes = p->stats.bytes;
 796                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 797                 tx_b += bytes;
 798                 tx_p += packets;
 799                 tx_restart += p->tx_stats.restart_queue;
 800                 tx_busy += p->tx_stats.tx_busy;
 801                 tx_linearize += p->tx_stats.tx_linearize;
 802                 tx_force_wb += p->tx_stats.tx_force_wb;
 803
 804                 /* locate Rx ring */
 805                 p = READ_ONCE(vsi->rx_rings[q]);
 806                 if (!p)
 807                         continue;
 808
 809                 do {
 810                         start = u64_stats_fetch_begin_irq(&p->syncp);
 811                         packets = p->stats.packets;
 812                         bytes = p->stats.bytes;
 813                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 814                 rx_b += bytes;
 815                 rx_p += packets;
 816                 rx_buf += p->rx_stats.alloc_buff_failed;
 817                 rx_page += p->rx_stats.alloc_page_failed;
 818
 819                 if (i40e_enabled_xdp_vsi(vsi)) {
 820                         /* locate XDP ring */
 821                         p = READ_ONCE(vsi->xdp_rings[q]);
 822                         if (!p)
 823                                 continue;
 824
 825                         do {
 826                                 start = u64_stats_fetch_begin_irq(&p->syncp);
 827                                 packets = p->stats.packets;
 828                                 bytes = p->stats.bytes;
 829                         } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 830                         tx_b += bytes;
 831                         tx_p += packets;
 832                         tx_restart += p->tx_stats.restart_queue;
 833                         tx_busy += p->tx_stats.tx_busy;
 834                         tx_linearize += p->tx_stats.tx_linearize;
 835                         tx_force_wb += p->tx_stats.tx_force_wb;
 836                 }
 837         }
 838         rcu_read_unlock();
 839         vsi->tx_restart = tx_restart;
 840         vsi->tx_busy = tx_busy;
 841         vsi->tx_linearize = tx_linearize;
 842         vsi->tx_force_wb = tx_force_wb;
 843         vsi->rx_page_failed = rx_page;
 844         vsi->rx_buf_failed = rx_buf;
 845
 846         ns->rx_packets = rx_p;
 847         ns->rx_bytes = rx_b;
 848         ns->tx_packets = tx_p;
 849         ns->tx_bytes = tx_b;
 850
 851         /* update netdev stats from eth stats */
 852         i40e_update_eth_stats(vsi);
 853         ons->tx_errors = oes->tx_errors;
 854         ns->tx_errors = es->tx_errors;
 855         ons->multicast = oes->rx_multicast;
 856         ns->multicast = es->rx_multicast;
 857         ons->rx_dropped = oes->rx_discards;
 858         ns->rx_dropped = es->rx_discards;
 859         ons->tx_dropped = oes->tx_discards;
 860         ns->tx_dropped = es->tx_discards;
 861
 862         /* pull in a couple PF stats if this is the main vsi */
 863         if (vsi == pf->vsi[pf->lan_vsi]) {
 864                 ns->rx_crc_errors = pf->stats.crc_errors;
 865                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 866                 ns->rx_length_errors = pf->stats.rx_length_errors;
 867         }
 868 }
 869
 870 /**
 871  * i40e_update_pf_stats - Update the PF statistics counters.
 872  * @pf: the PF to be updated
 873  **/
 874 static void i40e_update_pf_stats(struct i40e_pf *pf)
 875 {
 876         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 877         struct i40e_hw_port_stats *nsd = &pf->stats;
 878         struct i40e_hw *hw = &pf->hw;
 879         u32 val;
 880         int i;
 881
 882         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 883                            I40E_GLPRT_GORCL(hw->port),
 884                            pf->stat_offsets_loaded,
 885                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 886         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 887                            I40E_GLPRT_GOTCL(hw->port),
 888                            pf->stat_offsets_loaded,
 889                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 890         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 891                            pf->stat_offsets_loaded,
 892                            &osd->eth.rx_discards,
 893                            &nsd->eth.rx_discards);
 894         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 895                            I40E_GLPRT_UPRCL(hw->port),
 896                            pf->stat_offsets_loaded,
 897                            &osd->eth.rx_unicast,
 898                            &nsd->eth.rx_unicast);
 899         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 900                            I40E_GLPRT_MPRCL(hw->port),
 901                            pf->stat_offsets_loaded,
 902                            &osd->eth.rx_multicast,
 903                            &nsd->eth.rx_multicast);
 904         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 905                            I40E_GLPRT_BPRCL(hw->port),
 906                            pf->stat_offsets_loaded,
 907                            &osd->eth.rx_broadcast,
 908                            &nsd->eth.rx_broadcast);
 909         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 910                            I40E_GLPRT_UPTCL(hw->port),
 911                            pf->stat_offsets_loaded,
 912                            &osd->eth.tx_unicast,
 913                            &nsd->eth.tx_unicast);
 914         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 915                            I40E_GLPRT_MPTCL(hw->port),
 916                            pf->stat_offsets_loaded,
 917                            &osd->eth.tx_multicast,
 918                            &nsd->eth.tx_multicast);
 919         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 920                            I40E_GLPRT_BPTCL(hw->port),
 921                            pf->stat_offsets_loaded,
 922                            &osd->eth.tx_broadcast,
 923                            &nsd->eth.tx_broadcast);
 924
 925         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 926                            pf->stat_offsets_loaded,
 927                            &osd->tx_dropped_link_down,
 928                            &nsd->tx_dropped_link_down);
 929
 930         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 931                            pf->stat_offsets_loaded,
 932                            &osd->crc_errors, &nsd->crc_errors);
 933
 934         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 935                            pf->stat_offsets_loaded,
 936                            &osd->illegal_bytes, &nsd->illegal_bytes);
 937
 938         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 939                            pf->stat_offsets_loaded,
 940                            &osd->mac_local_faults,
 941                            &nsd->mac_local_faults);
 942         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 943                            pf->stat_offsets_loaded,
 944                            &osd->mac_remote_faults,
 945                            &nsd->mac_remote_faults);
 946
 947         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 948                            pf->stat_offsets_loaded,
 949                            &osd->rx_length_errors,
 950                            &nsd->rx_length_errors);
 951
 952         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 953                            pf->stat_offsets_loaded,
 954                            &osd->link_xon_rx, &nsd->link_xon_rx);
 955         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 956                            pf->stat_offsets_loaded,
 957                            &osd->link_xon_tx, &nsd->link_xon_tx);
 958         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 959                            pf->stat_offsets_loaded,
 960                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 961         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 962                            pf->stat_offsets_loaded,
 963                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 964
 965         for (i = 0; i < 8; i++) {
 966                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 967                                    pf->stat_offsets_loaded,
 968                                    &osd->priority_xoff_rx[i],
 969                                    &nsd->priority_xoff_rx[i]);
 970                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 971                                    pf->stat_offsets_loaded,
 972                                    &osd->priority_xon_rx[i],
 973                                    &nsd->priority_xon_rx[i]);
 974                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 975                                    pf->stat_offsets_loaded,
 976                                    &osd->priority_xon_tx[i],
 977                                    &nsd->priority_xon_tx[i]);
 978                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 979                                    pf->stat_offsets_loaded,
 980                                    &osd->priority_xoff_tx[i],
 981                                    &nsd->priority_xoff_tx[i]);
 982                 i40e_stat_update32(hw,
 983                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
 984                                    pf->stat_offsets_loaded,
 985                                    &osd->priority_xon_2_xoff[i],
 986                                    &nsd->priority_xon_2_xoff[i]);
 987         }
 988
 989         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
 990                            I40E_GLPRT_PRC64L(hw->port),
 991                            pf->stat_offsets_loaded,
 992                            &osd->rx_size_64, &nsd->rx_size_64);
 993         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
 994                            I40E_GLPRT_PRC127L(hw->port),
 995                            pf->stat_offsets_loaded,
 996                            &osd->rx_size_127, &nsd->rx_size_127);
 997         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
 998                            I40E_GLPRT_PRC255L(hw->port),
 999                            pf->stat_offsets_loaded,
1000                            &osd->rx_size_255, &nsd->rx_size_255);
1001         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1002                            I40E_GLPRT_PRC511L(hw->port),
1003                            pf->stat_offsets_loaded,
1004                            &osd->rx_size_511, &nsd->rx_size_511);
1005         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1006                            I40E_GLPRT_PRC1023L(hw->port),
1007                            pf->stat_offsets_loaded,
1008                            &osd->rx_size_1023, &nsd->rx_size_1023);
1009         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1010                            I40E_GLPRT_PRC1522L(hw->port),
1011                            pf->stat_offsets_loaded,
1012                            &osd->rx_size_1522, &nsd->rx_size_1522);
1013         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1014                            I40E_GLPRT_PRC9522L(hw->port),
1015                            pf->stat_offsets_loaded,
1016                            &osd->rx_size_big, &nsd->rx_size_big);
1017
1018         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1019                            I40E_GLPRT_PTC64L(hw->port),
1020                            pf->stat_offsets_loaded,
1021                            &osd->tx_size_64, &nsd->tx_size_64);
1022         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1023                            I40E_GLPRT_PTC127L(hw->port),
1024                            pf->stat_offsets_loaded,
1025                            &osd->tx_size_127, &nsd->tx_size_127);
1026         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1027                            I40E_GLPRT_PTC255L(hw->port),
1028                            pf->stat_offsets_loaded,
1029                            &osd->tx_size_255, &nsd->tx_size_255);
1030         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1031                            I40E_GLPRT_PTC511L(hw->port),
1032                            pf->stat_offsets_loaded,
1033                            &osd->tx_size_511, &nsd->tx_size_511);
1034         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1035                            I40E_GLPRT_PTC1023L(hw->port),
1036                            pf->stat_offsets_loaded,
1037                            &osd->tx_size_1023, &nsd->tx_size_1023);
1038         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1039                            I40E_GLPRT_PTC1522L(hw->port),
1040                            pf->stat_offsets_loaded,
1041                            &osd->tx_size_1522, &nsd->tx_size_1522);
1042         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1043                            I40E_GLPRT_PTC9522L(hw->port),
1044                            pf->stat_offsets_loaded,
1045                            &osd->tx_size_big, &nsd->tx_size_big);
1046
1047         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1048                            pf->stat_offsets_loaded,
1049                            &osd->rx_undersize, &nsd->rx_undersize);
1050         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1051                            pf->stat_offsets_loaded,
1052                            &osd->rx_fragments, &nsd->rx_fragments);
1053         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1054                            pf->stat_offsets_loaded,
1055                            &osd->rx_oversize, &nsd->rx_oversize);
1056         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1057                            pf->stat_offsets_loaded,
1058                            &osd->rx_jabber, &nsd->rx_jabber);
1059
1060         /* FDIR stats */
1061         i40e_stat_update_and_clear32(hw,
1062                         I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1063                         &nsd->fd_atr_match);
1064         i40e_stat_update_and_clear32(hw,
1065                         I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1066                         &nsd->fd_sb_match);
1067         i40e_stat_update_and_clear32(hw,
1068                         I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1069                         &nsd->fd_atr_tunnel_match);
1070
1071         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1072         nsd->tx_lpi_status =
1073                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1074                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1075         nsd->rx_lpi_status =
1076                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1077                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1078         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1079                            pf->stat_offsets_loaded,
1080                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1081         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1082                            pf->stat_offsets_loaded,
1083                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1084
1085         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1086             !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1087                 nsd->fd_sb_status = true;
1088         else
1089                 nsd->fd_sb_status = false;
1090
1091         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1092             !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1093                 nsd->fd_atr_status = true;
1094         else
1095                 nsd->fd_atr_status = false;
1096
1097         pf->stat_offsets_loaded = true;
1098 }
1099
1100 /**
1101  * i40e_update_stats - Update the various statistics counters.
1102  * @vsi: the VSI to be updated
1103  *
1104  * Update the various stats for this VSI and its related entities.
1105  **/
1106 void i40e_update_stats(struct i40e_vsi *vsi)
1107 {
1108         struct i40e_pf *pf = vsi->back;
1109
1110         if (vsi == pf->vsi[pf->lan_vsi])
1111                 i40e_update_pf_stats(pf);
1112
1113         i40e_update_vsi_stats(vsi);
1114 }
1115
1116 /**
1117  * i40e_count_filters - counts VSI mac filters
1118  * @vsi: the VSI to be searched
1119  *
1120  * Returns count of mac filters
1121  **/
1122 int i40e_count_filters(struct i40e_vsi *vsi)
1123 {
1124         struct i40e_mac_filter *f;
1125         struct hlist_node *h;
1126         int bkt;
1127         int cnt = 0;
1128
1129         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
1130                 ++cnt;
1131
1132         return cnt;
1133 }
1134
1135 /**
1136  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1137  * @vsi: the VSI to be searched
1138  * @macaddr: the MAC address
1139  * @vlan: the vlan
1140  *
1141  * Returns ptr to the filter object or NULL
1142  **/
1143 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1144                                                 const u8 *macaddr, s16 vlan)
1145 {
1146         struct i40e_mac_filter *f;
1147         u64 key;
1148
1149         if (!vsi || !macaddr)
1150                 return NULL;
1151
1152         key = i40e_addr_to_hkey(macaddr);
1153         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1154                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1155                     (vlan == f->vlan))
1156                         return f;
1157         }
1158         return NULL;
1159 }
1160
1161 /**
1162  * i40e_find_mac - Find a mac addr in the macvlan filters list
1163  * @vsi: the VSI to be searched
1164  * @macaddr: the MAC address we are searching for
1165  *
1166  * Returns the first filter with the provided MAC address or NULL if
1167  * MAC address was not found
1168  **/
1169 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1170 {
1171         struct i40e_mac_filter *f;
1172         u64 key;
1173
1174         if (!vsi || !macaddr)
1175                 return NULL;
1176
1177         key = i40e_addr_to_hkey(macaddr);
1178         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1179                 if ((ether_addr_equal(macaddr, f->macaddr)))
1180                         return f;
1181         }
1182         return NULL;
1183 }
1184
1185 /**
1186  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1187  * @vsi: the VSI to be searched
1188  *
1189  * Returns true if VSI is in vlan mode or false otherwise
1190  **/
1191 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1192 {
1193         /* If we have a PVID, always operate in VLAN mode */
1194         if (vsi->info.pvid)
1195                 return true;
1196
1197         /* We need to operate in VLAN mode whenever we have any filters with
1198          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1199          * time, incurring search cost repeatedly. However, we can notice two
1200          * things:
1201          *
1202          * 1) the only place where we can gain a VLAN filter is in
1203          *    i40e_add_filter.
1204          *
1205          * 2) the only place where filters are actually removed is in
1206          *    i40e_sync_filters_subtask.
1207          *
1208          * Thus, we can simply use a boolean value, has_vlan_filters which we
1209          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1210          * we have to perform the full search after deleting filters in
1211          * i40e_sync_filters_subtask, but we already have to search
1212          * filters here and can perform the check at the same time. This
1213          * results in avoiding embedding a loop for VLAN mode inside another
1214          * loop over all the filters, and should maintain correctness as noted
1215          * above.
1216          */
1217         return vsi->has_vlan_filter;
1218 }
1219
1220 /**
1221  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1222  * @vsi: the VSI to configure
1223  * @tmp_add_list: list of filters ready to be added
1224  * @tmp_del_list: list of filters ready to be deleted
1225  * @vlan_filters: the number of active VLAN filters
1226  *
1227  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1228  * behave as expected. If we have any active VLAN filters remaining or about
1229  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1230  * so that they only match against untagged traffic. If we no longer have any
1231  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1232  * so that they match against both tagged and untagged traffic. In this way,
1233  * we ensure that we correctly receive the desired traffic. This ensures that
1234  * when we have an active VLAN we will receive only untagged traffic and
1235  * traffic matching active VLANs. If we have no active VLANs then we will
1236  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1237  *
1238  * Finally, in a similar fashion, this function also corrects filters when
1239  * there is an active PVID assigned to this VSI.
1240  *
1241  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1242  *
1243  * This function is only expected to be called from within
1244  * i40e_sync_vsi_filters.
1245  *
1246  * NOTE: This function expects to be called while under the
1247  * mac_filter_hash_lock
1248  */
1249 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1250                                          struct hlist_head *tmp_add_list,
1251                                          struct hlist_head *tmp_del_list,
1252                                          int vlan_filters)
1253 {
1254         s16 pvid = le16_to_cpu(vsi->info.pvid);
1255         struct i40e_mac_filter *f, *add_head;
1256         struct i40e_new_mac_filter *new;
1257         struct hlist_node *h;
1258         int bkt, new_vlan;
1259
1260         /* To determine if a particular filter needs to be replaced we
1261          * have the three following conditions:
1262          *
1263          * a) if we have a PVID assigned, then all filters which are
1264          *    not marked as VLAN=PVID must be replaced with filters that
1265          *    are.
1266          * b) otherwise, if we have any active VLANS, all filters
1267          *    which are marked as VLAN=-1 must be replaced with
1268          *    filters marked as VLAN=0
1269          * c) finally, if we do not have any active VLANS, all filters
1270          *    which are marked as VLAN=0 must be replaced with filters
1271          *    marked as VLAN=-1
1272          */
1273
1274         /* Update the filters about to be added in place */
1275         hlist_for_each_entry(new, tmp_add_list, hlist) {
1276                 if (pvid && new->f->vlan != pvid)
1277                         new->f->vlan = pvid;
1278                 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1279                         new->f->vlan = 0;
1280                 else if (!vlan_filters && new->f->vlan == 0)
1281                         new->f->vlan = I40E_VLAN_ANY;
1282         }
1283
1284         /* Update the remaining active filters */
1285         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1286                 /* Combine the checks for whether a filter needs to be changed
1287                  * and then determine the new VLAN inside the if block, in
1288                  * order to avoid duplicating code for adding the new filter
1289                  * then deleting the old filter.
1290                  */
1291                 if ((pvid && f->vlan != pvid) ||
1292                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1293                     (!vlan_filters && f->vlan == 0)) {
1294                         /* Determine the new vlan we will be adding */
1295                         if (pvid)
1296                                 new_vlan = pvid;
1297                         else if (vlan_filters)
1298                                 new_vlan = 0;
1299                         else
1300                                 new_vlan = I40E_VLAN_ANY;
1301
1302                         /* Create the new filter */
1303                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1304                         if (!add_head)
1305                                 return -ENOMEM;
1306
1307                         /* Create a temporary i40e_new_mac_filter */
1308                         new = kzalloc(sizeof(*new), GFP_ATOMIC);
1309                         if (!new)
1310                                 return -ENOMEM;
1311
1312                         new->f = add_head;
1313                         new->state = add_head->state;
1314
1315                         /* Add the new filter to the tmp list */
1316                         hlist_add_head(&new->hlist, tmp_add_list);
1317
1318                         /* Put the original filter into the delete list */
1319                         f->state = I40E_FILTER_REMOVE;
1320                         hash_del(&f->hlist);
1321                         hlist_add_head(&f->hlist, tmp_del_list);
1322                 }
1323         }
1324
1325         vsi->has_vlan_filter = !!vlan_filters;
1326
1327         return 0;
1328 }
1329
1330 /**
1331  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1332  * @vsi: the PF Main VSI - inappropriate for any other VSI
1333  * @macaddr: the MAC address
1334  *
1335  * Remove whatever filter the firmware set up so the driver can manage
1336  * its own filtering intelligently.
1337  **/
1338 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1339 {
1340         struct i40e_aqc_remove_macvlan_element_data element;
1341         struct i40e_pf *pf = vsi->back;
1342
1343         /* Only appropriate for the PF main VSI */
1344         if (vsi->type != I40E_VSI_MAIN)
1345                 return;
1346
1347         memset(&element, 0, sizeof(element));
1348         ether_addr_copy(element.mac_addr, macaddr);
1349         element.vlan_tag = 0;
1350         /* Ignore error returns, some firmware does it this way... */
1351         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1352         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1353
1354         memset(&element, 0, sizeof(element));
1355         ether_addr_copy(element.mac_addr, macaddr);
1356         element.vlan_tag = 0;
1357         /* ...and some firmware does it this way. */
1358         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1359                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1360         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1361 }
1362
1363 /**
1364  * i40e_add_filter - Add a mac/vlan filter to the VSI
1365  * @vsi: the VSI to be searched
1366  * @macaddr: the MAC address
1367  * @vlan: the vlan
1368  *
1369  * Returns ptr to the filter object or NULL when no memory available.
1370  *
1371  * NOTE: This function is expected to be called with mac_filter_hash_lock
1372  * being held.
1373  **/
1374 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1375                                         const u8 *macaddr, s16 vlan)
1376 {
1377         struct i40e_mac_filter *f;
1378         u64 key;
1379
1380         if (!vsi || !macaddr)
1381                 return NULL;
1382
1383         f = i40e_find_filter(vsi, macaddr, vlan);
1384         if (!f) {
1385                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1386                 if (!f)
1387                         return NULL;
1388
1389                 /* Update the boolean indicating if we need to function in
1390                  * VLAN mode.
1391                  */
1392                 if (vlan >= 0)
1393                         vsi->has_vlan_filter = true;
1394
1395                 ether_addr_copy(f->macaddr, macaddr);
1396                 f->vlan = vlan;
1397                 f->state = I40E_FILTER_NEW;
1398                 INIT_HLIST_NODE(&f->hlist);
1399
1400                 key = i40e_addr_to_hkey(macaddr);
1401                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1402
1403                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1404                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1405         }
1406
1407         /* If we're asked to add a filter that has been marked for removal, it
1408          * is safe to simply restore it to active state. __i40e_del_filter
1409          * will have simply deleted any filters which were previously marked
1410          * NEW or FAILED, so if it is currently marked REMOVE it must have
1411          * previously been ACTIVE. Since we haven't yet run the sync filters
1412          * task, just restore this filter to the ACTIVE state so that the
1413          * sync task leaves it in place
1414          */
1415         if (f->state == I40E_FILTER_REMOVE)
1416                 f->state = I40E_FILTER_ACTIVE;
1417
1418         return f;
1419 }
1420
1421 /**
1422  * __i40e_del_filter - Remove a specific filter from the VSI
1423  * @vsi: VSI to remove from
1424  * @f: the filter to remove from the list
1425  *
1426  * This function should be called instead of i40e_del_filter only if you know
1427  * the exact filter you will remove already, such as via i40e_find_filter or
1428  * i40e_find_mac.
1429  *
1430  * NOTE: This function is expected to be called with mac_filter_hash_lock
1431  * being held.
1432  * ANOTHER NOTE: This function MUST be called from within the context of
1433  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1434  * instead of list_for_each_entry().
1435  **/
1436 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1437 {
1438         if (!f)
1439                 return;
1440
1441         /* If the filter was never added to firmware then we can just delete it
1442          * directly and we don't want to set the status to remove or else an
1443          * admin queue command will unnecessarily fire.
1444          */
1445         if ((f->state == I40E_FILTER_FAILED) ||
1446             (f->state == I40E_FILTER_NEW)) {
1447                 hash_del(&f->hlist);
1448                 kfree(f);
1449         } else {
1450                 f->state = I40E_FILTER_REMOVE;
1451         }
1452
1453         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1454         set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1455 }
1456
1457 /**
1458  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1459  * @vsi: the VSI to be searched
1460  * @macaddr: the MAC address
1461  * @vlan: the VLAN
1462  *
1463  * NOTE: This function is expected to be called with mac_filter_hash_lock
1464  * being held.
1465  * ANOTHER NOTE: This function MUST be called from within the context of
1466  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1467  * instead of list_for_each_entry().
1468  **/
1469 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1470 {
1471         struct i40e_mac_filter *f;
1472
1473         if (!vsi || !macaddr)
1474                 return;
1475
1476         f = i40e_find_filter(vsi, macaddr, vlan);
1477         __i40e_del_filter(vsi, f);
1478 }
1479
1480 /**
1481  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1482  * @vsi: the VSI to be searched
1483  * @macaddr: the mac address to be filtered
1484  *
1485  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1486  * go through all the macvlan filters and add a macvlan filter for each
1487  * unique vlan that already exists. If a PVID has been assigned, instead only
1488  * add the macaddr to that VLAN.
1489  *
1490  * Returns last filter added on success, else NULL
1491  **/
1492 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1493                                             const u8 *macaddr)
1494 {
1495         struct i40e_mac_filter *f, *add = NULL;
1496         struct hlist_node *h;
1497         int bkt;
1498
1499         if (vsi->info.pvid)
1500                 return i40e_add_filter(vsi, macaddr,
1501                                        le16_to_cpu(vsi->info.pvid));
1502
1503         if (!i40e_is_vsi_in_vlan(vsi))
1504                 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1505
1506         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1507                 if (f->state == I40E_FILTER_REMOVE)
1508                         continue;
1509                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1510                 if (!add)
1511                         return NULL;
1512         }
1513
1514         return add;
1515 }
1516
1517 /**
1518  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1519  * @vsi: the VSI to be searched
1520  * @macaddr: the mac address to be removed
1521  *
1522  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1523  * associated with.
1524  *
1525  * Returns 0 for success, or error
1526  **/
1527 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1528 {
1529         struct i40e_mac_filter *f;
1530         struct hlist_node *h;
1531         bool found = false;
1532         int bkt;
1533
1534         lockdep_assert_held(&vsi->mac_filter_hash_lock);
1535         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1536                 if (ether_addr_equal(macaddr, f->macaddr)) {
1537                         __i40e_del_filter(vsi, f);
1538                         found = true;
1539                 }
1540         }
1541
1542         if (found)
1543                 return 0;
1544         else
1545                 return -ENOENT;
1546 }
1547
1548 /**
1549  * i40e_set_mac - NDO callback to set mac address
1550  * @netdev: network interface device structure
1551  * @p: pointer to an address structure
1552  *
1553  * Returns 0 on success, negative on failure
1554  **/
1555 static int i40e_set_mac(struct net_device *netdev, void *p)
1556 {
1557         struct i40e_netdev_priv *np = netdev_priv(netdev);
1558         struct i40e_vsi *vsi = np->vsi;
1559         struct i40e_pf *pf = vsi->back;
1560         struct i40e_hw *hw = &pf->hw;
1561         struct sockaddr *addr = p;
1562
1563         if (!is_valid_ether_addr(addr->sa_data))
1564                 return -EADDRNOTAVAIL;
1565
1566         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1567                 netdev_info(netdev, "already using mac address %pM\n",
1568                             addr->sa_data);
1569                 return 0;
1570         }
1571
1572         if (test_bit(__I40E_DOWN, pf->state) ||
1573             test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1574                 return -EADDRNOTAVAIL;
1575
1576         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1577                 netdev_info(netdev, "returning to hw mac address %pM\n",
1578                             hw->mac.addr);
1579         else
1580                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1581
1582         /* Copy the address first, so that we avoid a possible race with
1583          * .set_rx_mode().
1584          * - Remove old address from MAC filter
1585          * - Copy new address
1586          * - Add new address to MAC filter
1587          */
1588         spin_lock_bh(&vsi->mac_filter_hash_lock);
1589         i40e_del_mac_filter(vsi, netdev->dev_addr);
1590         eth_hw_addr_set(netdev, addr->sa_data);
1591         i40e_add_mac_filter(vsi, netdev->dev_addr);
1592         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1593
1594         if (vsi->type == I40E_VSI_MAIN) {
1595                 i40e_status ret;
1596
1597                 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1598                                                 addr->sa_data, NULL);
1599                 if (ret)
1600                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1601                                     i40e_stat_str(hw, ret),
1602                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1603         }
1604
1605         /* schedule our worker thread which will take care of
1606          * applying the new filter changes
1607          */
1608         i40e_service_event_schedule(pf);
1609         return 0;
1610 }
1611
1612 /**
1613  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1614  * @vsi: vsi structure
1615  * @seed: RSS hash seed
1616  * @lut: pointer to lookup table of lut_size
1617  * @lut_size: size of the lookup table
1618  **/
1619 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1620                               u8 *lut, u16 lut_size)
1621 {
1622         struct i40e_pf *pf = vsi->back;
1623         struct i40e_hw *hw = &pf->hw;
1624         int ret = 0;
1625
1626         if (seed) {
1627                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1628                         (struct i40e_aqc_get_set_rss_key_data *)seed;
1629                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1630                 if (ret) {
1631                         dev_info(&pf->pdev->dev,
1632                                  "Cannot set RSS key, err %s aq_err %s\n",
1633                                  i40e_stat_str(hw, ret),
1634                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1635                         return ret;
1636                 }
1637         }
1638         if (lut) {
1639                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
1640
1641                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1642                 if (ret) {
1643                         dev_info(&pf->pdev->dev,
1644                                  "Cannot set RSS lut, err %s aq_err %s\n",
1645                                  i40e_stat_str(hw, ret),
1646                                  i40e_aq_str(hw, hw->aq.asq_last_status));
1647                         return ret;
1648                 }
1649         }
1650         return ret;
1651 }
1652
1653 /**
1654  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1655  * @vsi: VSI structure
1656  **/
1657 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1658 {
1659         struct i40e_pf *pf = vsi->back;
1660         u8 seed[I40E_HKEY_ARRAY_SIZE];
1661         u8 *lut;
1662         int ret;
1663
1664         if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1665                 return 0;
1666         if (!vsi->rss_size)
1667                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1668                                       vsi->num_queue_pairs);
1669         if (!vsi->rss_size)
1670                 return -EINVAL;
1671         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1672         if (!lut)
1673                 return -ENOMEM;
1674
1675         /* Use the user configured hash keys and lookup table if there is one,
1676          * otherwise use default
1677          */
1678         if (vsi->rss_lut_user)
1679                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1680         else
1681                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1682         if (vsi->rss_hkey_user)
1683                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1684         else
1685                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1686         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1687         kfree(lut);
1688         return ret;
1689 }
1690
1691 /**
1692  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1693  * @vsi: the VSI being configured,
1694  * @ctxt: VSI context structure
1695  * @enabled_tc: number of traffic classes to enable
1696  *
1697  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1698  **/
1699 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1700                                            struct i40e_vsi_context *ctxt,
1701                                            u8 enabled_tc)
1702 {
1703         u16 qcount = 0, max_qcount, qmap, sections = 0;
1704         int i, override_q, pow, num_qps, ret;
1705         u8 netdev_tc = 0, offset = 0;
1706
1707         if (vsi->type != I40E_VSI_MAIN)
1708                 return -EINVAL;
1709         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1710         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1711         vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1712         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1713         num_qps = vsi->mqprio_qopt.qopt.count[0];
1714
1715         /* find the next higher power-of-2 of num queue pairs */
1716         pow = ilog2(num_qps);
1717         if (!is_power_of_2(num_qps))
1718                 pow++;
1719         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1720                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1721
1722         /* Setup queue offset/count for all TCs for given VSI */
1723         max_qcount = vsi->mqprio_qopt.qopt.count[0];
1724         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1725                 /* See if the given TC is enabled for the given VSI */
1726                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1727                         offset = vsi->mqprio_qopt.qopt.offset[i];
1728                         qcount = vsi->mqprio_qopt.qopt.count[i];
1729                         if (qcount > max_qcount)
1730                                 max_qcount = qcount;
1731                         vsi->tc_config.tc_info[i].qoffset = offset;
1732                         vsi->tc_config.tc_info[i].qcount = qcount;
1733                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1734                 } else {
1735                         /* TC is not enabled so set the offset to
1736                          * default queue and allocate one queue
1737                          * for the given TC.
1738                          */
1739                         vsi->tc_config.tc_info[i].qoffset = 0;
1740                         vsi->tc_config.tc_info[i].qcount = 1;
1741                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1742                 }
1743         }
1744
1745         /* Set actual Tx/Rx queue pairs */
1746         vsi->num_queue_pairs = offset + qcount;
1747
1748         /* Setup queue TC[0].qmap for given VSI context */
1749         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1750         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1751         ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1752         ctxt->info.valid_sections |= cpu_to_le16(sections);
1753
1754         /* Reconfigure RSS for main VSI with max queue count */
1755         vsi->rss_size = max_qcount;
1756         ret = i40e_vsi_config_rss(vsi);
1757         if (ret) {
1758                 dev_info(&vsi->back->pdev->dev,
1759                          "Failed to reconfig rss for num_queues (%u)\n",
1760                          max_qcount);
1761                 return ret;
1762         }
1763         vsi->reconfig_rss = true;
1764         dev_dbg(&vsi->back->pdev->dev,
1765                 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1766
1767         /* Find queue count available for channel VSIs and starting offset
1768          * for channel VSIs
1769          */
1770         override_q = vsi->mqprio_qopt.qopt.count[0];
1771         if (override_q && override_q < vsi->num_queue_pairs) {
1772                 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1773                 vsi->next_base_queue = override_q;
1774         }
1775         return 0;
1776 }
1777
1778 /**
1779  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1780  * @vsi: the VSI being setup
1781  * @ctxt: VSI context structure
1782  * @enabled_tc: Enabled TCs bitmap
1783  * @is_add: True if called before Add VSI
1784  *
1785  * Setup VSI queue mapping for enabled traffic classes.
1786  **/
1787 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1788                                      struct i40e_vsi_context *ctxt,
1789                                      u8 enabled_tc,
1790                                      bool is_add)
1791 {
1792         struct i40e_pf *pf = vsi->back;
1793         u16 num_tc_qps = 0;
1794         u16 sections = 0;
1795         u8 netdev_tc = 0;
1796         u16 numtc = 1;
1797         u16 qcount;
1798         u8 offset;
1799         u16 qmap;
1800         int i;
1801
1802         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1803         offset = 0;
1804         /* zero out queue mapping, it will get updated on the end of the function */
1805         memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
1806
1807         if (vsi->type == I40E_VSI_MAIN) {
1808                 /* This code helps add more queue to the VSI if we have
1809                  * more cores than RSS can support, the higher cores will
1810                  * be served by ATR or other filters. Furthermore, the
1811                  * non-zero req_queue_pairs says that user requested a new
1812                  * queue count via ethtool's set_channels, so use this
1813                  * value for queues distribution across traffic classes
1814                  */
1815                 if (vsi->req_queue_pairs > 0)
1816                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1817                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1818                         vsi->num_queue_pairs = pf->num_lan_msix;
1819         }
1820
1821         /* Number of queues per enabled TC */
1822         if (vsi->type == I40E_VSI_MAIN ||
1823             (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
1824                 num_tc_qps = vsi->num_queue_pairs;
1825         else
1826                 num_tc_qps = vsi->alloc_queue_pairs;
1827
1828         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1829                 /* Find numtc from enabled TC bitmap */
1830                 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1831                         if (enabled_tc & BIT(i)) /* TC is enabled */
1832                                 numtc++;
1833                 }
1834                 if (!numtc) {
1835                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1836                         numtc = 1;
1837                 }
1838                 num_tc_qps = num_tc_qps / numtc;
1839                 num_tc_qps = min_t(int, num_tc_qps,
1840                                    i40e_pf_get_max_q_per_tc(pf));
1841         }
1842
1843         vsi->tc_config.numtc = numtc;
1844         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1845
1846         /* Do not allow use more TC queue pairs than MSI-X vectors exist */
1847         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1848                 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
1849
1850         /* Setup queue offset/count for all TCs for given VSI */
1851         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1852                 /* See if the given TC is enabled for the given VSI */
1853                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1854                         /* TC is enabled */
1855                         int pow, num_qps;
1856
1857                         switch (vsi->type) {
1858                         case I40E_VSI_MAIN:
1859                                 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
1860                                     I40E_FLAG_FD_ATR_ENABLED)) ||
1861                                     vsi->tc_config.enabled_tc != 1) {
1862                                         qcount = min_t(int, pf->alloc_rss_size,
1863                                                        num_tc_qps);
1864                                         break;
1865                                 }
1866                                 fallthrough;
1867                         case I40E_VSI_FDIR:
1868                         case I40E_VSI_SRIOV:
1869                         case I40E_VSI_VMDQ2:
1870                         default:
1871                                 qcount = num_tc_qps;
1872                                 WARN_ON(i != 0);
1873                                 break;
1874                         }
1875                         vsi->tc_config.tc_info[i].qoffset = offset;
1876                         vsi->tc_config.tc_info[i].qcount = qcount;
1877
1878                         /* find the next higher power-of-2 of num queue pairs */
1879                         num_qps = qcount;
1880                         pow = 0;
1881                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1882                                 pow++;
1883                                 num_qps >>= 1;
1884                         }
1885
1886                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1887                         qmap =
1888                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1889                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1890
1891                         offset += qcount;
1892                 } else {
1893                         /* TC is not enabled so set the offset to
1894                          * default queue and allocate one queue
1895                          * for the given TC.
1896                          */
1897                         vsi->tc_config.tc_info[i].qoffset = 0;
1898                         vsi->tc_config.tc_info[i].qcount = 1;
1899                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1900
1901                         qmap = 0;
1902                 }
1903                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1904         }
1905         /* Do not change previously set num_queue_pairs for PFs and VFs*/
1906         if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
1907             (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
1908             (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
1909                 vsi->num_queue_pairs = offset;
1910
1911         /* Scheduler section valid can only be set for ADD VSI */
1912         if (is_add) {
1913                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1914
1915                 ctxt->info.up_enable_bits = enabled_tc;
1916         }
1917         if (vsi->type == I40E_VSI_SRIOV) {
1918                 ctxt->info.mapping_flags |=
1919                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1920                 for (i = 0; i < vsi->num_queue_pairs; i++)
1921                         ctxt->info.queue_mapping[i] =
1922                                                cpu_to_le16(vsi->base_queue + i);
1923         } else {
1924                 ctxt->info.mapping_flags |=
1925                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1926                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1927         }
1928         ctxt->info.valid_sections |= cpu_to_le16(sections);
1929 }
1930
1931 /**
1932  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1933  * @netdev: the netdevice
1934  * @addr: address to add
1935  *
1936  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1937  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1938  */
1939 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1940 {
1941         struct i40e_netdev_priv *np = netdev_priv(netdev);
1942         struct i40e_vsi *vsi = np->vsi;
1943
1944         if (i40e_add_mac_filter(vsi, addr))
1945                 return 0;
1946         else
1947                 return -ENOMEM;
1948 }
1949
1950 /**
1951  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1952  * @netdev: the netdevice
1953  * @addr: address to add
1954  *
1955  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1956  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1957  */
1958 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1959 {
1960         struct i40e_netdev_priv *np = netdev_priv(netdev);
1961         struct i40e_vsi *vsi = np->vsi;
1962
1963         /* Under some circumstances, we might receive a request to delete
1964          * our own device address from our uc list. Because we store the
1965          * device address in the VSI's MAC/VLAN filter list, we need to ignore
1966          * such requests and not delete our device address from this list.
1967          */
1968         if (ether_addr_equal(addr, netdev->dev_addr))
1969                 return 0;
1970
1971         i40e_del_mac_filter(vsi, addr);
1972
1973         return 0;
1974 }
1975
1976 /**
1977  * i40e_set_rx_mode - NDO callback to set the netdev filters
1978  * @netdev: network interface device structure
1979  **/
1980 static void i40e_set_rx_mode(struct net_device *netdev)
1981 {
1982         struct i40e_netdev_priv *np = netdev_priv(netdev);
1983         struct i40e_vsi *vsi = np->vsi;
1984
1985         spin_lock_bh(&vsi->mac_filter_hash_lock);
1986
1987         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1988         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1989
1990         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1991
1992         /* check for other flag changes */
1993         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1994                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1995                 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1996         }
1997 }
1998
1999 /**
2000  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2001  * @vsi: Pointer to VSI struct
2002  * @from: Pointer to list which contains MAC filter entries - changes to
2003  *        those entries needs to be undone.
2004  *
2005  * MAC filter entries from this list were slated for deletion.
2006  **/
2007 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2008                                          struct hlist_head *from)
2009 {
2010         struct i40e_mac_filter *f;
2011         struct hlist_node *h;
2012
2013         hlist_for_each_entry_safe(f, h, from, hlist) {
2014                 u64 key = i40e_addr_to_hkey(f->macaddr);
2015
2016                 /* Move the element back into MAC filter list*/
2017                 hlist_del(&f->hlist);
2018                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2019         }
2020 }
2021
2022 /**
2023  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2024  * @vsi: Pointer to vsi struct
2025  * @from: Pointer to list which contains MAC filter entries - changes to
2026  *        those entries needs to be undone.
2027  *
2028  * MAC filter entries from this list were slated for addition.
2029  **/
2030 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2031                                          struct hlist_head *from)
2032 {
2033         struct i40e_new_mac_filter *new;
2034         struct hlist_node *h;
2035
2036         hlist_for_each_entry_safe(new, h, from, hlist) {
2037                 /* We can simply free the wrapper structure */
2038                 hlist_del(&new->hlist);
2039                 kfree(new);
2040         }
2041 }
2042
2043 /**
2044  * i40e_next_filter - Get the next non-broadcast filter from a list
2045  * @next: pointer to filter in list
2046  *
2047  * Returns the next non-broadcast filter in the list. Required so that we
2048  * ignore broadcast filters within the list, since these are not handled via
2049  * the normal firmware update path.
2050  */
2051 static
2052 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2053 {
2054         hlist_for_each_entry_continue(next, hlist) {
2055                 if (!is_broadcast_ether_addr(next->f->macaddr))
2056                         return next;
2057         }
2058
2059         return NULL;
2060 }
2061
2062 /**
2063  * i40e_update_filter_state - Update filter state based on return data
2064  * from firmware
2065  * @count: Number of filters added
2066  * @add_list: return data from fw
2067  * @add_head: pointer to first filter in current batch
2068  *
2069  * MAC filter entries from list were slated to be added to device. Returns
2070  * number of successful filters. Note that 0 does NOT mean success!
2071  **/
2072 static int
2073 i40e_update_filter_state(int count,
2074                          struct i40e_aqc_add_macvlan_element_data *add_list,
2075                          struct i40e_new_mac_filter *add_head)
2076 {
2077         int retval = 0;
2078         int i;
2079
2080         for (i = 0; i < count; i++) {
2081                 /* Always check status of each filter. We don't need to check
2082                  * the firmware return status because we pre-set the filter
2083                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2084                  * request to the adminq. Thus, if it no longer matches then
2085                  * we know the filter is active.
2086                  */
2087                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2088                         add_head->state = I40E_FILTER_FAILED;
2089                 } else {
2090                         add_head->state = I40E_FILTER_ACTIVE;
2091                         retval++;
2092                 }
2093
2094                 add_head = i40e_next_filter(add_head);
2095                 if (!add_head)
2096                         break;
2097         }
2098
2099         return retval;
2100 }
2101
2102 /**
2103  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2104  * @vsi: ptr to the VSI
2105  * @vsi_name: name to display in messages
2106  * @list: the list of filters to send to firmware
2107  * @num_del: the number of filters to delete
2108  * @retval: Set to -EIO on failure to delete
2109  *
2110  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2111  * *retval instead of a return value so that success does not force ret_val to
2112  * be set to 0. This ensures that a sequence of calls to this function
2113  * preserve the previous value of *retval on successful delete.
2114  */
2115 static
2116 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2117                           struct i40e_aqc_remove_macvlan_element_data *list,
2118                           int num_del, int *retval)
2119 {
2120         struct i40e_hw *hw = &vsi->back->hw;
2121         i40e_status aq_ret;
2122         int aq_err;
2123
2124         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
2125         aq_err = hw->aq.asq_last_status;
2126
2127         /* Explicitly ignore and do not report when firmware returns ENOENT */
2128         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
2129                 *retval = -EIO;
2130                 dev_info(&vsi->back->pdev->dev,
2131                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
2132                          vsi_name, i40e_stat_str(hw, aq_ret),
2133                          i40e_aq_str(hw, aq_err));
2134         }
2135 }
2136
2137 /**
2138  * i40e_aqc_add_filters - Request firmware to add a set of filters
2139  * @vsi: ptr to the VSI
2140  * @vsi_name: name to display in messages
2141  * @list: the list of filters to send to firmware
2142  * @add_head: Position in the add hlist
2143  * @num_add: the number of filters to add
2144  *
2145  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2146  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2147  * space for more filters.
2148  */
2149 static
2150 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2151                           struct i40e_aqc_add_macvlan_element_data *list,
2152                           struct i40e_new_mac_filter *add_head,
2153                           int num_add)
2154 {
2155         struct i40e_hw *hw = &vsi->back->hw;
2156         int aq_err, fcnt;
2157
2158         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
2159         aq_err = hw->aq.asq_last_status;
2160         fcnt = i40e_update_filter_state(num_add, list, add_head);
2161
2162         if (fcnt != num_add) {
2163                 if (vsi->type == I40E_VSI_MAIN) {
2164                         set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2165                         dev_warn(&vsi->back->pdev->dev,
2166                                  "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2167                                  i40e_aq_str(hw, aq_err), vsi_name);
2168                 } else if (vsi->type == I40E_VSI_SRIOV ||
2169                            vsi->type == I40E_VSI_VMDQ1 ||
2170                            vsi->type == I40E_VSI_VMDQ2) {
2171                         dev_warn(&vsi->back->pdev->dev,
2172                                  "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2173                                  i40e_aq_str(hw, aq_err), vsi_name, vsi_name);
2174                 } else {
2175                         dev_warn(&vsi->back->pdev->dev,
2176                                  "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2177                                  i40e_aq_str(hw, aq_err), vsi_name, vsi->type);
2178                 }
2179         }
2180 }
2181
2182 /**
2183  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2184  * @vsi: pointer to the VSI
2185  * @vsi_name: the VSI name
2186  * @f: filter data
2187  *
2188  * This function sets or clears the promiscuous broadcast flags for VLAN
2189  * filters in order to properly receive broadcast frames. Assumes that only
2190  * broadcast filters are passed.
2191  *
2192  * Returns status indicating success or failure;
2193  **/
2194 static i40e_status
2195 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2196                           struct i40e_mac_filter *f)
2197 {
2198         bool enable = f->state == I40E_FILTER_NEW;
2199         struct i40e_hw *hw = &vsi->back->hw;
2200         i40e_status aq_ret;
2201
2202         if (f->vlan == I40E_VLAN_ANY) {
2203                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2204                                                    vsi->seid,
2205                                                    enable,
2206                                                    NULL);
2207         } else {
2208                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2209                                                             vsi->seid,
2210                                                             enable,
2211                                                             f->vlan,
2212                                                             NULL);
2213         }
2214
2215         if (aq_ret) {
2216                 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2217                 dev_warn(&vsi->back->pdev->dev,
2218                          "Error %s, forcing overflow promiscuous on %s\n",
2219                          i40e_aq_str(hw, hw->aq.asq_last_status),
2220                          vsi_name);
2221         }
2222
2223         return aq_ret;
2224 }
2225
2226 /**
2227  * i40e_set_promiscuous - set promiscuous mode
2228  * @pf: board private structure
2229  * @promisc: promisc on or off
2230  *
2231  * There are different ways of setting promiscuous mode on a PF depending on
2232  * what state/environment we're in.  This identifies and sets it appropriately.
2233  * Returns 0 on success.
2234  **/
2235 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2236 {
2237         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2238         struct i40e_hw *hw = &pf->hw;
2239         i40e_status aq_ret;
2240
2241         if (vsi->type == I40E_VSI_MAIN &&
2242             pf->lan_veb != I40E_NO_VEB &&
2243             !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2244                 /* set defport ON for Main VSI instead of true promisc
2245                  * this way we will get all unicast/multicast and VLAN
2246                  * promisc behavior but will not get VF or VMDq traffic
2247                  * replicated on the Main VSI.
2248                  */
2249                 if (promisc)
2250                         aq_ret = i40e_aq_set_default_vsi(hw,
2251                                                          vsi->seid,
2252                                                          NULL);
2253                 else
2254                         aq_ret = i40e_aq_clear_default_vsi(hw,
2255                                                            vsi->seid,
2256                                                            NULL);
2257                 if (aq_ret) {
2258                         dev_info(&pf->pdev->dev,
2259                                  "Set default VSI failed, err %s, aq_err %s\n",
2260                                  i40e_stat_str(hw, aq_ret),
2261                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2262                 }
2263         } else {
2264                 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2265                                                   hw,
2266                                                   vsi->seid,
2267                                                   promisc, NULL,
2268                                                   true);
2269                 if (aq_ret) {
2270                         dev_info(&pf->pdev->dev,
2271                                  "set unicast promisc failed, err %s, aq_err %s\n",
2272                                  i40e_stat_str(hw, aq_ret),
2273                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2274                 }
2275                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2276                                                   hw,
2277                                                   vsi->seid,
2278                                                   promisc, NULL);
2279                 if (aq_ret) {
2280                         dev_info(&pf->pdev->dev,
2281                                  "set multicast promisc failed, err %s, aq_err %s\n",
2282                                  i40e_stat_str(hw, aq_ret),
2283                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2284                 }
2285         }
2286
2287         if (!aq_ret)
2288                 pf->cur_promisc = promisc;
2289
2290         return aq_ret;
2291 }
2292
2293 /**
2294  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2295  * @vsi: ptr to the VSI
2296  *
2297  * Push any outstanding VSI filter changes through the AdminQ.
2298  *
2299  * Returns 0 or error value
2300  **/
2301 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2302 {
2303         struct hlist_head tmp_add_list, tmp_del_list;
2304         struct i40e_mac_filter *f;
2305         struct i40e_new_mac_filter *new, *add_head = NULL;
2306         struct i40e_hw *hw = &vsi->back->hw;
2307         bool old_overflow, new_overflow;
2308         unsigned int failed_filters = 0;
2309         unsigned int vlan_filters = 0;
2310         char vsi_name[16] = "PF";
2311         int filter_list_len = 0;
2312         i40e_status aq_ret = 0;
2313         u32 changed_flags = 0;
2314         struct hlist_node *h;
2315         struct i40e_pf *pf;
2316         int num_add = 0;
2317         int num_del = 0;
2318         int retval = 0;
2319         u16 cmd_flags;
2320         int list_size;
2321         int bkt;
2322
2323         /* empty array typed pointers, kcalloc later */
2324         struct i40e_aqc_add_macvlan_element_data *add_list;
2325         struct i40e_aqc_remove_macvlan_element_data *del_list;
2326
2327         while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2328                 usleep_range(1000, 2000);
2329         pf = vsi->back;
2330
2331         old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2332
2333         if (vsi->netdev) {
2334                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2335                 vsi->current_netdev_flags = vsi->netdev->flags;
2336         }
2337
2338         INIT_HLIST_HEAD(&tmp_add_list);
2339         INIT_HLIST_HEAD(&tmp_del_list);
2340
2341         if (vsi->type == I40E_VSI_SRIOV)
2342                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2343         else if (vsi->type != I40E_VSI_MAIN)
2344                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2345
2346         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2347                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2348
2349                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2350                 /* Create a list of filters to delete. */
2351                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2352                         if (f->state == I40E_FILTER_REMOVE) {
2353                                 /* Move the element into temporary del_list */
2354                                 hash_del(&f->hlist);
2355                                 hlist_add_head(&f->hlist, &tmp_del_list);
2356
2357                                 /* Avoid counting removed filters */
2358                                 continue;
2359                         }
2360                         if (f->state == I40E_FILTER_NEW) {
2361                                 /* Create a temporary i40e_new_mac_filter */
2362                                 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2363                                 if (!new)
2364                                         goto err_no_memory_locked;
2365
2366                                 /* Store pointer to the real filter */
2367                                 new->f = f;
2368                                 new->state = f->state;
2369
2370                                 /* Add it to the hash list */
2371                                 hlist_add_head(&new->hlist, &tmp_add_list);
2372                         }
2373
2374                         /* Count the number of active (current and new) VLAN
2375                          * filters we have now. Does not count filters which
2376                          * are marked for deletion.
2377                          */
2378                         if (f->vlan > 0)
2379                                 vlan_filters++;
2380                 }
2381
2382                 retval = i40e_correct_mac_vlan_filters(vsi,
2383                                                        &tmp_add_list,
2384                                                        &tmp_del_list,
2385                                                        vlan_filters);
2386                 if (retval)
2387                         goto err_no_memory_locked;
2388
2389                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2390         }
2391
2392         /* Now process 'del_list' outside the lock */
2393         if (!hlist_empty(&tmp_del_list)) {
2394                 filter_list_len = hw->aq.asq_buf_size /
2395                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2396                 list_size = filter_list_len *
2397                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2398                 del_list = kzalloc(list_size, GFP_ATOMIC);
2399                 if (!del_list)
2400                         goto err_no_memory;
2401
2402                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2403                         cmd_flags = 0;
2404
2405                         /* handle broadcast filters by updating the broadcast
2406                          * promiscuous flag and release filter list.
2407                          */
2408                         if (is_broadcast_ether_addr(f->macaddr)) {
2409                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2410
2411                                 hlist_del(&f->hlist);
2412                                 kfree(f);
2413                                 continue;
2414                         }
2415
2416                         /* add to delete list */
2417                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2418                         if (f->vlan == I40E_VLAN_ANY) {
2419                                 del_list[num_del].vlan_tag = 0;
2420                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2421                         } else {
2422                                 del_list[num_del].vlan_tag =
2423                                         cpu_to_le16((u16)(f->vlan));
2424                         }
2425
2426                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2427                         del_list[num_del].flags = cmd_flags;
2428                         num_del++;
2429
2430                         /* flush a full buffer */
2431                         if (num_del == filter_list_len) {
2432                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2433                                                      num_del, &retval);
2434                                 memset(del_list, 0, list_size);
2435                                 num_del = 0;
2436                         }
2437                         /* Release memory for MAC filter entries which were
2438                          * synced up with HW.
2439                          */
2440                         hlist_del(&f->hlist);
2441                         kfree(f);
2442                 }
2443
2444                 if (num_del) {
2445                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2446                                              num_del, &retval);
2447                 }
2448
2449                 kfree(del_list);
2450                 del_list = NULL;
2451         }
2452
2453         if (!hlist_empty(&tmp_add_list)) {
2454                 /* Do all the adds now. */
2455                 filter_list_len = hw->aq.asq_buf_size /
2456                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2457                 list_size = filter_list_len *
2458                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2459                 add_list = kzalloc(list_size, GFP_ATOMIC);
2460                 if (!add_list)
2461                         goto err_no_memory;
2462
2463                 num_add = 0;
2464                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2465                         /* handle broadcast filters by updating the broadcast
2466                          * promiscuous flag instead of adding a MAC filter.
2467                          */
2468                         if (is_broadcast_ether_addr(new->f->macaddr)) {
2469                                 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2470                                                               new->f))
2471                                         new->state = I40E_FILTER_FAILED;
2472                                 else
2473                                         new->state = I40E_FILTER_ACTIVE;
2474                                 continue;
2475                         }
2476
2477                         /* add to add array */
2478                         if (num_add == 0)
2479                                 add_head = new;
2480                         cmd_flags = 0;
2481                         ether_addr_copy(add_list[num_add].mac_addr,
2482                                         new->f->macaddr);
2483                         if (new->f->vlan == I40E_VLAN_ANY) {
2484                                 add_list[num_add].vlan_tag = 0;
2485                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2486                         } else {
2487                                 add_list[num_add].vlan_tag =
2488                                         cpu_to_le16((u16)(new->f->vlan));
2489                         }
2490                         add_list[num_add].queue_number = 0;
2491                         /* set invalid match method for later detection */
2492                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2493                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2494                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2495                         num_add++;
2496
2497                         /* flush a full buffer */
2498                         if (num_add == filter_list_len) {
2499                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2500                                                      add_head, num_add);
2501                                 memset(add_list, 0, list_size);
2502                                 num_add = 0;
2503                         }
2504                 }
2505                 if (num_add) {
2506                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2507                                              num_add);
2508                 }
2509                 /* Now move all of the filters from the temp add list back to
2510                  * the VSI's list.
2511                  */
2512                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2513                 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2514                         /* Only update the state if we're still NEW */
2515                         if (new->f->state == I40E_FILTER_NEW)
2516                                 new->f->state = new->state;
2517                         hlist_del(&new->hlist);
2518                         kfree(new);
2519                 }
2520                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2521                 kfree(add_list);
2522                 add_list = NULL;
2523         }
2524
2525         /* Determine the number of active and failed filters. */
2526         spin_lock_bh(&vsi->mac_filter_hash_lock);
2527         vsi->active_filters = 0;
2528         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2529                 if (f->state == I40E_FILTER_ACTIVE)
2530                         vsi->active_filters++;
2531                 else if (f->state == I40E_FILTER_FAILED)
2532                         failed_filters++;
2533         }
2534         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2535
2536         /* Check if we are able to exit overflow promiscuous mode. We can
2537          * safely exit if we didn't just enter, we no longer have any failed
2538          * filters, and we have reduced filters below the threshold value.
2539          */
2540         if (old_overflow && !failed_filters &&
2541             vsi->active_filters < vsi->promisc_threshold) {
2542                 dev_info(&pf->pdev->dev,
2543                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2544                          vsi_name);
2545                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2546                 vsi->promisc_threshold = 0;
2547         }
2548
2549         /* if the VF is not trusted do not do promisc */
2550         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2551                 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2552                 goto out;
2553         }
2554
2555         new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2556
2557         /* If we are entering overflow promiscuous, we need to calculate a new
2558          * threshold for when we are safe to exit
2559          */
2560         if (!old_overflow && new_overflow)
2561                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2562
2563         /* check for changes in promiscuous modes */
2564         if (changed_flags & IFF_ALLMULTI) {
2565                 bool cur_multipromisc;
2566
2567                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2568                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2569                                                                vsi->seid,
2570                                                                cur_multipromisc,
2571                                                                NULL);
2572                 if (aq_ret) {
2573                         retval = i40e_aq_rc_to_posix(aq_ret,
2574                                                      hw->aq.asq_last_status);
2575                         dev_info(&pf->pdev->dev,
2576                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2577                                  vsi_name,
2578                                  i40e_stat_str(hw, aq_ret),
2579                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2580                 } else {
2581                         dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2582                                  cur_multipromisc ? "entering" : "leaving");
2583                 }
2584         }
2585
2586         if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2587                 bool cur_promisc;
2588
2589                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2590                                new_overflow);
2591                 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2592                 if (aq_ret) {
2593                         retval = i40e_aq_rc_to_posix(aq_ret,
2594                                                      hw->aq.asq_last_status);
2595                         dev_info(&pf->pdev->dev,
2596                                  "Setting promiscuous %s failed on %s, err %s aq_err %s\n",
2597                                  cur_promisc ? "on" : "off",
2598                                  vsi_name,
2599                                  i40e_stat_str(hw, aq_ret),
2600                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2601                 }
2602         }
2603 out:
2604         /* if something went wrong then set the changed flag so we try again */
2605         if (retval)
2606                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2607
2608         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2609         return retval;
2610
2611 err_no_memory:
2612         /* Restore elements on the temporary add and delete lists */
2613         spin_lock_bh(&vsi->mac_filter_hash_lock);
2614 err_no_memory_locked:
2615         i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2616         i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2617         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2618
2619         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2620         clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2621         return -ENOMEM;
2622 }
2623
2624 /**
2625  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2626  * @pf: board private structure
2627  **/
2628 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2629 {
2630         int v;
2631
2632         if (!pf)
2633                 return;
2634         if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2635                 return;
2636         if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2637                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2638                 return;
2639         }
2640
2641         for (v = 0; v < pf->num_alloc_vsi; v++) {
2642                 if (pf->vsi[v] &&
2643                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2644                     !test_bit(__I40E_VSI_RELEASING, pf->vsi[v]->state)) {
2645                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2646
2647                         if (ret) {
2648                                 /* come back and try again later */
2649                                 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2650                                         pf->state);
2651                                 break;
2652                         }
2653                 }
2654         }
2655 }
2656
2657 /**
2658  * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2659  * @vsi: the vsi
2660  **/
2661 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2662 {
2663         if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2664                 return I40E_RXBUFFER_2048;
2665         else
2666                 return I40E_RXBUFFER_3072;
2667 }
2668
2669 /**
2670  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2671  * @netdev: network interface device structure
2672  * @new_mtu: new value for maximum frame size
2673  *
2674  * Returns 0 on success, negative on failure
2675  **/
2676 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2677 {
2678         struct i40e_netdev_priv *np = netdev_priv(netdev);
2679         struct i40e_vsi *vsi = np->vsi;
2680         struct i40e_pf *pf = vsi->back;
2681
2682         if (i40e_enabled_xdp_vsi(vsi)) {
2683                 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2684
2685                 if (frame_size > i40e_max_xdp_frame_size(vsi))
2686                         return -EINVAL;
2687         }
2688
2689         netdev_dbg(netdev, "changing MTU from %d to %d\n",
2690                    netdev->mtu, new_mtu);
2691         netdev->mtu = new_mtu;
2692         if (netif_running(netdev))
2693                 i40e_vsi_reinit_locked(vsi);
2694         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2695         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2696         return 0;
2697 }
2698
2699 /**
2700  * i40e_ioctl - Access the hwtstamp interface
2701  * @netdev: network interface device structure
2702  * @ifr: interface request data
2703  * @cmd: ioctl command
2704  **/
2705 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2706 {
2707         struct i40e_netdev_priv *np = netdev_priv(netdev);
2708         struct i40e_pf *pf = np->vsi->back;
2709
2710         switch (cmd) {
2711         case SIOCGHWTSTAMP:
2712                 return i40e_ptp_get_ts_config(pf, ifr);
2713         case SIOCSHWTSTAMP:
2714                 return i40e_ptp_set_ts_config(pf, ifr);
2715         default:
2716                 return -EOPNOTSUPP;
2717         }
2718 }
2719
2720 /**
2721  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2722  * @vsi: the vsi being adjusted
2723  **/
2724 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2725 {
2726         struct i40e_vsi_context ctxt;
2727         i40e_status ret;
2728
2729         /* Don't modify stripping options if a port VLAN is active */
2730         if (vsi->info.pvid)
2731                 return;
2732
2733         if ((vsi->info.valid_sections &
2734              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2735             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2736                 return;  /* already enabled */
2737
2738         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2739         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2740                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2741
2742         ctxt.seid = vsi->seid;
2743         ctxt.info = vsi->info;
2744         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2745         if (ret) {
2746                 dev_info(&vsi->back->pdev->dev,
2747                          "update vlan stripping failed, err %s aq_err %s\n",
2748                          i40e_stat_str(&vsi->back->hw, ret),
2749                          i40e_aq_str(&vsi->back->hw,
2750                                      vsi->back->hw.aq.asq_last_status));
2751         }
2752 }
2753
2754 /**
2755  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2756  * @vsi: the vsi being adjusted
2757  **/
2758 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2759 {
2760         struct i40e_vsi_context ctxt;
2761         i40e_status ret;
2762
2763         /* Don't modify stripping options if a port VLAN is active */
2764         if (vsi->info.pvid)
2765                 return;
2766
2767         if ((vsi->info.valid_sections &
2768              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2769             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2770              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2771                 return;  /* already disabled */
2772
2773         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2774         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2775                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2776
2777         ctxt.seid = vsi->seid;
2778         ctxt.info = vsi->info;
2779         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2780         if (ret) {
2781                 dev_info(&vsi->back->pdev->dev,
2782                          "update vlan stripping failed, err %s aq_err %s\n",
2783                          i40e_stat_str(&vsi->back->hw, ret),
2784                          i40e_aq_str(&vsi->back->hw,
2785                                      vsi->back->hw.aq.asq_last_status));
2786         }
2787 }
2788
2789 /**
2790  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2791  * @vsi: the vsi being configured
2792  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2793  *
2794  * This is a helper function for adding a new MAC/VLAN filter with the
2795  * specified VLAN for each existing MAC address already in the hash table.
2796  * This function does *not* perform any accounting to update filters based on
2797  * VLAN mode.
2798  *
2799  * NOTE: this function expects to be called while under the
2800  * mac_filter_hash_lock
2801  **/
2802 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2803 {
2804         struct i40e_mac_filter *f, *add_f;
2805         struct hlist_node *h;
2806         int bkt;
2807
2808         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2809                 if (f->state == I40E_FILTER_REMOVE)
2810                         continue;
2811                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2812                 if (!add_f) {
2813                         dev_info(&vsi->back->pdev->dev,
2814                                  "Could not add vlan filter %d for %pM\n",
2815                                  vid, f->macaddr);
2816                         return -ENOMEM;
2817                 }
2818         }
2819
2820         return 0;
2821 }
2822
2823 /**
2824  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2825  * @vsi: the VSI being configured
2826  * @vid: VLAN id to be added
2827  **/
2828 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2829 {
2830         int err;
2831
2832         if (vsi->info.pvid)
2833                 return -EINVAL;
2834
2835         /* The network stack will attempt to add VID=0, with the intention to
2836          * receive priority tagged packets with a VLAN of 0. Our HW receives
2837          * these packets by default when configured to receive untagged
2838          * packets, so we don't need to add a filter for this case.
2839          * Additionally, HW interprets adding a VID=0 filter as meaning to
2840          * receive *only* tagged traffic and stops receiving untagged traffic.
2841          * Thus, we do not want to actually add a filter for VID=0
2842          */
2843         if (!vid)
2844                 return 0;
2845
2846         /* Locked once because all functions invoked below iterates list*/
2847         spin_lock_bh(&vsi->mac_filter_hash_lock);
2848         err = i40e_add_vlan_all_mac(vsi, vid);
2849         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2850         if (err)
2851                 return err;
2852
2853         /* schedule our worker thread which will take care of
2854          * applying the new filter changes
2855          */
2856         i40e_service_event_schedule(vsi->back);
2857         return 0;
2858 }
2859
2860 /**
2861  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2862  * @vsi: the vsi being configured
2863  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2864  *
2865  * This function should be used to remove all VLAN filters which match the
2866  * given VID. It does not schedule the service event and does not take the
2867  * mac_filter_hash_lock so it may be combined with other operations under
2868  * a single invocation of the mac_filter_hash_lock.
2869  *
2870  * NOTE: this function expects to be called while under the
2871  * mac_filter_hash_lock
2872  */
2873 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2874 {
2875         struct i40e_mac_filter *f;
2876         struct hlist_node *h;
2877         int bkt;
2878
2879         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2880                 if (f->vlan == vid)
2881                         __i40e_del_filter(vsi, f);
2882         }
2883 }
2884
2885 /**
2886  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2887  * @vsi: the VSI being configured
2888  * @vid: VLAN id to be removed
2889  **/
2890 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2891 {
2892         if (!vid || vsi->info.pvid)
2893                 return;
2894
2895         spin_lock_bh(&vsi->mac_filter_hash_lock);
2896         i40e_rm_vlan_all_mac(vsi, vid);
2897         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2898
2899         /* schedule our worker thread which will take care of
2900          * applying the new filter changes
2901          */
2902         i40e_service_event_schedule(vsi->back);
2903 }
2904
2905 /**
2906  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2907  * @netdev: network interface to be adjusted
2908  * @proto: unused protocol value
2909  * @vid: vlan id to be added
2910  *
2911  * net_device_ops implementation for adding vlan ids
2912  **/
2913 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2914                                 __always_unused __be16 proto, u16 vid)
2915 {
2916         struct i40e_netdev_priv *np = netdev_priv(netdev);
2917         struct i40e_vsi *vsi = np->vsi;
2918         int ret = 0;
2919
2920         if (vid >= VLAN_N_VID)
2921                 return -EINVAL;
2922
2923         ret = i40e_vsi_add_vlan(vsi, vid);
2924         if (!ret)
2925                 set_bit(vid, vsi->active_vlans);
2926
2927         return ret;
2928 }
2929
2930 /**
2931  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
2932  * @netdev: network interface to be adjusted
2933  * @proto: unused protocol value
2934  * @vid: vlan id to be added
2935  **/
2936 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
2937                                     __always_unused __be16 proto, u16 vid)
2938 {
2939         struct i40e_netdev_priv *np = netdev_priv(netdev);
2940         struct i40e_vsi *vsi = np->vsi;
2941
2942         if (vid >= VLAN_N_VID)
2943                 return;
2944         set_bit(vid, vsi->active_vlans);
2945 }
2946
2947 /**
2948  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2949  * @netdev: network interface to be adjusted
2950  * @proto: unused protocol value
2951  * @vid: vlan id to be removed
2952  *
2953  * net_device_ops implementation for removing vlan ids
2954  **/
2955 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2956                                  __always_unused __be16 proto, u16 vid)
2957 {
2958         struct i40e_netdev_priv *np = netdev_priv(netdev);
2959         struct i40e_vsi *vsi = np->vsi;
2960
2961         /* return code is ignored as there is nothing a user
2962          * can do about failure to remove and a log message was
2963          * already printed from the other function
2964          */
2965         i40e_vsi_kill_vlan(vsi, vid);
2966
2967         clear_bit(vid, vsi->active_vlans);
2968
2969         return 0;
2970 }
2971
2972 /**
2973  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2974  * @vsi: the vsi being brought back up
2975  **/
2976 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2977 {
2978         u16 vid;
2979
2980         if (!vsi->netdev)
2981                 return;
2982
2983         if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2984                 i40e_vlan_stripping_enable(vsi);
2985         else
2986                 i40e_vlan_stripping_disable(vsi);
2987
2988         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2989                 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
2990                                         vid);
2991 }
2992
2993 /**
2994  * i40e_vsi_add_pvid - Add pvid for the VSI
2995  * @vsi: the vsi being adjusted
2996  * @vid: the vlan id to set as a PVID
2997  **/
2998 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2999 {
3000         struct i40e_vsi_context ctxt;
3001         i40e_status ret;
3002
3003         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3004         vsi->info.pvid = cpu_to_le16(vid);
3005         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3006                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
3007                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
3008
3009         ctxt.seid = vsi->seid;
3010         ctxt.info = vsi->info;
3011         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3012         if (ret) {
3013                 dev_info(&vsi->back->pdev->dev,
3014                          "add pvid failed, err %s aq_err %s\n",
3015                          i40e_stat_str(&vsi->back->hw, ret),
3016                          i40e_aq_str(&vsi->back->hw,
3017                                      vsi->back->hw.aq.asq_last_status));
3018                 return -ENOENT;
3019         }
3020
3021         return 0;
3022 }
3023
3024 /**
3025  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3026  * @vsi: the vsi being adjusted
3027  *
3028  * Just use the vlan_rx_register() service to put it back to normal
3029  **/
3030 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3031 {
3032         vsi->info.pvid = 0;
3033
3034         i40e_vlan_stripping_disable(vsi);
3035 }
3036
3037 /**
3038  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3039  * @vsi: ptr to the VSI
3040  *
3041  * If this function returns with an error, then it's possible one or
3042  * more of the rings is populated (while the rest are not).  It is the
3043  * callers duty to clean those orphaned rings.
3044  *
3045  * Return 0 on success, negative on failure
3046  **/
3047 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3048 {
3049         int i, err = 0;
3050
3051         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3052                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3053
3054         if (!i40e_enabled_xdp_vsi(vsi))
3055                 return err;
3056
3057         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3058                 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3059
3060         return err;
3061 }
3062
3063 /**
3064  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3065  * @vsi: ptr to the VSI
3066  *
3067  * Free VSI's transmit software resources
3068  **/
3069 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3070 {
3071         int i;
3072
3073         if (vsi->tx_rings) {
3074                 for (i = 0; i < vsi->num_queue_pairs; i++)
3075                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3076                                 i40e_free_tx_resources(vsi->tx_rings[i]);
3077         }
3078
3079         if (vsi->xdp_rings) {
3080                 for (i = 0; i < vsi->num_queue_pairs; i++)
3081                         if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3082                                 i40e_free_tx_resources(vsi->xdp_rings[i]);
3083         }
3084 }
3085
3086 /**
3087  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3088  * @vsi: ptr to the VSI
3089  *
3090  * If this function returns with an error, then it's possible one or
3091  * more of the rings is populated (while the rest are not).  It is the
3092  * callers duty to clean those orphaned rings.
3093  *
3094  * Return 0 on success, negative on failure
3095  **/
3096 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3097 {
3098         int i, err = 0;
3099
3100         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3101                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3102         return err;
3103 }
3104
3105 /**
3106  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3107  * @vsi: ptr to the VSI
3108  *
3109  * Free all receive software resources
3110  **/
3111 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3112 {
3113         int i;
3114
3115         if (!vsi->rx_rings)
3116                 return;
3117
3118         for (i = 0; i < vsi->num_queue_pairs; i++)
3119                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3120                         i40e_free_rx_resources(vsi->rx_rings[i]);
3121 }
3122
3123 /**
3124  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3125  * @ring: The Tx ring to configure
3126  *
3127  * This enables/disables XPS for a given Tx descriptor ring
3128  * based on the TCs enabled for the VSI that ring belongs to.
3129  **/
3130 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3131 {
3132         int cpu;
3133
3134         if (!ring->q_vector || !ring->netdev || ring->ch)
3135                 return;
3136
3137         /* We only initialize XPS once, so as not to overwrite user settings */
3138         if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3139                 return;
3140
3141         cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3142         netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3143                             ring->queue_index);
3144 }
3145
3146 /**
3147  * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3148  * @ring: The Tx or Rx ring
3149  *
3150  * Returns the AF_XDP buffer pool or NULL.
3151  **/
3152 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3153 {
3154         bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3155         int qid = ring->queue_index;
3156
3157         if (ring_is_xdp(ring))
3158                 qid -= ring->vsi->alloc_queue_pairs;
3159
3160         if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3161                 return NULL;
3162
3163         return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3164 }
3165
3166 /**
3167  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3168  * @ring: The Tx ring to configure
3169  *
3170  * Configure the Tx descriptor ring in the HMC context.
3171  **/
3172 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3173 {
3174         struct i40e_vsi *vsi = ring->vsi;
3175         u16 pf_q = vsi->base_queue + ring->queue_index;
3176         struct i40e_hw *hw = &vsi->back->hw;
3177         struct i40e_hmc_obj_txq tx_ctx;
3178         i40e_status err = 0;
3179         u32 qtx_ctl = 0;
3180
3181         if (ring_is_xdp(ring))
3182                 ring->xsk_pool = i40e_xsk_pool(ring);
3183
3184         /* some ATR related tx ring init */
3185         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3186                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
3187                 ring->atr_count = 0;
3188         } else {
3189                 ring->atr_sample_rate = 0;
3190         }
3191
3192         /* configure XPS */
3193         i40e_config_xps_tx_ring(ring);
3194
3195         /* clear the context structure first */
3196         memset(&tx_ctx, 0, sizeof(tx_ctx));
3197
3198         tx_ctx.new_context = 1;
3199         tx_ctx.base = (ring->dma / 128);
3200         tx_ctx.qlen = ring->count;
3201         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3202                                                I40E_FLAG_FD_ATR_ENABLED));
3203         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3204         /* FDIR VSI tx ring can still use RS bit and writebacks */
3205         if (vsi->type != I40E_VSI_FDIR)
3206                 tx_ctx.head_wb_ena = 1;
3207         tx_ctx.head_wb_addr = ring->dma +
3208                               (ring->count * sizeof(struct i40e_tx_desc));
3209
3210         /* As part of VSI creation/update, FW allocates certain
3211          * Tx arbitration queue sets for each TC enabled for
3212          * the VSI. The FW returns the handles to these queue
3213          * sets as part of the response buffer to Add VSI,
3214          * Update VSI, etc. AQ commands. It is expected that
3215          * these queue set handles be associated with the Tx
3216          * queues by the driver as part of the TX queue context
3217          * initialization. This has to be done regardless of
3218          * DCB as by default everything is mapped to TC0.
3219          */
3220
3221         if (ring->ch)
3222                 tx_ctx.rdylist =
3223                         le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3224
3225         else
3226                 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3227
3228         tx_ctx.rdylist_act = 0;
3229
3230         /* clear the context in the HMC */
3231         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3232         if (err) {
3233                 dev_info(&vsi->back->pdev->dev,
3234                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3235                          ring->queue_index, pf_q, err);
3236                 return -ENOMEM;
3237         }
3238
3239         /* set the context in the HMC */
3240         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3241         if (err) {
3242                 dev_info(&vsi->back->pdev->dev,
3243                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3244                          ring->queue_index, pf_q, err);
3245                 return -ENOMEM;
3246         }
3247
3248         /* Now associate this queue with this PCI function */
3249         if (ring->ch) {
3250                 if (ring->ch->type == I40E_VSI_VMDQ2)
3251                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3252                 else
3253                         return -EINVAL;
3254
3255                 qtx_ctl |= (ring->ch->vsi_number <<
3256                             I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3257                             I40E_QTX_CTL_VFVM_INDX_MASK;
3258         } else {
3259                 if (vsi->type == I40E_VSI_VMDQ2) {
3260                         qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3261                         qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3262                                     I40E_QTX_CTL_VFVM_INDX_MASK;
3263                 } else {
3264                         qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3265                 }
3266         }
3267
3268         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3269                     I40E_QTX_CTL_PF_INDX_MASK);
3270         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3271         i40e_flush(hw);
3272
3273         /* cache tail off for easier writes later */
3274         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3275
3276         return 0;
3277 }
3278
3279 /**
3280  * i40e_rx_offset - Return expected offset into page to access data
3281  * @rx_ring: Ring we are requesting offset of
3282  *
3283  * Returns the offset value for ring into the data buffer.
3284  */
3285 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
3286 {
3287         return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
3288 }
3289
3290 /**
3291  * i40e_configure_rx_ring - Configure a receive ring context
3292  * @ring: The Rx ring to configure
3293  *
3294  * Configure the Rx descriptor ring in the HMC context.
3295  **/
3296 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3297 {
3298         struct i40e_vsi *vsi = ring->vsi;
3299         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3300         u16 pf_q = vsi->base_queue + ring->queue_index;
3301         struct i40e_hw *hw = &vsi->back->hw;
3302         struct i40e_hmc_obj_rxq rx_ctx;
3303         i40e_status err = 0;
3304         bool ok;
3305         int ret;
3306
3307         bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3308
3309         /* clear the context structure first */
3310         memset(&rx_ctx, 0, sizeof(rx_ctx));
3311
3312         if (ring->vsi->type == I40E_VSI_MAIN)
3313                 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
3314
3315         kfree(ring->rx_bi);
3316         ring->xsk_pool = i40e_xsk_pool(ring);
3317         if (ring->xsk_pool) {
3318                 ret = i40e_alloc_rx_bi_zc(ring);
3319                 if (ret)
3320                         return ret;
3321                 ring->rx_buf_len =
3322                   xsk_pool_get_rx_frame_size(ring->xsk_pool);
3323                 /* For AF_XDP ZC, we disallow packets to span on
3324                  * multiple buffers, thus letting us skip that
3325                  * handling in the fast-path.
3326                  */
3327                 chain_len = 1;
3328                 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3329                                                  MEM_TYPE_XSK_BUFF_POOL,
3330                                                  NULL);
3331                 if (ret)
3332                         return ret;
3333                 dev_info(&vsi->back->pdev->dev,
3334                          "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3335                          ring->queue_index);
3336
3337         } else {
3338                 ret = i40e_alloc_rx_bi(ring);
3339                 if (ret)
3340                         return ret;
3341                 ring->rx_buf_len = vsi->rx_buf_len;
3342                 if (ring->vsi->type == I40E_VSI_MAIN) {
3343                         ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3344                                                          MEM_TYPE_PAGE_SHARED,
3345                                                          NULL);
3346                         if (ret)
3347                                 return ret;
3348                 }
3349         }
3350
3351         rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3352                                     BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3353
3354         rx_ctx.base = (ring->dma / 128);
3355         rx_ctx.qlen = ring->count;
3356
3357         /* use 16 byte descriptors */
3358         rx_ctx.dsize = 0;
3359
3360         /* descriptor type is always zero
3361          * rx_ctx.dtype = 0;
3362          */
3363         rx_ctx.hsplit_0 = 0;
3364
3365         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3366         if (hw->revision_id == 0)
3367                 rx_ctx.lrxqthresh = 0;
3368         else
3369                 rx_ctx.lrxqthresh = 1;
3370         rx_ctx.crcstrip = 1;
3371         rx_ctx.l2tsel = 1;
3372         /* this controls whether VLAN is stripped from inner headers */
3373         rx_ctx.showiv = 0;
3374         /* set the prefena field to 1 because the manual says to */
3375         rx_ctx.prefena = 1;
3376
3377         /* clear the context in the HMC */
3378         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3379         if (err) {
3380                 dev_info(&vsi->back->pdev->dev,
3381                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3382                          ring->queue_index, pf_q, err);
3383                 return -ENOMEM;
3384         }
3385
3386         /* set the context in the HMC */
3387         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3388         if (err) {
3389                 dev_info(&vsi->back->pdev->dev,
3390                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3391                          ring->queue_index, pf_q, err);
3392                 return -ENOMEM;
3393         }
3394
3395         /* configure Rx buffer alignment */
3396         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3397                 clear_ring_build_skb_enabled(ring);
3398         else
3399                 set_ring_build_skb_enabled(ring);
3400
3401         ring->rx_offset = i40e_rx_offset(ring);
3402
3403         /* cache tail for quicker writes, and clear the reg before use */
3404         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3405         writel(0, ring->tail);
3406
3407         if (ring->xsk_pool) {
3408                 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3409                 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3410         } else {
3411                 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3412         }
3413         if (!ok) {
3414                 /* Log this in case the user has forgotten to give the kernel
3415                  * any buffers, even later in the application.
3416                  */
3417                 dev_info(&vsi->back->pdev->dev,
3418                          "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3419                          ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3420                          ring->queue_index, pf_q);
3421         }
3422
3423         return 0;
3424 }
3425
3426 /**
3427  * i40e_vsi_configure_tx - Configure the VSI for Tx
3428  * @vsi: VSI structure describing this set of rings and resources
3429  *
3430  * Configure the Tx VSI for operation.
3431  **/
3432 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3433 {
3434         int err = 0;
3435         u16 i;
3436
3437         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3438                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3439
3440         if (err || !i40e_enabled_xdp_vsi(vsi))
3441                 return err;
3442
3443         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3444                 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3445
3446         return err;
3447 }
3448
3449 /**
3450  * i40e_vsi_configure_rx - Configure the VSI for Rx
3451  * @vsi: the VSI being configured
3452  *
3453  * Configure the Rx VSI for operation.
3454  **/
3455 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3456 {
3457         int err = 0;
3458         u16 i;
3459
3460         if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3461                 vsi->max_frame = I40E_MAX_RXBUFFER;
3462                 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3463 #if (PAGE_SIZE < 8192)
3464         } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3465                    (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3466                 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3467                 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3468 #endif
3469         } else {
3470                 vsi->max_frame = I40E_MAX_RXBUFFER;
3471                 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3472                                                        I40E_RXBUFFER_2048;
3473         }
3474
3475         /* set up individual rings */
3476         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3477                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3478
3479         return err;
3480 }
3481
3482 /**
3483  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3484  * @vsi: ptr to the VSI
3485  **/
3486 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3487 {
3488         struct i40e_ring *tx_ring, *rx_ring;
3489         u16 qoffset, qcount;
3490         int i, n;
3491
3492         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3493                 /* Reset the TC information */
3494                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3495                         rx_ring = vsi->rx_rings[i];
3496                         tx_ring = vsi->tx_rings[i];
3497                         rx_ring->dcb_tc = 0;
3498                         tx_ring->dcb_tc = 0;
3499                 }
3500                 return;
3501         }
3502
3503         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3504                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3505                         continue;
3506
3507                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3508                 qcount = vsi->tc_config.tc_info[n].qcount;
3509                 for (i = qoffset; i < (qoffset + qcount); i++) {
3510                         rx_ring = vsi->rx_rings[i];
3511                         tx_ring = vsi->tx_rings[i];
3512                         rx_ring->dcb_tc = n;
3513                         tx_ring->dcb_tc = n;
3514                 }
3515         }
3516 }
3517
3518 /**
3519  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3520  * @vsi: ptr to the VSI
3521  **/
3522 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3523 {
3524         if (vsi->netdev)
3525                 i40e_set_rx_mode(vsi->netdev);
3526 }
3527
3528 /**
3529  * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3530  * @pf: Pointer to the targeted PF
3531  *
3532  * Set all flow director counters to 0.
3533  */
3534 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3535 {
3536         pf->fd_tcp4_filter_cnt = 0;
3537         pf->fd_udp4_filter_cnt = 0;
3538         pf->fd_sctp4_filter_cnt = 0;
3539         pf->fd_ip4_filter_cnt = 0;
3540         pf->fd_tcp6_filter_cnt = 0;
3541         pf->fd_udp6_filter_cnt = 0;
3542         pf->fd_sctp6_filter_cnt = 0;
3543         pf->fd_ip6_filter_cnt = 0;
3544 }
3545
3546 /**
3547  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3548  * @vsi: Pointer to the targeted VSI
3549  *
3550  * This function replays the hlist on the hw where all the SB Flow Director
3551  * filters were saved.
3552  **/
3553 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3554 {
3555         struct i40e_fdir_filter *filter;
3556         struct i40e_pf *pf = vsi->back;
3557         struct hlist_node *node;
3558
3559         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3560                 return;
3561
3562         /* Reset FDir counters as we're replaying all existing filters */
3563         i40e_reset_fdir_filter_cnt(pf);
3564
3565         hlist_for_each_entry_safe(filter, node,
3566                                   &pf->fdir_filter_list, fdir_node) {
3567                 i40e_add_del_fdir(vsi, filter, true);
3568         }
3569 }
3570
3571 /**
3572  * i40e_vsi_configure - Set up the VSI for action
3573  * @vsi: the VSI being configured
3574  **/
3575 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3576 {
3577         int err;
3578
3579         i40e_set_vsi_rx_mode(vsi);
3580         i40e_restore_vlan(vsi);
3581         i40e_vsi_config_dcb_rings(vsi);
3582         err = i40e_vsi_configure_tx(vsi);
3583         if (!err)
3584                 err = i40e_vsi_configure_rx(vsi);
3585
3586         return err;
3587 }
3588
3589 /**
3590  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3591  * @vsi: the VSI being configured
3592  **/
3593 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3594 {
3595         bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3596         struct i40e_pf *pf = vsi->back;
3597         struct i40e_hw *hw = &pf->hw;
3598         u16 vector;
3599         int i, q;
3600         u32 qp;
3601
3602         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3603          * and PFINT_LNKLSTn registers, e.g.:
3604          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3605          */
3606         qp = vsi->base_queue;
3607         vector = vsi->base_vector;
3608         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3609                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3610
3611                 q_vector->rx.next_update = jiffies + 1;
3612                 q_vector->rx.target_itr =
3613                         ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3614                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3615                      q_vector->rx.target_itr >> 1);
3616                 q_vector->rx.current_itr = q_vector->rx.target_itr;
3617
3618                 q_vector->tx.next_update = jiffies + 1;
3619                 q_vector->tx.target_itr =
3620                         ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3621                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3622                      q_vector->tx.target_itr >> 1);
3623                 q_vector->tx.current_itr = q_vector->tx.target_itr;
3624
3625                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3626                      i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3627
3628                 /* Linked list for the queuepairs assigned to this vector */
3629                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3630                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3631                         u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3632                         u32 val;
3633
3634                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3635                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3636                               (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3637                               (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3638                               (I40E_QUEUE_TYPE_TX <<
3639                                I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3640
3641                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3642
3643                         if (has_xdp) {
3644                                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3645                                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3646                                       (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3647                                       (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3648                                       (I40E_QUEUE_TYPE_TX <<
3649                                        I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3650
3651                                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3652                         }
3653
3654                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3655                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3656                               (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3657                               ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3658                               (I40E_QUEUE_TYPE_RX <<
3659                                I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3660
3661                         /* Terminate the linked list */
3662                         if (q == (q_vector->num_ringpairs - 1))
3663                                 val |= (I40E_QUEUE_END_OF_LIST <<
3664                                         I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3665
3666                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3667                         qp++;
3668                 }
3669         }
3670
3671         i40e_flush(hw);
3672 }
3673
3674 /**
3675  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3676  * @pf: pointer to private device data structure
3677  **/
3678 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3679 {
3680         struct i40e_hw *hw = &pf->hw;
3681         u32 val;
3682
3683         /* clear things first */
3684         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3685         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3686
3687         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3688               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3689               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3690               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3691               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3692               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3693               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3694               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3695
3696         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3697                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3698
3699         if (pf->flags & I40E_FLAG_PTP)
3700                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3701
3702         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3703
3704         /* SW_ITR_IDX = 0, but don't change INTENA */
3705         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3706                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3707
3708         /* OTHER_ITR_IDX = 0 */
3709         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3710 }
3711
3712 /**
3713  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3714  * @vsi: the VSI being configured
3715  **/
3716 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3717 {
3718         u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3719         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3720         struct i40e_pf *pf = vsi->back;
3721         struct i40e_hw *hw = &pf->hw;
3722         u32 val;
3723
3724         /* set the ITR configuration */
3725         q_vector->rx.next_update = jiffies + 1;
3726         q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3727         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
3728         q_vector->rx.current_itr = q_vector->rx.target_itr;
3729         q_vector->tx.next_update = jiffies + 1;
3730         q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3731         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
3732         q_vector->tx.current_itr = q_vector->tx.target_itr;
3733
3734         i40e_enable_misc_int_causes(pf);
3735
3736         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3737         wr32(hw, I40E_PFINT_LNKLST0, 0);
3738
3739         /* Associate the queue pair to the vector and enable the queue int */
3740         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                   |
3741               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3742               (nextqp      << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3743               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3744
3745         wr32(hw, I40E_QINT_RQCTL(0), val);
3746
3747         if (i40e_enabled_xdp_vsi(vsi)) {
3748                 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                 |
3749                       (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3750                       (I40E_QUEUE_TYPE_TX
3751                        << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3752
3753                 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3754         }
3755
3756         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3757               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3758               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3759
3760         wr32(hw, I40E_QINT_TQCTL(0), val);
3761         i40e_flush(hw);
3762 }
3763
3764 /**
3765  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3766  * @pf: board private structure
3767  **/
3768 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3769 {
3770         struct i40e_hw *hw = &pf->hw;
3771
3772         wr32(hw, I40E_PFINT_DYN_CTL0,
3773              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3774         i40e_flush(hw);
3775 }
3776
3777 /**
3778  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3779  * @pf: board private structure
3780  **/
3781 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
3782 {
3783         struct i40e_hw *hw = &pf->hw;
3784         u32 val;
3785
3786         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3787               I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3788               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3789
3790         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3791         i40e_flush(hw);
3792 }
3793
3794 /**
3795  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3796  * @irq: interrupt number
3797  * @data: pointer to a q_vector
3798  **/
3799 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3800 {
3801         struct i40e_q_vector *q_vector = data;
3802
3803         if (!q_vector->tx.ring && !q_vector->rx.ring)
3804                 return IRQ_HANDLED;
3805
3806         napi_schedule_irqoff(&q_vector->napi);
3807
3808         return IRQ_HANDLED;
3809 }
3810
3811 /**
3812  * i40e_irq_affinity_notify - Callback for affinity changes
3813  * @notify: context as to what irq was changed
3814  * @mask: the new affinity mask
3815  *
3816  * This is a callback function used by the irq_set_affinity_notifier function
3817  * so that we may register to receive changes to the irq affinity masks.
3818  **/
3819 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3820                                      const cpumask_t *mask)
3821 {
3822         struct i40e_q_vector *q_vector =
3823                 container_of(notify, struct i40e_q_vector, affinity_notify);
3824
3825         cpumask_copy(&q_vector->affinity_mask, mask);
3826 }
3827
3828 /**
3829  * i40e_irq_affinity_release - Callback for affinity notifier release
3830  * @ref: internal core kernel usage
3831  *
3832  * This is a callback function used by the irq_set_affinity_notifier function
3833  * to inform the current notification subscriber that they will no longer
3834  * receive notifications.
3835  **/
3836 static void i40e_irq_affinity_release(struct kref *ref) {}
3837
3838 /**
3839  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3840  * @vsi: the VSI being configured
3841  * @basename: name for the vector
3842  *
3843  * Allocates MSI-X vectors and requests interrupts from the kernel.
3844  **/
3845 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3846 {
3847         int q_vectors = vsi->num_q_vectors;
3848         struct i40e_pf *pf = vsi->back;
3849         int base = vsi->base_vector;
3850         int rx_int_idx = 0;
3851         int tx_int_idx = 0;
3852         int vector, err;
3853         int irq_num;
3854         int cpu;
3855
3856         for (vector = 0; vector < q_vectors; vector++) {
3857                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3858
3859                 irq_num = pf->msix_entries[base + vector].vector;
3860
3861                 if (q_vector->tx.ring && q_vector->rx.ring) {
3862                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3863                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3864                         tx_int_idx++;
3865                 } else if (q_vector->rx.ring) {
3866                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3867                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3868                 } else if (q_vector->tx.ring) {
3869                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3870                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3871                 } else {
3872                         /* skip this unused q_vector */
3873                         continue;
3874                 }
3875                 err = request_irq(irq_num,
3876                                   vsi->irq_handler,
3877                                   0,
3878                                   q_vector->name,
3879                                   q_vector);
3880                 if (err) {
3881                         dev_info(&pf->pdev->dev,
3882                                  "MSIX request_irq failed, error: %d\n", err);
3883                         goto free_queue_irqs;
3884                 }
3885
3886                 /* register for affinity change notifications */
3887                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3888                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3889                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3890                 /* Spread affinity hints out across online CPUs.
3891                  *
3892                  * get_cpu_mask returns a static constant mask with
3893                  * a permanent lifetime so it's ok to pass to
3894                  * irq_set_affinity_hint without making a copy.
3895                  */
3896                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
3897                 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
3898         }
3899
3900         vsi->irqs_ready = true;
3901         return 0;
3902
3903 free_queue_irqs:
3904         while (vector) {
3905                 vector--;
3906                 irq_num = pf->msix_entries[base + vector].vector;
3907                 irq_set_affinity_notifier(irq_num, NULL);
3908                 irq_set_affinity_hint(irq_num, NULL);
3909                 free_irq(irq_num, &vsi->q_vectors[vector]);
3910         }
3911         return err;
3912 }
3913
3914 /**
3915  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3916  * @vsi: the VSI being un-configured
3917  **/
3918 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3919 {
3920         struct i40e_pf *pf = vsi->back;
3921         struct i40e_hw *hw = &pf->hw;
3922         int base = vsi->base_vector;
3923         int i;
3924
3925         /* disable interrupt causation from each queue */
3926         for (i = 0; i < vsi->num_queue_pairs; i++) {
3927                 u32 val;
3928
3929                 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3930                 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3931                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3932
3933                 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3934                 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3935                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3936
3937                 if (!i40e_enabled_xdp_vsi(vsi))
3938                         continue;
3939                 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3940         }
3941
3942         /* disable each interrupt */
3943         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3944                 for (i = vsi->base_vector;
3945                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3946                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3947
3948                 i40e_flush(hw);
3949                 for (i = 0; i < vsi->num_q_vectors; i++)
3950                         synchronize_irq(pf->msix_entries[i + base].vector);
3951         } else {
3952                 /* Legacy and MSI mode - this stops all interrupt handling */
3953                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3954                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3955                 i40e_flush(hw);
3956                 synchronize_irq(pf->pdev->irq);
3957         }
3958 }
3959
3960 /**
3961  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3962  * @vsi: the VSI being configured
3963  **/
3964 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3965 {
3966         struct i40e_pf *pf = vsi->back;
3967         int i;
3968
3969         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3970                 for (i = 0; i < vsi->num_q_vectors; i++)
3971                         i40e_irq_dynamic_enable(vsi, i);
3972         } else {
3973                 i40e_irq_dynamic_enable_icr0(pf);
3974         }
3975
3976         i40e_flush(&pf->hw);
3977         return 0;
3978 }
3979
3980 /**
3981  * i40e_free_misc_vector - Free the vector that handles non-queue events
3982  * @pf: board private structure
3983  **/
3984 static void i40e_free_misc_vector(struct i40e_pf *pf)
3985 {
3986         /* Disable ICR 0 */
3987         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3988         i40e_flush(&pf->hw);
3989
3990         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
3991                 synchronize_irq(pf->msix_entries[0].vector);
3992                 free_irq(pf->msix_entries[0].vector, pf);
3993                 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
3994         }
3995 }
3996
3997 /**
3998  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3999  * @irq: interrupt number
4000  * @data: pointer to a q_vector
4001  *
4002  * This is the handler used for all MSI/Legacy interrupts, and deals
4003  * with both queue and non-queue interrupts.  This is also used in
4004  * MSIX mode to handle the non-queue interrupts.
4005  **/
4006 static irqreturn_t i40e_intr(int irq, void *data)
4007 {
4008         struct i40e_pf *pf = (struct i40e_pf *)data;
4009         struct i40e_hw *hw = &pf->hw;
4010         irqreturn_t ret = IRQ_NONE;
4011         u32 icr0, icr0_remaining;
4012         u32 val, ena_mask;
4013
4014         icr0 = rd32(hw, I40E_PFINT_ICR0);
4015         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4016
4017         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
4018         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4019                 goto enable_intr;
4020
4021         /* if interrupt but no bits showing, must be SWINT */
4022         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4023             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4024                 pf->sw_int_count++;
4025
4026         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
4027             (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4028                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4029                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4030                 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4031         }
4032
4033         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4034         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4035                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
4036                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4037
4038                 /* We do not have a way to disarm Queue causes while leaving
4039                  * interrupt enabled for all other causes, ideally
4040                  * interrupt should be disabled while we are in NAPI but
4041                  * this is not a performance path and napi_schedule()
4042                  * can deal with rescheduling.
4043                  */
4044                 if (!test_bit(__I40E_DOWN, pf->state))
4045                         napi_schedule_irqoff(&q_vector->napi);
4046         }
4047
4048         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4049                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4050                 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4051                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4052         }
4053
4054         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4055                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4056                 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4057         }
4058
4059         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4060                 /* disable any further VFLR event notifications */
4061                 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4062                         u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4063
4064                         reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4065                         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4066                 } else {
4067                         ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4068                         set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4069                 }
4070         }
4071
4072         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4073                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4074                         set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4075                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4076                 val = rd32(hw, I40E_GLGEN_RSTAT);
4077                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
4078                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
4079                 if (val == I40E_RESET_CORER) {
4080                         pf->corer_count++;
4081                 } else if (val == I40E_RESET_GLOBR) {
4082                         pf->globr_count++;
4083                 } else if (val == I40E_RESET_EMPR) {
4084                         pf->empr_count++;
4085                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4086                 }
4087         }
4088
4089         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4090                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4091                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4092                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4093                          rd32(hw, I40E_PFHMC_ERRORINFO),
4094                          rd32(hw, I40E_PFHMC_ERRORDATA));
4095         }
4096
4097         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4098                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4099
4100                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)
4101                         schedule_work(&pf->ptp_extts0_work);
4102
4103                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)
4104                         i40e_ptp_tx_hwtstamp(pf);
4105
4106                 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4107         }
4108
4109         /* If a critical error is pending we have no choice but to reset the
4110          * device.
4111          * Report and mask out any remaining unexpected interrupts.
4112          */
4113         icr0_remaining = icr0 & ena_mask;
4114         if (icr0_remaining) {
4115                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4116                          icr0_remaining);
4117                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4118                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4119                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4120                         dev_info(&pf->pdev->dev, "device will be reset\n");
4121                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4122                         i40e_service_event_schedule(pf);
4123                 }
4124                 ena_mask &= ~icr0_remaining;
4125         }
4126         ret = IRQ_HANDLED;
4127
4128 enable_intr:
4129         /* re-enable interrupt causes */
4130         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4131         if (!test_bit(__I40E_DOWN, pf->state) ||
4132             test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4133                 i40e_service_event_schedule(pf);
4134                 i40e_irq_dynamic_enable_icr0(pf);
4135         }
4136
4137         return ret;
4138 }
4139
4140 /**
4141  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4142  * @tx_ring:  tx ring to clean
4143  * @budget:   how many cleans we're allowed
4144  *
4145  * Returns true if there's any budget left (e.g. the clean is finished)
4146  **/
4147 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4148 {
4149         struct i40e_vsi *vsi = tx_ring->vsi;
4150         u16 i = tx_ring->next_to_clean;
4151         struct i40e_tx_buffer *tx_buf;
4152         struct i40e_tx_desc *tx_desc;
4153
4154         tx_buf = &tx_ring->tx_bi[i];
4155         tx_desc = I40E_TX_DESC(tx_ring, i);
4156         i -= tx_ring->count;
4157
4158         do {
4159                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4160
4161                 /* if next_to_watch is not set then there is no work pending */
4162                 if (!eop_desc)
4163                         break;
4164
4165                 /* prevent any other reads prior to eop_desc */
4166                 smp_rmb();
4167
4168                 /* if the descriptor isn't done, no work yet to do */
4169                 if (!(eop_desc->cmd_type_offset_bsz &
4170                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4171                         break;
4172
4173                 /* clear next_to_watch to prevent false hangs */
4174                 tx_buf->next_to_watch = NULL;
4175
4176                 tx_desc->buffer_addr = 0;
4177                 tx_desc->cmd_type_offset_bsz = 0;
4178                 /* move past filter desc */
4179                 tx_buf++;
4180                 tx_desc++;
4181                 i++;
4182                 if (unlikely(!i)) {
4183                         i -= tx_ring->count;
4184                         tx_buf = tx_ring->tx_bi;
4185                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4186                 }
4187                 /* unmap skb header data */
4188                 dma_unmap_single(tx_ring->dev,
4189                                  dma_unmap_addr(tx_buf, dma),
4190                                  dma_unmap_len(tx_buf, len),
4191                                  DMA_TO_DEVICE);
4192                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4193                         kfree(tx_buf->raw_buf);
4194
4195                 tx_buf->raw_buf = NULL;
4196                 tx_buf->tx_flags = 0;
4197                 tx_buf->next_to_watch = NULL;
4198                 dma_unmap_len_set(tx_buf, len, 0);
4199                 tx_desc->buffer_addr = 0;
4200                 tx_desc->cmd_type_offset_bsz = 0;
4201
4202                 /* move us past the eop_desc for start of next FD desc */
4203                 tx_buf++;
4204                 tx_desc++;
4205                 i++;
4206                 if (unlikely(!i)) {
4207                         i -= tx_ring->count;
4208                         tx_buf = tx_ring->tx_bi;
4209                         tx_desc = I40E_TX_DESC(tx_ring, 0);
4210                 }
4211
4212                 /* update budget accounting */
4213                 budget--;
4214         } while (likely(budget));
4215
4216         i += tx_ring->count;
4217         tx_ring->next_to_clean = i;
4218
4219         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4220                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4221
4222         return budget > 0;
4223 }
4224
4225 /**
4226  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4227  * @irq: interrupt number
4228  * @data: pointer to a q_vector
4229  **/
4230 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4231 {
4232         struct i40e_q_vector *q_vector = data;
4233         struct i40e_vsi *vsi;
4234
4235         if (!q_vector->tx.ring)
4236                 return IRQ_HANDLED;
4237
4238         vsi = q_vector->tx.ring->vsi;
4239         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4240
4241         return IRQ_HANDLED;
4242 }
4243
4244 /**
4245  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4246  * @vsi: the VSI being configured
4247  * @v_idx: vector index
4248  * @qp_idx: queue pair index
4249  **/
4250 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4251 {
4252         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4253         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4254         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4255
4256         tx_ring->q_vector = q_vector;
4257         tx_ring->next = q_vector->tx.ring;
4258         q_vector->tx.ring = tx_ring;
4259         q_vector->tx.count++;
4260
4261         /* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4262         if (i40e_enabled_xdp_vsi(vsi)) {
4263                 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4264
4265                 xdp_ring->q_vector = q_vector;
4266                 xdp_ring->next = q_vector->tx.ring;
4267                 q_vector->tx.ring = xdp_ring;
4268                 q_vector->tx.count++;
4269         }
4270
4271         rx_ring->q_vector = q_vector;
4272         rx_ring->next = q_vector->rx.ring;
4273         q_vector->rx.ring = rx_ring;
4274         q_vector->rx.count++;
4275 }
4276
4277 /**
4278  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4279  * @vsi: the VSI being configured
4280  *
4281  * This function maps descriptor rings to the queue-specific vectors
4282  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4283  * one vector per queue pair, but on a constrained vector budget, we
4284  * group the queue pairs as "efficiently" as possible.
4285  **/
4286 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4287 {
4288         int qp_remaining = vsi->num_queue_pairs;
4289         int q_vectors = vsi->num_q_vectors;
4290         int num_ringpairs;
4291         int v_start = 0;
4292         int qp_idx = 0;
4293
4294         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4295          * group them so there are multiple queues per vector.
4296          * It is also important to go through all the vectors available to be
4297          * sure that if we don't use all the vectors, that the remaining vectors
4298          * are cleared. This is especially important when decreasing the
4299          * number of queues in use.
4300          */
4301         for (; v_start < q_vectors; v_start++) {
4302                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4303
4304                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4305
4306                 q_vector->num_ringpairs = num_ringpairs;
4307                 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4308
4309                 q_vector->rx.count = 0;
4310                 q_vector->tx.count = 0;
4311                 q_vector->rx.ring = NULL;
4312                 q_vector->tx.ring = NULL;
4313
4314                 while (num_ringpairs--) {
4315                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4316                         qp_idx++;
4317                         qp_remaining--;
4318                 }
4319         }
4320 }
4321
4322 /**
4323  * i40e_vsi_request_irq - Request IRQ from the OS
4324  * @vsi: the VSI being configured
4325  * @basename: name for the vector
4326  **/
4327 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4328 {
4329         struct i40e_pf *pf = vsi->back;
4330         int err;
4331
4332         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4333                 err = i40e_vsi_request_irq_msix(vsi, basename);
4334         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4335                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4336                                   pf->int_name, pf);
4337         else
4338                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4339                                   pf->int_name, pf);
4340
4341         if (err)
4342                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4343
4344         return err;
4345 }
4346
4347 #ifdef CONFIG_NET_POLL_CONTROLLER
4348 /**
4349  * i40e_netpoll - A Polling 'interrupt' handler
4350  * @netdev: network interface device structure
4351  *
4352  * This is used by netconsole to send skbs without having to re-enable
4353  * interrupts.  It's not called while the normal interrupt routine is executing.
4354  **/
4355 static void i40e_netpoll(struct net_device *netdev)
4356 {
4357         struct i40e_netdev_priv *np = netdev_priv(netdev);
4358         struct i40e_vsi *vsi = np->vsi;
4359         struct i40e_pf *pf = vsi->back;
4360         int i;
4361
4362         /* if interface is down do nothing */
4363         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4364                 return;
4365
4366         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4367                 for (i = 0; i < vsi->num_q_vectors; i++)
4368                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4369         } else {
4370                 i40e_intr(pf->pdev->irq, netdev);
4371         }
4372 }
4373 #endif
4374
4375 #define I40E_QTX_ENA_WAIT_COUNT 50
4376
4377 /**
4378  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4379  * @pf: the PF being configured
4380  * @pf_q: the PF queue
4381  * @enable: enable or disable state of the queue
4382  *
4383  * This routine will wait for the given Tx queue of the PF to reach the
4384  * enabled or disabled state.
4385  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4386  * multiple retries; else will return 0 in case of success.
4387  **/
4388 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4389 {
4390         int i;
4391         u32 tx_reg;
4392
4393         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4394                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4395                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4396                         break;
4397
4398                 usleep_range(10, 20);
4399         }
4400         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4401                 return -ETIMEDOUT;
4402
4403         return 0;
4404 }
4405
4406 /**
4407  * i40e_control_tx_q - Start or stop a particular Tx queue
4408  * @pf: the PF structure
4409  * @pf_q: the PF queue to configure
4410  * @enable: start or stop the queue
4411  *
4412  * This function enables or disables a single queue. Note that any delay
4413  * required after the operation is expected to be handled by the caller of
4414  * this function.
4415  **/
4416 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4417 {
4418         struct i40e_hw *hw = &pf->hw;
4419         u32 tx_reg;
4420         int i;
4421
4422         /* warn the TX unit of coming changes */
4423         i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4424         if (!enable)
4425                 usleep_range(10, 20);
4426
4427         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4428                 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4429                 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4430                     ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4431                         break;
4432                 usleep_range(1000, 2000);
4433         }
4434
4435         /* Skip if the queue is already in the requested state */
4436         if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4437                 return;
4438
4439         /* turn on/off the queue */
4440         if (enable) {
4441                 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4442                 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4443         } else {
4444                 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4445         }
4446
4447         wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4448 }
4449
4450 /**
4451  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4452  * @seid: VSI SEID
4453  * @pf: the PF structure
4454  * @pf_q: the PF queue to configure
4455  * @is_xdp: true if the queue is used for XDP
4456  * @enable: start or stop the queue
4457  **/
4458 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4459                            bool is_xdp, bool enable)
4460 {
4461         int ret;
4462
4463         i40e_control_tx_q(pf, pf_q, enable);
4464
4465         /* wait for the change to finish */
4466         ret = i40e_pf_txq_wait(pf, pf_q, enable);
4467         if (ret) {
4468                 dev_info(&pf->pdev->dev,
4469                          "VSI seid %d %sTx ring %d %sable timeout\n",
4470                          seid, (is_xdp ? "XDP " : ""), pf_q,
4471                          (enable ? "en" : "dis"));
4472         }
4473
4474         return ret;
4475 }
4476
4477 /**
4478  * i40e_vsi_enable_tx - Start a VSI's rings
4479  * @vsi: the VSI being configured
4480  **/
4481 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4482 {
4483         struct i40e_pf *pf = vsi->back;
4484         int i, pf_q, ret = 0;
4485
4486         pf_q = vsi->base_queue;
4487         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4488                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4489                                              pf_q,
4490                                              false /*is xdp*/, true);
4491                 if (ret)
4492                         break;
4493
4494                 if (!i40e_enabled_xdp_vsi(vsi))
4495                         continue;
4496
4497                 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4498                                              pf_q + vsi->alloc_queue_pairs,
4499                                              true /*is xdp*/, true);
4500                 if (ret)
4501                         break;
4502         }
4503         return ret;
4504 }
4505
4506 /**
4507  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4508  * @pf: the PF being configured
4509  * @pf_q: the PF queue
4510  * @enable: enable or disable state of the queue
4511  *
4512  * This routine will wait for the given Rx queue of the PF to reach the
4513  * enabled or disabled state.
4514  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4515  * multiple retries; else will return 0 in case of success.
4516  **/
4517 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4518 {
4519         int i;
4520         u32 rx_reg;
4521
4522         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4523                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4524                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4525                         break;
4526
4527                 usleep_range(10, 20);
4528         }
4529         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4530                 return -ETIMEDOUT;
4531
4532         return 0;
4533 }
4534
4535 /**
4536  * i40e_control_rx_q - Start or stop a particular Rx queue
4537  * @pf: the PF structure
4538  * @pf_q: the PF queue to configure
4539  * @enable: start or stop the queue
4540  *
4541  * This function enables or disables a single queue. Note that
4542  * any delay required after the operation is expected to be
4543  * handled by the caller of this function.
4544  **/
4545 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4546 {
4547         struct i40e_hw *hw = &pf->hw;
4548         u32 rx_reg;
4549         int i;
4550
4551         for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4552                 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4553                 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4554                     ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4555                         break;
4556                 usleep_range(1000, 2000);
4557         }
4558
4559         /* Skip if the queue is already in the requested state */
4560         if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4561                 return;
4562
4563         /* turn on/off the queue */
4564         if (enable)
4565                 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4566         else
4567                 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4568
4569         wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4570 }
4571
4572 /**
4573  * i40e_control_wait_rx_q
4574  * @pf: the PF structure
4575  * @pf_q: queue being configured
4576  * @enable: start or stop the rings
4577  *
4578  * This function enables or disables a single queue along with waiting
4579  * for the change to finish. The caller of this function should handle
4580  * the delays needed in the case of disabling queues.
4581  **/
4582 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4583 {
4584         int ret = 0;
4585
4586         i40e_control_rx_q(pf, pf_q, enable);
4587
4588         /* wait for the change to finish */
4589         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4590         if (ret)
4591                 return ret;
4592
4593         return ret;
4594 }
4595
4596 /**
4597  * i40e_vsi_enable_rx - Start a VSI's rings
4598  * @vsi: the VSI being configured
4599  **/
4600 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4601 {
4602         struct i40e_pf *pf = vsi->back;
4603         int i, pf_q, ret = 0;
4604
4605         pf_q = vsi->base_queue;
4606         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4607                 ret = i40e_control_wait_rx_q(pf, pf_q, true);
4608                 if (ret) {
4609                         dev_info(&pf->pdev->dev,
4610                                  "VSI seid %d Rx ring %d enable timeout\n",
4611                                  vsi->seid, pf_q);
4612                         break;
4613                 }
4614         }
4615
4616         return ret;
4617 }
4618
4619 /**
4620  * i40e_vsi_start_rings - Start a VSI's rings
4621  * @vsi: the VSI being configured
4622  **/
4623 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4624 {
4625         int ret = 0;
4626
4627         /* do rx first for enable and last for disable */
4628         ret = i40e_vsi_enable_rx(vsi);
4629         if (ret)
4630                 return ret;
4631         ret = i40e_vsi_enable_tx(vsi);
4632
4633         return ret;
4634 }
4635
4636 #define I40E_DISABLE_TX_GAP_MSEC        50
4637
4638 /**
4639  * i40e_vsi_stop_rings - Stop a VSI's rings
4640  * @vsi: the VSI being configured
4641  **/
4642 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4643 {
4644         struct i40e_pf *pf = vsi->back;
4645         int pf_q, err, q_end;
4646
4647         /* When port TX is suspended, don't wait */
4648         if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4649                 return i40e_vsi_stop_rings_no_wait(vsi);
4650
4651         q_end = vsi->base_queue + vsi->num_queue_pairs;
4652         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4653                 i40e_pre_tx_queue_cfg(&pf->hw, (u32)pf_q, false);
4654
4655         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++) {
4656                 err = i40e_control_wait_rx_q(pf, pf_q, false);
4657                 if (err)
4658                         dev_info(&pf->pdev->dev,
4659                                  "VSI seid %d Rx ring %d disable timeout\n",
4660                                  vsi->seid, pf_q);
4661         }
4662
4663         msleep(I40E_DISABLE_TX_GAP_MSEC);
4664         pf_q = vsi->base_queue;
4665         for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4666                 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4667
4668         i40e_vsi_wait_queues_disabled(vsi);
4669 }
4670
4671 /**
4672  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4673  * @vsi: the VSI being shutdown
4674  *
4675  * This function stops all the rings for a VSI but does not delay to verify
4676  * that rings have been disabled. It is expected that the caller is shutting
4677  * down multiple VSIs at once and will delay together for all the VSIs after
4678  * initiating the shutdown. This is particularly useful for shutting down lots
4679  * of VFs together. Otherwise, a large delay can be incurred while configuring
4680  * each VSI in serial.
4681  **/
4682 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4683 {
4684         struct i40e_pf *pf = vsi->back;
4685         int i, pf_q;
4686
4687         pf_q = vsi->base_queue;
4688         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4689                 i40e_control_tx_q(pf, pf_q, false);
4690                 i40e_control_rx_q(pf, pf_q, false);
4691         }
4692 }
4693
4694 /**
4695  * i40e_vsi_free_irq - Free the irq association with the OS
4696  * @vsi: the VSI being configured
4697  **/
4698 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4699 {
4700         struct i40e_pf *pf = vsi->back;
4701         struct i40e_hw *hw = &pf->hw;
4702         int base = vsi->base_vector;
4703         u32 val, qp;
4704         int i;
4705
4706         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4707                 if (!vsi->q_vectors)
4708                         return;
4709
4710                 if (!vsi->irqs_ready)
4711                         return;
4712
4713                 vsi->irqs_ready = false;
4714                 for (i = 0; i < vsi->num_q_vectors; i++) {
4715                         int irq_num;
4716                         u16 vector;
4717
4718                         vector = i + base;
4719                         irq_num = pf->msix_entries[vector].vector;
4720
4721                         /* free only the irqs that were actually requested */
4722                         if (!vsi->q_vectors[i] ||
4723                             !vsi->q_vectors[i]->num_ringpairs)
4724                                 continue;
4725
4726                         /* clear the affinity notifier in the IRQ descriptor */
4727                         irq_set_affinity_notifier(irq_num, NULL);
4728                         /* remove our suggested affinity mask for this IRQ */
4729                         irq_set_affinity_hint(irq_num, NULL);
4730                         synchronize_irq(irq_num);
4731                         free_irq(irq_num, vsi->q_vectors[i]);
4732
4733                         /* Tear down the interrupt queue link list
4734                          *
4735                          * We know that they come in pairs and always
4736                          * the Rx first, then the Tx.  To clear the
4737                          * link list, stick the EOL value into the
4738                          * next_q field of the registers.
4739                          */
4740                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4741                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4742                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4743                         val |= I40E_QUEUE_END_OF_LIST
4744                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4745                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4746
4747                         while (qp != I40E_QUEUE_END_OF_LIST) {
4748                                 u32 next;
4749
4750                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4751
4752                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4753                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4754                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4755                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4756
4757                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4758                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4759
4760                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4761
4762                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4763
4764                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4765                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4766
4767                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4768                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4769                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4770                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4771
4772                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4773                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4774
4775                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4776                                 qp = next;
4777                         }
4778                 }
4779         } else {
4780                 free_irq(pf->pdev->irq, pf);
4781
4782                 val = rd32(hw, I40E_PFINT_LNKLST0);
4783                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4784                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4785                 val |= I40E_QUEUE_END_OF_LIST
4786                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4787                 wr32(hw, I40E_PFINT_LNKLST0, val);
4788
4789                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4790                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4791                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4792                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4793                          I40E_QINT_RQCTL_INTEVENT_MASK);
4794
4795                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4796                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4797
4798                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4799
4800                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4801
4802                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4803                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4804                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4805                          I40E_QINT_TQCTL_INTEVENT_MASK);
4806
4807                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4808                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4809
4810                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4811         }
4812 }
4813
4814 /**
4815  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4816  * @vsi: the VSI being configured
4817  * @v_idx: Index of vector to be freed
4818  *
4819  * This function frees the memory allocated to the q_vector.  In addition if
4820  * NAPI is enabled it will delete any references to the NAPI struct prior
4821  * to freeing the q_vector.
4822  **/
4823 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4824 {
4825         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4826         struct i40e_ring *ring;
4827
4828         if (!q_vector)
4829                 return;
4830
4831         /* disassociate q_vector from rings */
4832         i40e_for_each_ring(ring, q_vector->tx)
4833                 ring->q_vector = NULL;
4834
4835         i40e_for_each_ring(ring, q_vector->rx)
4836                 ring->q_vector = NULL;
4837
4838         /* only VSI w/ an associated netdev is set up w/ NAPI */
4839         if (vsi->netdev)
4840                 netif_napi_del(&q_vector->napi);
4841
4842         vsi->q_vectors[v_idx] = NULL;
4843
4844         kfree_rcu(q_vector, rcu);
4845 }
4846
4847 /**
4848  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4849  * @vsi: the VSI being un-configured
4850  *
4851  * This frees the memory allocated to the q_vectors and
4852  * deletes references to the NAPI struct.
4853  **/
4854 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4855 {
4856         int v_idx;
4857
4858         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4859                 i40e_free_q_vector(vsi, v_idx);
4860 }
4861
4862 /**
4863  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4864  * @pf: board private structure
4865  **/
4866 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4867 {
4868         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4869         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4870                 pci_disable_msix(pf->pdev);
4871                 kfree(pf->msix_entries);
4872                 pf->msix_entries = NULL;
4873                 kfree(pf->irq_pile);
4874                 pf->irq_pile = NULL;
4875         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4876                 pci_disable_msi(pf->pdev);
4877         }
4878         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4879 }
4880
4881 /**
4882  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4883  * @pf: board private structure
4884  *
4885  * We go through and clear interrupt specific resources and reset the structure
4886  * to pre-load conditions
4887  **/
4888 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4889 {
4890         int i;
4891
4892         if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
4893                 i40e_free_misc_vector(pf);
4894
4895         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4896                       I40E_IWARP_IRQ_PILE_ID);
4897
4898         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4899         for (i = 0; i < pf->num_alloc_vsi; i++)
4900                 if (pf->vsi[i])
4901                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4902         i40e_reset_interrupt_capability(pf);
4903 }
4904
4905 /**
4906  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4907  * @vsi: the VSI being configured
4908  **/
4909 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4910 {
4911         int q_idx;
4912
4913         if (!vsi->netdev)
4914                 return;
4915
4916         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4917                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4918
4919                 if (q_vector->rx.ring || q_vector->tx.ring)
4920                         napi_enable(&q_vector->napi);
4921         }
4922 }
4923
4924 /**
4925  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4926  * @vsi: the VSI being configured
4927  **/
4928 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4929 {
4930         int q_idx;
4931
4932         if (!vsi->netdev)
4933                 return;
4934
4935         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4936                 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4937
4938                 if (q_vector->rx.ring || q_vector->tx.ring)
4939                         napi_disable(&q_vector->napi);
4940         }
4941 }
4942
4943 /**
4944  * i40e_vsi_close - Shut down a VSI
4945  * @vsi: the vsi to be quelled
4946  **/
4947 static void i40e_vsi_close(struct i40e_vsi *vsi)
4948 {
4949         struct i40e_pf *pf = vsi->back;
4950         if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4951                 i40e_down(vsi);
4952         i40e_vsi_free_irq(vsi);
4953         i40e_vsi_free_tx_resources(vsi);
4954         i40e_vsi_free_rx_resources(vsi);
4955         vsi->current_netdev_flags = 0;
4956         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
4957         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4958                 set_bit(__I40E_CLIENT_RESET, pf->state);
4959 }
4960
4961 /**
4962  * i40e_quiesce_vsi - Pause a given VSI
4963  * @vsi: the VSI being paused
4964  **/
4965 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4966 {
4967         if (test_bit(__I40E_VSI_DOWN, vsi->state))
4968                 return;
4969
4970         set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4971         if (vsi->netdev && netif_running(vsi->netdev))
4972                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4973         else
4974                 i40e_vsi_close(vsi);
4975 }
4976
4977 /**
4978  * i40e_unquiesce_vsi - Resume a given VSI
4979  * @vsi: the VSI being resumed
4980  **/
4981 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4982 {
4983         if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4984                 return;
4985
4986         if (vsi->netdev && netif_running(vsi->netdev))
4987                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4988         else
4989                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4990 }
4991
4992 /**
4993  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4994  * @pf: the PF
4995  **/
4996 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4997 {
4998         int v;
4999
5000         for (v = 0; v < pf->num_alloc_vsi; v++) {
5001                 if (pf->vsi[v])
5002                         i40e_quiesce_vsi(pf->vsi[v]);
5003         }
5004 }
5005
5006 /**
5007  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5008  * @pf: the PF
5009  **/
5010 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5011 {
5012         int v;
5013
5014         for (v = 0; v < pf->num_alloc_vsi; v++) {
5015                 if (pf->vsi[v])
5016                         i40e_unquiesce_vsi(pf->vsi[v]);
5017         }
5018 }
5019
5020 /**
5021  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5022  * @vsi: the VSI being configured
5023  *
5024  * Wait until all queues on a given VSI have been disabled.
5025  **/
5026 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5027 {
5028         struct i40e_pf *pf = vsi->back;
5029         int i, pf_q, ret;
5030
5031         pf_q = vsi->base_queue;
5032         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5033                 /* Check and wait for the Tx queue */
5034                 ret = i40e_pf_txq_wait(pf, pf_q, false);
5035                 if (ret) {
5036                         dev_info(&pf->pdev->dev,
5037                                  "VSI seid %d Tx ring %d disable timeout\n",
5038                                  vsi->seid, pf_q);
5039                         return ret;
5040                 }
5041
5042                 if (!i40e_enabled_xdp_vsi(vsi))
5043                         goto wait_rx;
5044
5045                 /* Check and wait for the XDP Tx queue */
5046                 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5047                                        false);
5048                 if (ret) {
5049                         dev_info(&pf->pdev->dev,
5050                                  "VSI seid %d XDP Tx ring %d disable timeout\n",
5051                                  vsi->seid, pf_q);
5052                         return ret;
5053                 }
5054 wait_rx:
5055                 /* Check and wait for the Rx queue */
5056                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
5057                 if (ret) {
5058                         dev_info(&pf->pdev->dev,
5059                                  "VSI seid %d Rx ring %d disable timeout\n",
5060                                  vsi->seid, pf_q);
5061                         return ret;
5062                 }
5063         }
5064
5065         return 0;
5066 }
5067
5068 #ifdef CONFIG_I40E_DCB
5069 /**
5070  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5071  * @pf: the PF
5072  *
5073  * This function waits for the queues to be in disabled state for all the
5074  * VSIs that are managed by this PF.
5075  **/
5076 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5077 {
5078         int v, ret = 0;
5079
5080         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
5081                 if (pf->vsi[v]) {
5082                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
5083                         if (ret)
5084                                 break;
5085                 }
5086         }
5087
5088         return ret;
5089 }
5090
5091 #endif
5092
5093 /**
5094  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5095  * @pf: pointer to PF
5096  *
5097  * Get TC map for ISCSI PF type that will include iSCSI TC
5098  * and LAN TC.
5099  **/
5100 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5101 {
5102         struct i40e_dcb_app_priority_table app;
5103         struct i40e_hw *hw = &pf->hw;
5104         u8 enabled_tc = 1; /* TC0 is always enabled */
5105         u8 tc, i;
5106         /* Get the iSCSI APP TLV */
5107         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5108
5109         for (i = 0; i < dcbcfg->numapps; i++) {
5110                 app = dcbcfg->app[i];
5111                 if (app.selector == I40E_APP_SEL_TCPIP &&
5112                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
5113                         tc = dcbcfg->etscfg.prioritytable[app.priority];
5114                         enabled_tc |= BIT(tc);
5115                         break;
5116                 }
5117         }
5118
5119         return enabled_tc;
5120 }
5121
5122 /**
5123  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5124  * @dcbcfg: the corresponding DCBx configuration structure
5125  *
5126  * Return the number of TCs from given DCBx configuration
5127  **/
5128 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5129 {
5130         int i, tc_unused = 0;
5131         u8 num_tc = 0;
5132         u8 ret = 0;
5133
5134         /* Scan the ETS Config Priority Table to find
5135          * traffic class enabled for a given priority
5136          * and create a bitmask of enabled TCs
5137          */
5138         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5139                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5140
5141         /* Now scan the bitmask to check for
5142          * contiguous TCs starting with TC0
5143          */
5144         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5145                 if (num_tc & BIT(i)) {
5146                         if (!tc_unused) {
5147                                 ret++;
5148                         } else {
5149                                 pr_err("Non-contiguous TC - Disabling DCB\n");
5150                                 return 1;
5151                         }
5152                 } else {
5153                         tc_unused = 1;
5154                 }
5155         }
5156
5157         /* There is always at least TC0 */
5158         if (!ret)
5159                 ret = 1;
5160
5161         return ret;
5162 }
5163
5164 /**
5165  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5166  * @dcbcfg: the corresponding DCBx configuration structure
5167  *
5168  * Query the current DCB configuration and return the number of
5169  * traffic classes enabled from the given DCBX config
5170  **/
5171 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5172 {
5173         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5174         u8 enabled_tc = 1;
5175         u8 i;
5176
5177         for (i = 0; i < num_tc; i++)
5178                 enabled_tc |= BIT(i);
5179
5180         return enabled_tc;
5181 }
5182
5183 /**
5184  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5185  * @pf: PF being queried
5186  *
5187  * Query the current MQPRIO configuration and return the number of
5188  * traffic classes enabled.
5189  **/
5190 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5191 {
5192         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5193         u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5194         u8 enabled_tc = 1, i;
5195
5196         for (i = 1; i < num_tc; i++)
5197                 enabled_tc |= BIT(i);
5198         return enabled_tc;
5199 }
5200
5201 /**
5202  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5203  * @pf: PF being queried
5204  *
5205  * Return number of traffic classes enabled for the given PF
5206  **/
5207 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5208 {
5209         struct i40e_hw *hw = &pf->hw;
5210         u8 i, enabled_tc = 1;
5211         u8 num_tc = 0;
5212         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5213
5214         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5215                 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5216
5217         /* If neither MQPRIO nor DCB is enabled, then always use single TC */
5218         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5219                 return 1;
5220
5221         /* SFP mode will be enabled for all TCs on port */
5222         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5223                 return i40e_dcb_get_num_tc(dcbcfg);
5224
5225         /* MFP mode return count of enabled TCs for this PF */
5226         if (pf->hw.func_caps.iscsi)
5227                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
5228         else
5229                 return 1; /* Only TC0 */
5230
5231         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5232                 if (enabled_tc & BIT(i))
5233                         num_tc++;
5234         }
5235         return num_tc;
5236 }
5237
5238 /**
5239  * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5240  * @pf: PF being queried
5241  *
5242  * Return a bitmap for enabled traffic classes for this PF.
5243  **/
5244 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5245 {
5246         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5247                 return i40e_mqprio_get_enabled_tc(pf);
5248
5249         /* If neither MQPRIO nor DCB is enabled for this PF then just return
5250          * default TC
5251          */
5252         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5253                 return I40E_DEFAULT_TRAFFIC_CLASS;
5254
5255         /* SFP mode we want PF to be enabled for all TCs */
5256         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5257                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5258
5259         /* MFP enabled and iSCSI PF type */
5260         if (pf->hw.func_caps.iscsi)
5261                 return i40e_get_iscsi_tc_map(pf);
5262         else
5263                 return I40E_DEFAULT_TRAFFIC_CLASS;
5264 }
5265
5266 /**
5267  * i40e_vsi_get_bw_info - Query VSI BW Information
5268  * @vsi: the VSI being queried
5269  *
5270  * Returns 0 on success, negative value on failure
5271  **/
5272 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5273 {
5274         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5275         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5276         struct i40e_pf *pf = vsi->back;
5277         struct i40e_hw *hw = &pf->hw;
5278         i40e_status ret;
5279         u32 tc_bw_max;
5280         int i;
5281
5282         /* Get the VSI level BW configuration */
5283         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5284         if (ret) {
5285                 dev_info(&pf->pdev->dev,
5286                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
5287                          i40e_stat_str(&pf->hw, ret),
5288                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5289                 return -EINVAL;
5290         }
5291
5292         /* Get the VSI level BW configuration per TC */
5293         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5294                                                NULL);
5295         if (ret) {
5296                 dev_info(&pf->pdev->dev,
5297                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
5298                          i40e_stat_str(&pf->hw, ret),
5299                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5300                 return -EINVAL;
5301         }
5302
5303         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5304                 dev_info(&pf->pdev->dev,
5305                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5306                          bw_config.tc_valid_bits,
5307                          bw_ets_config.tc_valid_bits);
5308                 /* Still continuing */
5309         }
5310
5311         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5312         vsi->bw_max_quanta = bw_config.max_bw;
5313         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5314                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5315         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5316                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5317                 vsi->bw_ets_limit_credits[i] =
5318                                         le16_to_cpu(bw_ets_config.credits[i]);
5319                 /* 3 bits out of 4 for each TC */
5320                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5321         }
5322
5323         return 0;
5324 }
5325
5326 /**
5327  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5328  * @vsi: the VSI being configured
5329  * @enabled_tc: TC bitmap
5330  * @bw_share: BW shared credits per TC
5331  *
5332  * Returns 0 on success, negative value on failure
5333  **/
5334 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5335                                        u8 *bw_share)
5336 {
5337         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5338         struct i40e_pf *pf = vsi->back;
5339         i40e_status ret;
5340         int i;
5341
5342         /* There is no need to reset BW when mqprio mode is on.  */
5343         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5344                 return 0;
5345         if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5346                 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5347                 if (ret)
5348                         dev_info(&pf->pdev->dev,
5349                                  "Failed to reset tx rate for vsi->seid %u\n",
5350                                  vsi->seid);
5351                 return ret;
5352         }
5353         memset(&bw_data, 0, sizeof(bw_data));
5354         bw_data.tc_valid_bits = enabled_tc;
5355         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5356                 bw_data.tc_bw_credits[i] = bw_share[i];
5357
5358         ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5359         if (ret) {
5360                 dev_info(&pf->pdev->dev,
5361                          "AQ command Config VSI BW allocation per TC failed = %d\n",
5362                          pf->hw.aq.asq_last_status);
5363                 return -EINVAL;
5364         }
5365
5366         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5367                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5368
5369         return 0;
5370 }
5371
5372 /**
5373  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5374  * @vsi: the VSI being configured
5375  * @enabled_tc: TC map to be enabled
5376  *
5377  **/
5378 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5379 {
5380         struct net_device *netdev = vsi->netdev;
5381         struct i40e_pf *pf = vsi->back;
5382         struct i40e_hw *hw = &pf->hw;
5383         u8 netdev_tc = 0;
5384         int i;
5385         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5386
5387         if (!netdev)
5388                 return;
5389
5390         if (!enabled_tc) {
5391                 netdev_reset_tc(netdev);
5392                 return;
5393         }
5394
5395         /* Set up actual enabled TCs on the VSI */
5396         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5397                 return;
5398
5399         /* set per TC queues for the VSI */
5400         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5401                 /* Only set TC queues for enabled tcs
5402                  *
5403                  * e.g. For a VSI that has TC0 and TC3 enabled the
5404                  * enabled_tc bitmap would be 0x00001001; the driver
5405                  * will set the numtc for netdev as 2 that will be
5406                  * referenced by the netdev layer as TC 0 and 1.
5407                  */
5408                 if (vsi->tc_config.enabled_tc & BIT(i))
5409                         netdev_set_tc_queue(netdev,
5410                                         vsi->tc_config.tc_info[i].netdev_tc,
5411                                         vsi->tc_config.tc_info[i].qcount,
5412                                         vsi->tc_config.tc_info[i].qoffset);
5413         }
5414
5415         if (pf->flags & I40E_FLAG_TC_MQPRIO)
5416                 return;
5417
5418         /* Assign UP2TC map for the VSI */
5419         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5420                 /* Get the actual TC# for the UP */
5421                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5422                 /* Get the mapped netdev TC# for the UP */
5423                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5424                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5425         }
5426 }
5427
5428 /**
5429  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5430  * @vsi: the VSI being configured
5431  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5432  **/
5433 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5434                                       struct i40e_vsi_context *ctxt)
5435 {
5436         /* copy just the sections touched not the entire info
5437          * since not all sections are valid as returned by
5438          * update vsi params
5439          */
5440         vsi->info.mapping_flags = ctxt->info.mapping_flags;
5441         memcpy(&vsi->info.queue_mapping,
5442                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5443         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5444                sizeof(vsi->info.tc_mapping));
5445 }
5446
5447 /**
5448  * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5449  * @vsi: the VSI being reconfigured
5450  * @vsi_offset: offset from main VF VSI
5451  */
5452 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5453 {
5454         struct i40e_vsi_context ctxt = {};
5455         struct i40e_pf *pf;
5456         struct i40e_hw *hw;
5457         int ret;
5458
5459         if (!vsi)
5460                 return I40E_ERR_PARAM;
5461         pf = vsi->back;
5462         hw = &pf->hw;
5463
5464         ctxt.seid = vsi->seid;
5465         ctxt.pf_num = hw->pf_id;
5466         ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5467         ctxt.uplink_seid = vsi->uplink_seid;
5468         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5469         ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5470         ctxt.info = vsi->info;
5471
5472         i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
5473                                  false);
5474         if (vsi->reconfig_rss) {
5475                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
5476                                       vsi->num_queue_pairs);
5477                 ret = i40e_vsi_config_rss(vsi);
5478                 if (ret) {
5479                         dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5480                         return ret;
5481                 }
5482                 vsi->reconfig_rss = false;
5483         }
5484
5485         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5486         if (ret) {
5487                 dev_info(&pf->pdev->dev, "Update vsi config failed, err %s aq_err %s\n",
5488                          i40e_stat_str(hw, ret),
5489                          i40e_aq_str(hw, hw->aq.asq_last_status));
5490                 return ret;
5491         }
5492         /* update the local VSI info with updated queue map */
5493         i40e_vsi_update_queue_map(vsi, &ctxt);
5494         vsi->info.valid_sections = 0;
5495
5496         return ret;
5497 }
5498
5499 /**
5500  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5501  * @vsi: VSI to be configured
5502  * @enabled_tc: TC bitmap
5503  *
5504  * This configures a particular VSI for TCs that are mapped to the
5505  * given TC bitmap. It uses default bandwidth share for TCs across
5506  * VSIs to configure TC for a particular VSI.
5507  *
5508  * NOTE:
5509  * It is expected that the VSI queues have been quisced before calling
5510  * this function.
5511  **/
5512 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5513 {
5514         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5515         struct i40e_pf *pf = vsi->back;
5516         struct i40e_hw *hw = &pf->hw;
5517         struct i40e_vsi_context ctxt;
5518         int ret = 0;
5519         int i;
5520
5521         /* Check if enabled_tc is same as existing or new TCs */
5522         if (vsi->tc_config.enabled_tc == enabled_tc &&
5523             vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5524                 return ret;
5525
5526         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5527         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5528                 if (enabled_tc & BIT(i))
5529                         bw_share[i] = 1;
5530         }
5531
5532         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5533         if (ret) {
5534                 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5535
5536                 dev_info(&pf->pdev->dev,
5537                          "Failed configuring TC map %d for VSI %d\n",
5538                          enabled_tc, vsi->seid);
5539                 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5540                                                   &bw_config, NULL);
5541                 if (ret) {
5542                         dev_info(&pf->pdev->dev,
5543                                  "Failed querying vsi bw info, err %s aq_err %s\n",
5544                                  i40e_stat_str(hw, ret),
5545                                  i40e_aq_str(hw, hw->aq.asq_last_status));
5546                         goto out;
5547                 }
5548                 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5549                         u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5550
5551                         if (!valid_tc)
5552                                 valid_tc = bw_config.tc_valid_bits;
5553                         /* Always enable TC0, no matter what */
5554                         valid_tc |= 1;
5555                         dev_info(&pf->pdev->dev,
5556                                  "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5557                                  enabled_tc, bw_config.tc_valid_bits, valid_tc);
5558                         enabled_tc = valid_tc;
5559                 }
5560
5561                 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5562                 if (ret) {
5563                         dev_err(&pf->pdev->dev,
5564                                 "Unable to  configure TC map %d for VSI %d\n",
5565                                 enabled_tc, vsi->seid);
5566                         goto out;
5567                 }
5568         }
5569
5570         /* Update Queue Pairs Mapping for currently enabled UPs */
5571         ctxt.seid = vsi->seid;
5572         ctxt.pf_num = vsi->back->hw.pf_id;
5573         ctxt.vf_num = 0;
5574         ctxt.uplink_seid = vsi->uplink_seid;
5575         ctxt.info = vsi->info;
5576         if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) {
5577                 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5578                 if (ret)
5579                         goto out;
5580         } else {
5581                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5582         }
5583
5584         /* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5585          * queues changed.
5586          */
5587         if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5588                 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5589                                       vsi->num_queue_pairs);
5590                 ret = i40e_vsi_config_rss(vsi);
5591                 if (ret) {
5592                         dev_info(&vsi->back->pdev->dev,
5593                                  "Failed to reconfig rss for num_queues\n");
5594                         return ret;
5595                 }
5596                 vsi->reconfig_rss = false;
5597         }
5598         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5599                 ctxt.info.valid_sections |=
5600                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5601                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5602         }
5603
5604         /* Update the VSI after updating the VSI queue-mapping
5605          * information
5606          */
5607         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5608         if (ret) {
5609                 dev_info(&pf->pdev->dev,
5610                          "Update vsi tc config failed, err %s aq_err %s\n",
5611                          i40e_stat_str(hw, ret),
5612                          i40e_aq_str(hw, hw->aq.asq_last_status));
5613                 goto out;
5614         }
5615         /* update the local VSI info with updated queue map */
5616         i40e_vsi_update_queue_map(vsi, &ctxt);
5617         vsi->info.valid_sections = 0;
5618
5619         /* Update current VSI BW information */
5620         ret = i40e_vsi_get_bw_info(vsi);
5621         if (ret) {
5622                 dev_info(&pf->pdev->dev,
5623                          "Failed updating vsi bw info, err %s aq_err %s\n",
5624                          i40e_stat_str(hw, ret),
5625                          i40e_aq_str(hw, hw->aq.asq_last_status));
5626                 goto out;
5627         }
5628
5629         /* Update the netdev TC setup */
5630         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5631 out:
5632         return ret;
5633 }
5634
5635 /**
5636  * i40e_get_link_speed - Returns link speed for the interface
5637  * @vsi: VSI to be configured
5638  *
5639  **/
5640 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5641 {
5642         struct i40e_pf *pf = vsi->back;
5643
5644         switch (pf->hw.phy.link_info.link_speed) {
5645         case I40E_LINK_SPEED_40GB:
5646                 return 40000;
5647         case I40E_LINK_SPEED_25GB:
5648                 return 25000;
5649         case I40E_LINK_SPEED_20GB:
5650                 return 20000;
5651         case I40E_LINK_SPEED_10GB:
5652                 return 10000;
5653         case I40E_LINK_SPEED_1GB:
5654                 return 1000;
5655         default:
5656                 return -EINVAL;
5657         }
5658 }
5659
5660 /**
5661  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5662  * @vsi: VSI to be configured
5663  * @seid: seid of the channel/VSI
5664  * @max_tx_rate: max TX rate to be configured as BW limit
5665  *
5666  * Helper function to set BW limit for a given VSI
5667  **/
5668 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5669 {
5670         struct i40e_pf *pf = vsi->back;
5671         u64 credits = 0;
5672         int speed = 0;
5673         int ret = 0;
5674
5675         speed = i40e_get_link_speed(vsi);
5676         if (max_tx_rate > speed) {
5677                 dev_err(&pf->pdev->dev,
5678                         "Invalid max tx rate %llu specified for VSI seid %d.",
5679                         max_tx_rate, seid);
5680                 return -EINVAL;
5681         }
5682         if (max_tx_rate && max_tx_rate < 50) {
5683                 dev_warn(&pf->pdev->dev,
5684                          "Setting max tx rate to minimum usable value of 50Mbps.\n");
5685                 max_tx_rate = 50;
5686         }
5687
5688         /* Tx rate credits are in values of 50Mbps, 0 is disabled */
5689         credits = max_tx_rate;
5690         do_div(credits, I40E_BW_CREDIT_DIVISOR);
5691         ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5692                                           I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5693         if (ret)
5694                 dev_err(&pf->pdev->dev,
5695                         "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n",
5696                         max_tx_rate, seid, i40e_stat_str(&pf->hw, ret),
5697                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5698         return ret;
5699 }
5700
5701 /**
5702  * i40e_remove_queue_channels - Remove queue channels for the TCs
5703  * @vsi: VSI to be configured
5704  *
5705  * Remove queue channels for the TCs
5706  **/
5707 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5708 {
5709         enum i40e_admin_queue_err last_aq_status;
5710         struct i40e_cloud_filter *cfilter;
5711         struct i40e_channel *ch, *ch_tmp;
5712         struct i40e_pf *pf = vsi->back;
5713         struct hlist_node *node;
5714         int ret, i;
5715
5716         /* Reset rss size that was stored when reconfiguring rss for
5717          * channel VSIs with non-power-of-2 queue count.
5718          */
5719         vsi->current_rss_size = 0;
5720
5721         /* perform cleanup for channels if they exist */
5722         if (list_empty(&vsi->ch_list))
5723                 return;
5724
5725         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5726                 struct i40e_vsi *p_vsi;
5727
5728                 list_del(&ch->list);
5729                 p_vsi = ch->parent_vsi;
5730                 if (!p_vsi || !ch->initialized) {
5731                         kfree(ch);
5732                         continue;
5733                 }
5734                 /* Reset queue contexts */
5735                 for (i = 0; i < ch->num_queue_pairs; i++) {
5736                         struct i40e_ring *tx_ring, *rx_ring;
5737                         u16 pf_q;
5738
5739                         pf_q = ch->base_queue + i;
5740                         tx_ring = vsi->tx_rings[pf_q];
5741                         tx_ring->ch = NULL;
5742
5743                         rx_ring = vsi->rx_rings[pf_q];
5744                         rx_ring->ch = NULL;
5745                 }
5746
5747                 /* Reset BW configured for this VSI via mqprio */
5748                 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
5749                 if (ret)
5750                         dev_info(&vsi->back->pdev->dev,
5751                                  "Failed to reset tx rate for ch->seid %u\n",
5752                                  ch->seid);
5753
5754                 /* delete cloud filters associated with this channel */
5755                 hlist_for_each_entry_safe(cfilter, node,
5756                                           &pf->cloud_filter_list, cloud_node) {
5757                         if (cfilter->seid != ch->seid)
5758                                 continue;
5759
5760                         hash_del(&cfilter->cloud_node);
5761                         if (cfilter->dst_port)
5762                                 ret = i40e_add_del_cloud_filter_big_buf(vsi,
5763                                                                         cfilter,
5764                                                                         false);
5765                         else
5766                                 ret = i40e_add_del_cloud_filter(vsi, cfilter,
5767                                                                 false);
5768                         last_aq_status = pf->hw.aq.asq_last_status;
5769                         if (ret)
5770                                 dev_info(&pf->pdev->dev,
5771                                          "Failed to delete cloud filter, err %s aq_err %s\n",
5772                                          i40e_stat_str(&pf->hw, ret),
5773                                          i40e_aq_str(&pf->hw, last_aq_status));
5774                         kfree(cfilter);
5775                 }
5776
5777                 /* delete VSI from FW */
5778                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
5779                                              NULL);
5780                 if (ret)
5781                         dev_err(&vsi->back->pdev->dev,
5782                                 "unable to remove channel (%d) for parent VSI(%d)\n",
5783                                 ch->seid, p_vsi->seid);
5784                 kfree(ch);
5785         }
5786         INIT_LIST_HEAD(&vsi->ch_list);
5787 }
5788
5789 /**
5790  * i40e_get_max_queues_for_channel
5791  * @vsi: ptr to VSI to which channels are associated with
5792  *
5793  * Helper function which returns max value among the queue counts set on the
5794  * channels/TCs created.
5795  **/
5796 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
5797 {
5798         struct i40e_channel *ch, *ch_tmp;
5799         int max = 0;
5800
5801         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5802                 if (!ch->initialized)
5803                         continue;
5804                 if (ch->num_queue_pairs > max)
5805                         max = ch->num_queue_pairs;
5806         }
5807
5808         return max;
5809 }
5810
5811 /**
5812  * i40e_validate_num_queues - validate num_queues w.r.t channel
5813  * @pf: ptr to PF device
5814  * @num_queues: number of queues
5815  * @vsi: the parent VSI
5816  * @reconfig_rss: indicates should the RSS be reconfigured or not
5817  *
5818  * This function validates number of queues in the context of new channel
5819  * which is being established and determines if RSS should be reconfigured
5820  * or not for parent VSI.
5821  **/
5822 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
5823                                     struct i40e_vsi *vsi, bool *reconfig_rss)
5824 {
5825         int max_ch_queues;
5826
5827         if (!reconfig_rss)
5828                 return -EINVAL;
5829
5830         *reconfig_rss = false;
5831         if (vsi->current_rss_size) {
5832                 if (num_queues > vsi->current_rss_size) {
5833                         dev_dbg(&pf->pdev->dev,
5834                                 "Error: num_queues (%d) > vsi's current_size(%d)\n",
5835                                 num_queues, vsi->current_rss_size);
5836                         return -EINVAL;
5837                 } else if ((num_queues < vsi->current_rss_size) &&
5838                            (!is_power_of_2(num_queues))) {
5839                         dev_dbg(&pf->pdev->dev,
5840                                 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
5841                                 num_queues, vsi->current_rss_size);
5842                         return -EINVAL;
5843                 }
5844         }
5845
5846         if (!is_power_of_2(num_queues)) {
5847                 /* Find the max num_queues configured for channel if channel
5848                  * exist.
5849                  * if channel exist, then enforce 'num_queues' to be more than
5850                  * max ever queues configured for channel.
5851                  */
5852                 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
5853                 if (num_queues < max_ch_queues) {
5854                         dev_dbg(&pf->pdev->dev,
5855                                 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
5856                                 num_queues, max_ch_queues);
5857                         return -EINVAL;
5858                 }
5859                 *reconfig_rss = true;
5860         }
5861
5862         return 0;
5863 }
5864
5865 /**
5866  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
5867  * @vsi: the VSI being setup
5868  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
5869  *
5870  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
5871  **/
5872 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
5873 {
5874         struct i40e_pf *pf = vsi->back;
5875         u8 seed[I40E_HKEY_ARRAY_SIZE];
5876         struct i40e_hw *hw = &pf->hw;
5877         int local_rss_size;
5878         u8 *lut;
5879         int ret;
5880
5881         if (!vsi->rss_size)
5882                 return -EINVAL;
5883
5884         if (rss_size > vsi->rss_size)
5885                 return -EINVAL;
5886
5887         local_rss_size = min_t(int, vsi->rss_size, rss_size);
5888         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
5889         if (!lut)
5890                 return -ENOMEM;
5891
5892         /* Ignoring user configured lut if there is one */
5893         i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
5894
5895         /* Use user configured hash key if there is one, otherwise
5896          * use default.
5897          */
5898         if (vsi->rss_hkey_user)
5899                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
5900         else
5901                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
5902
5903         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
5904         if (ret) {
5905                 dev_info(&pf->pdev->dev,
5906                          "Cannot set RSS lut, err %s aq_err %s\n",
5907                          i40e_stat_str(hw, ret),
5908                          i40e_aq_str(hw, hw->aq.asq_last_status));
5909                 kfree(lut);
5910                 return ret;
5911         }
5912         kfree(lut);
5913
5914         /* Do the update w.r.t. storing rss_size */
5915         if (!vsi->orig_rss_size)
5916                 vsi->orig_rss_size = vsi->rss_size;
5917         vsi->current_rss_size = local_rss_size;
5918
5919         return ret;
5920 }
5921
5922 /**
5923  * i40e_channel_setup_queue_map - Setup a channel queue map
5924  * @pf: ptr to PF device
5925  * @ctxt: VSI context structure
5926  * @ch: ptr to channel structure
5927  *
5928  * Setup queue map for a specific channel
5929  **/
5930 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
5931                                          struct i40e_vsi_context *ctxt,
5932                                          struct i40e_channel *ch)
5933 {
5934         u16 qcount, qmap, sections = 0;
5935         u8 offset = 0;
5936         int pow;
5937
5938         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
5939         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
5940
5941         qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
5942         ch->num_queue_pairs = qcount;
5943
5944         /* find the next higher power-of-2 of num queue pairs */
5945         pow = ilog2(qcount);
5946         if (!is_power_of_2(qcount))
5947                 pow++;
5948
5949         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5950                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
5951
5952         /* Setup queue TC[0].qmap for given VSI context */
5953         ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
5954
5955         ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
5956         ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
5957         ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
5958         ctxt->info.valid_sections |= cpu_to_le16(sections);
5959 }
5960
5961 /**
5962  * i40e_add_channel - add a channel by adding VSI
5963  * @pf: ptr to PF device
5964  * @uplink_seid: underlying HW switching element (VEB) ID
5965  * @ch: ptr to channel structure
5966  *
5967  * Add a channel (VSI) using add_vsi and queue_map
5968  **/
5969 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
5970                             struct i40e_channel *ch)
5971 {
5972         struct i40e_hw *hw = &pf->hw;
5973         struct i40e_vsi_context ctxt;
5974         u8 enabled_tc = 0x1; /* TC0 enabled */
5975         int ret;
5976
5977         if (ch->type != I40E_VSI_VMDQ2) {
5978                 dev_info(&pf->pdev->dev,
5979                          "add new vsi failed, ch->type %d\n", ch->type);
5980                 return -EINVAL;
5981         }
5982
5983         memset(&ctxt, 0, sizeof(ctxt));
5984         ctxt.pf_num = hw->pf_id;
5985         ctxt.vf_num = 0;
5986         ctxt.uplink_seid = uplink_seid;
5987         ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5988         if (ch->type == I40E_VSI_VMDQ2)
5989                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
5990
5991         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
5992                 ctxt.info.valid_sections |=
5993                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5994                 ctxt.info.switch_id =
5995                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
5996         }
5997
5998         /* Set queue map for a given VSI context */
5999         i40e_channel_setup_queue_map(pf, &ctxt, ch);
6000
6001         /* Now time to create VSI */
6002         ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6003         if (ret) {
6004                 dev_info(&pf->pdev->dev,
6005                          "add new vsi failed, err %s aq_err %s\n",
6006                          i40e_stat_str(&pf->hw, ret),
6007                          i40e_aq_str(&pf->hw,
6008                                      pf->hw.aq.asq_last_status));
6009                 return -ENOENT;
6010         }
6011
6012         /* Success, update channel, set enabled_tc only if the channel
6013          * is not a macvlan
6014          */
6015         ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6016         ch->seid = ctxt.seid;
6017         ch->vsi_number = ctxt.vsi_number;
6018         ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6019
6020         /* copy just the sections touched not the entire info
6021          * since not all sections are valid as returned by
6022          * update vsi params
6023          */
6024         ch->info.mapping_flags = ctxt.info.mapping_flags;
6025         memcpy(&ch->info.queue_mapping,
6026                &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6027         memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6028                sizeof(ctxt.info.tc_mapping));
6029
6030         return 0;
6031 }
6032
6033 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6034                                   u8 *bw_share)
6035 {
6036         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6037         i40e_status ret;
6038         int i;
6039
6040         memset(&bw_data, 0, sizeof(bw_data));
6041         bw_data.tc_valid_bits = ch->enabled_tc;
6042         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6043                 bw_data.tc_bw_credits[i] = bw_share[i];
6044
6045         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
6046                                        &bw_data, NULL);
6047         if (ret) {
6048                 dev_info(&vsi->back->pdev->dev,
6049                          "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6050                          vsi->back->hw.aq.asq_last_status, ch->seid);
6051                 return -EINVAL;
6052         }
6053
6054         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6055                 ch->info.qs_handle[i] = bw_data.qs_handles[i];
6056
6057         return 0;
6058 }
6059
6060 /**
6061  * i40e_channel_config_tx_ring - config TX ring associated with new channel
6062  * @pf: ptr to PF device
6063  * @vsi: the VSI being setup
6064  * @ch: ptr to channel structure
6065  *
6066  * Configure TX rings associated with channel (VSI) since queues are being
6067  * from parent VSI.
6068  **/
6069 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6070                                        struct i40e_vsi *vsi,
6071                                        struct i40e_channel *ch)
6072 {
6073         i40e_status ret;
6074         int i;
6075         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6076
6077         /* Enable ETS TCs with equal BW Share for now across all VSIs */
6078         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6079                 if (ch->enabled_tc & BIT(i))
6080                         bw_share[i] = 1;
6081         }
6082
6083         /* configure BW for new VSI */
6084         ret = i40e_channel_config_bw(vsi, ch, bw_share);
6085         if (ret) {
6086                 dev_info(&vsi->back->pdev->dev,
6087                          "Failed configuring TC map %d for channel (seid %u)\n",
6088                          ch->enabled_tc, ch->seid);
6089                 return ret;
6090         }
6091
6092         for (i = 0; i < ch->num_queue_pairs; i++) {
6093                 struct i40e_ring *tx_ring, *rx_ring;
6094                 u16 pf_q;
6095
6096                 pf_q = ch->base_queue + i;
6097
6098                 /* Get to TX ring ptr of main VSI, for re-setup TX queue
6099                  * context
6100                  */
6101                 tx_ring = vsi->tx_rings[pf_q];
6102                 tx_ring->ch = ch;
6103
6104                 /* Get the RX ring ptr */
6105                 rx_ring = vsi->rx_rings[pf_q];
6106                 rx_ring->ch = ch;
6107         }
6108
6109         return 0;
6110 }
6111
6112 /**
6113  * i40e_setup_hw_channel - setup new channel
6114  * @pf: ptr to PF device
6115  * @vsi: the VSI being setup
6116  * @ch: ptr to channel structure
6117  * @uplink_seid: underlying HW switching element (VEB) ID
6118  * @type: type of channel to be created (VMDq2/VF)
6119  *
6120  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6121  * and configures TX rings accordingly
6122  **/
6123 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6124                                         struct i40e_vsi *vsi,
6125                                         struct i40e_channel *ch,
6126                                         u16 uplink_seid, u8 type)
6127 {
6128         int ret;
6129
6130         ch->initialized = false;
6131         ch->base_queue = vsi->next_base_queue;
6132         ch->type = type;
6133
6134         /* Proceed with creation of channel (VMDq2) VSI */
6135         ret = i40e_add_channel(pf, uplink_seid, ch);
6136         if (ret) {
6137                 dev_info(&pf->pdev->dev,
6138                          "failed to add_channel using uplink_seid %u\n",
6139                          uplink_seid);
6140                 return ret;
6141         }
6142
6143         /* Mark the successful creation of channel */
6144         ch->initialized = true;
6145
6146         /* Reconfigure TX queues using QTX_CTL register */
6147         ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6148         if (ret) {
6149                 dev_info(&pf->pdev->dev,
6150                          "failed to configure TX rings for channel %u\n",
6151                          ch->seid);
6152                 return ret;
6153         }
6154
6155         /* update 'next_base_queue' */
6156         vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6157         dev_dbg(&pf->pdev->dev,
6158                 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6159                 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6160                 ch->num_queue_pairs,
6161                 vsi->next_base_queue);
6162         return ret;
6163 }
6164
6165 /**
6166  * i40e_setup_channel - setup new channel using uplink element
6167  * @pf: ptr to PF device
6168  * @vsi: pointer to the VSI to set up the channel within
6169  * @ch: ptr to channel structure
6170  *
6171  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6172  * and uplink switching element (uplink_seid)
6173  **/
6174 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6175                                struct i40e_channel *ch)
6176 {
6177         u8 vsi_type;
6178         u16 seid;
6179         int ret;
6180
6181         if (vsi->type == I40E_VSI_MAIN) {
6182                 vsi_type = I40E_VSI_VMDQ2;
6183         } else {
6184                 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6185                         vsi->type);
6186                 return false;
6187         }
6188
6189         /* underlying switching element */
6190         seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6191
6192         /* create channel (VSI), configure TX rings */
6193         ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6194         if (ret) {
6195                 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6196                 return false;
6197         }
6198
6199         return ch->initialized ? true : false;
6200 }
6201
6202 /**
6203  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6204  * @vsi: ptr to VSI which has PF backing
6205  *
6206  * Sets up switch mode correctly if it needs to be changed and perform
6207  * what are allowed modes.
6208  **/
6209 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6210 {
6211         u8 mode;
6212         struct i40e_pf *pf = vsi->back;
6213         struct i40e_hw *hw = &pf->hw;
6214         int ret;
6215
6216         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6217         if (ret)
6218                 return -EINVAL;
6219
6220         if (hw->dev_caps.switch_mode) {
6221                 /* if switch mode is set, support mode2 (non-tunneled for
6222                  * cloud filter) for now
6223                  */
6224                 u32 switch_mode = hw->dev_caps.switch_mode &
6225                                   I40E_SWITCH_MODE_MASK;
6226                 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6227                         if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6228                                 return 0;
6229                         dev_err(&pf->pdev->dev,
6230                                 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6231                                 hw->dev_caps.switch_mode);
6232                         return -EINVAL;
6233                 }
6234         }
6235
6236         /* Set Bit 7 to be valid */
6237         mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6238
6239         /* Set L4type for TCP support */
6240         mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6241
6242         /* Set cloud filter mode */
6243         mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6244
6245         /* Prep mode field for set_switch_config */
6246         ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6247                                         pf->last_sw_conf_valid_flags,
6248                                         mode, NULL);
6249         if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6250                 dev_err(&pf->pdev->dev,
6251                         "couldn't set switch config bits, err %s aq_err %s\n",
6252                         i40e_stat_str(hw, ret),
6253                         i40e_aq_str(hw,
6254                                     hw->aq.asq_last_status));
6255
6256         return ret;
6257 }
6258
6259 /**
6260  * i40e_create_queue_channel - function to create channel
6261  * @vsi: VSI to be configured
6262  * @ch: ptr to channel (it contains channel specific params)
6263  *
6264  * This function creates channel (VSI) using num_queues specified by user,
6265  * reconfigs RSS if needed.
6266  **/
6267 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6268                               struct i40e_channel *ch)
6269 {
6270         struct i40e_pf *pf = vsi->back;
6271         bool reconfig_rss;
6272         int err;
6273
6274         if (!ch)
6275                 return -EINVAL;
6276
6277         if (!ch->num_queue_pairs) {
6278                 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6279                         ch->num_queue_pairs);
6280                 return -EINVAL;
6281         }
6282
6283         /* validate user requested num_queues for channel */
6284         err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6285                                        &reconfig_rss);
6286         if (err) {
6287                 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6288                          ch->num_queue_pairs);
6289                 return -EINVAL;
6290         }
6291
6292         /* By default we are in VEPA mode, if this is the first VF/VMDq
6293          * VSI to be added switch to VEB mode.
6294          */
6295
6296         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6297                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6298
6299                 if (vsi->type == I40E_VSI_MAIN) {
6300                         if (pf->flags & I40E_FLAG_TC_MQPRIO)
6301                                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
6302                         else
6303                                 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6304                 }
6305                 /* now onwards for main VSI, number of queues will be value
6306                  * of TC0's queue count
6307                  */
6308         }
6309
6310         /* By this time, vsi->cnt_q_avail shall be set to non-zero and
6311          * it should be more than num_queues
6312          */
6313         if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6314                 dev_dbg(&pf->pdev->dev,
6315                         "Error: cnt_q_avail (%u) less than num_queues %d\n",
6316                         vsi->cnt_q_avail, ch->num_queue_pairs);
6317                 return -EINVAL;
6318         }
6319
6320         /* reconfig_rss only if vsi type is MAIN_VSI */
6321         if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6322                 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6323                 if (err) {
6324                         dev_info(&pf->pdev->dev,
6325                                  "Error: unable to reconfig rss for num_queues (%u)\n",
6326                                  ch->num_queue_pairs);
6327                         return -EINVAL;
6328                 }
6329         }
6330
6331         if (!i40e_setup_channel(pf, vsi, ch)) {
6332                 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6333                 return -EINVAL;
6334         }
6335
6336         dev_info(&pf->pdev->dev,
6337                  "Setup channel (id:%u) utilizing num_queues %d\n",
6338                  ch->seid, ch->num_queue_pairs);
6339
6340         /* configure VSI for BW limit */
6341         if (ch->max_tx_rate) {
6342                 u64 credits = ch->max_tx_rate;
6343
6344                 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6345                         return -EINVAL;
6346
6347                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6348                 dev_dbg(&pf->pdev->dev,
6349                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6350                         ch->max_tx_rate,
6351                         credits,
6352                         ch->seid);
6353         }
6354
6355         /* in case of VF, this will be main SRIOV VSI */
6356         ch->parent_vsi = vsi;
6357
6358         /* and update main_vsi's count for queue_available to use */
6359         vsi->cnt_q_avail -= ch->num_queue_pairs;
6360
6361         return 0;
6362 }
6363
6364 /**
6365  * i40e_configure_queue_channels - Add queue channel for the given TCs
6366  * @vsi: VSI to be configured
6367  *
6368  * Configures queue channel mapping to the given TCs
6369  **/
6370 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6371 {
6372         struct i40e_channel *ch;
6373         u64 max_rate = 0;
6374         int ret = 0, i;
6375
6376         /* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6377         vsi->tc_seid_map[0] = vsi->seid;
6378         for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6379                 if (vsi->tc_config.enabled_tc & BIT(i)) {
6380                         ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6381                         if (!ch) {
6382                                 ret = -ENOMEM;
6383                                 goto err_free;
6384                         }
6385
6386                         INIT_LIST_HEAD(&ch->list);
6387                         ch->num_queue_pairs =
6388                                 vsi->tc_config.tc_info[i].qcount;
6389                         ch->base_queue =
6390                                 vsi->tc_config.tc_info[i].qoffset;
6391
6392                         /* Bandwidth limit through tc interface is in bytes/s,
6393                          * change to Mbit/s
6394                          */
6395                         max_rate = vsi->mqprio_qopt.max_rate[i];
6396                         do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6397                         ch->max_tx_rate = max_rate;
6398
6399                         list_add_tail(&ch->list, &vsi->ch_list);
6400
6401                         ret = i40e_create_queue_channel(vsi, ch);
6402                         if (ret) {
6403                                 dev_err(&vsi->back->pdev->dev,
6404                                         "Failed creating queue channel with TC%d: queues %d\n",
6405                                         i, ch->num_queue_pairs);
6406                                 goto err_free;
6407                         }
6408                         vsi->tc_seid_map[i] = ch->seid;
6409                 }
6410         }
6411         return ret;
6412
6413 err_free:
6414         i40e_remove_queue_channels(vsi);
6415         return ret;
6416 }
6417
6418 /**
6419  * i40e_veb_config_tc - Configure TCs for given VEB
6420  * @veb: given VEB
6421  * @enabled_tc: TC bitmap
6422  *
6423  * Configures given TC bitmap for VEB (switching) element
6424  **/
6425 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6426 {
6427         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6428         struct i40e_pf *pf = veb->pf;
6429         int ret = 0;
6430         int i;
6431
6432         /* No TCs or already enabled TCs just return */
6433         if (!enabled_tc || veb->enabled_tc == enabled_tc)
6434                 return ret;
6435
6436         bw_data.tc_valid_bits = enabled_tc;
6437         /* bw_data.absolute_credits is not set (relative) */
6438
6439         /* Enable ETS TCs with equal BW Share for now */
6440         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6441                 if (enabled_tc & BIT(i))
6442                         bw_data.tc_bw_share_credits[i] = 1;
6443         }
6444
6445         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6446                                                    &bw_data, NULL);
6447         if (ret) {
6448                 dev_info(&pf->pdev->dev,
6449                          "VEB bw config failed, err %s aq_err %s\n",
6450                          i40e_stat_str(&pf->hw, ret),
6451                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6452                 goto out;
6453         }
6454
6455         /* Update the BW information */
6456         ret = i40e_veb_get_bw_info(veb);
6457         if (ret) {
6458                 dev_info(&pf->pdev->dev,
6459                          "Failed getting veb bw config, err %s aq_err %s\n",
6460                          i40e_stat_str(&pf->hw, ret),
6461                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6462         }
6463
6464 out:
6465         return ret;
6466 }
6467
6468 #ifdef CONFIG_I40E_DCB
6469 /**
6470  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6471  * @pf: PF struct
6472  *
6473  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6474  * the caller would've quiesce all the VSIs before calling
6475  * this function
6476  **/
6477 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6478 {
6479         u8 tc_map = 0;
6480         int ret;
6481         u8 v;
6482
6483         /* Enable the TCs available on PF to all VEBs */
6484         tc_map = i40e_pf_get_tc_map(pf);
6485         if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6486                 return;
6487
6488         for (v = 0; v < I40E_MAX_VEB; v++) {
6489                 if (!pf->veb[v])
6490                         continue;
6491                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6492                 if (ret) {
6493                         dev_info(&pf->pdev->dev,
6494                                  "Failed configuring TC for VEB seid=%d\n",
6495                                  pf->veb[v]->seid);
6496                         /* Will try to configure as many components */
6497                 }
6498         }
6499
6500         /* Update each VSI */
6501         for (v = 0; v < pf->num_alloc_vsi; v++) {
6502                 if (!pf->vsi[v])
6503                         continue;
6504
6505                 /* - Enable all TCs for the LAN VSI
6506                  * - For all others keep them at TC0 for now
6507                  */
6508                 if (v == pf->lan_vsi)
6509                         tc_map = i40e_pf_get_tc_map(pf);
6510                 else
6511                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6512
6513                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6514                 if (ret) {
6515                         dev_info(&pf->pdev->dev,
6516                                  "Failed configuring TC for VSI seid=%d\n",
6517                                  pf->vsi[v]->seid);
6518                         /* Will try to configure as many components */
6519                 } else {
6520                         /* Re-configure VSI vectors based on updated TC map */
6521                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6522                         if (pf->vsi[v]->netdev)
6523                                 i40e_dcbnl_set_all(pf->vsi[v]);
6524                 }
6525         }
6526 }
6527
6528 /**
6529  * i40e_resume_port_tx - Resume port Tx
6530  * @pf: PF struct
6531  *
6532  * Resume a port's Tx and issue a PF reset in case of failure to
6533  * resume.
6534  **/
6535 static int i40e_resume_port_tx(struct i40e_pf *pf)
6536 {
6537         struct i40e_hw *hw = &pf->hw;
6538         int ret;
6539
6540         ret = i40e_aq_resume_port_tx(hw, NULL);
6541         if (ret) {
6542                 dev_info(&pf->pdev->dev,
6543                          "Resume Port Tx failed, err %s aq_err %s\n",
6544                           i40e_stat_str(&pf->hw, ret),
6545                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6546                 /* Schedule PF reset to recover */
6547                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6548                 i40e_service_event_schedule(pf);
6549         }
6550
6551         return ret;
6552 }
6553
6554 /**
6555  * i40e_suspend_port_tx - Suspend port Tx
6556  * @pf: PF struct
6557  *
6558  * Suspend a port's Tx and issue a PF reset in case of failure.
6559  **/
6560 static int i40e_suspend_port_tx(struct i40e_pf *pf)
6561 {
6562         struct i40e_hw *hw = &pf->hw;
6563         int ret;
6564
6565         ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);
6566         if (ret) {
6567                 dev_info(&pf->pdev->dev,
6568                          "Suspend Port Tx failed, err %s aq_err %s\n",
6569                          i40e_stat_str(&pf->hw, ret),
6570                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6571                 /* Schedule PF reset to recover */
6572                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6573                 i40e_service_event_schedule(pf);
6574         }
6575
6576         return ret;
6577 }
6578
6579 /**
6580  * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6581  * @pf: PF being configured
6582  * @new_cfg: New DCBX configuration
6583  *
6584  * Program DCB settings into HW and reconfigure VEB/VSIs on
6585  * given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6586  **/
6587 static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6588                                   struct i40e_dcbx_config *new_cfg)
6589 {
6590         struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6591         int ret;
6592
6593         /* Check if need reconfiguration */
6594         if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {
6595                 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6596                 return 0;
6597         }
6598
6599         /* Config change disable all VSIs */
6600         i40e_pf_quiesce_all_vsi(pf);
6601
6602         /* Copy the new config to the current config */
6603         *old_cfg = *new_cfg;
6604         old_cfg->etsrec = old_cfg->etscfg;
6605         ret = i40e_set_dcb_config(&pf->hw);
6606         if (ret) {
6607                 dev_info(&pf->pdev->dev,
6608                          "Set DCB Config failed, err %s aq_err %s\n",
6609                          i40e_stat_str(&pf->hw, ret),
6610                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6611                 goto out;
6612         }
6613
6614         /* Changes in configuration update VEB/VSI */
6615         i40e_dcb_reconfigure(pf);
6616 out:
6617         /* In case of reset do not try to resume anything */
6618         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6619                 /* Re-start the VSIs if disabled */
6620                 ret = i40e_resume_port_tx(pf);
6621                 /* In case of error no point in resuming VSIs */
6622                 if (ret)
6623                         goto err;
6624                 i40e_pf_unquiesce_all_vsi(pf);
6625         }
6626 err:
6627         return ret;
6628 }
6629
6630 /**
6631  * i40e_hw_dcb_config - Program new DCBX settings into HW
6632  * @pf: PF being configured
6633  * @new_cfg: New DCBX configuration
6634  *
6635  * Program DCB settings into HW and reconfigure VEB/VSIs on
6636  * given PF
6637  **/
6638 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6639 {
6640         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6641         u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6642         u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6643         struct i40e_dcbx_config *old_cfg;
6644         u8 mode[I40E_MAX_TRAFFIC_CLASS];
6645         struct i40e_rx_pb_config pb_cfg;
6646         struct i40e_hw *hw = &pf->hw;
6647         u8 num_ports = hw->num_ports;
6648         bool need_reconfig;
6649         int ret = -EINVAL;
6650         u8 lltc_map = 0;
6651         u8 tc_map = 0;
6652         u8 new_numtc;
6653         u8 i;
6654
6655         dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6656         /* Un-pack information to Program ETS HW via shared API
6657          * numtc, tcmap
6658          * LLTC map
6659          * ETS/NON-ETS arbiter mode
6660          * max exponent (credit refills)
6661          * Total number of ports
6662          * PFC priority bit-map
6663          * Priority Table
6664          * BW % per TC
6665          * Arbiter mode between UPs sharing same TC
6666          * TSA table (ETS or non-ETS)
6667          * EEE enabled or not
6668          * MFS TC table
6669          */
6670
6671         new_numtc = i40e_dcb_get_num_tc(new_cfg);
6672
6673         memset(&ets_data, 0, sizeof(ets_data));
6674         for (i = 0; i < new_numtc; i++) {
6675                 tc_map |= BIT(i);
6676                 switch (new_cfg->etscfg.tsatable[i]) {
6677                 case I40E_IEEE_TSA_ETS:
6678                         prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
6679                         ets_data.tc_bw_share_credits[i] =
6680                                         new_cfg->etscfg.tcbwtable[i];
6681                         break;
6682                 case I40E_IEEE_TSA_STRICT:
6683                         prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
6684                         lltc_map |= BIT(i);
6685                         ets_data.tc_bw_share_credits[i] =
6686                                         I40E_DCB_STRICT_PRIO_CREDITS;
6687                         break;
6688                 default:
6689                         /* Invalid TSA type */
6690                         need_reconfig = false;
6691                         goto out;
6692                 }
6693         }
6694
6695         old_cfg = &hw->local_dcbx_config;
6696         /* Check if need reconfiguration */
6697         need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
6698
6699         /* If needed, enable/disable frame tagging, disable all VSIs
6700          * and suspend port tx
6701          */
6702         if (need_reconfig) {
6703                 /* Enable DCB tagging only when more than one TC */
6704                 if (new_numtc > 1)
6705                         pf->flags |= I40E_FLAG_DCB_ENABLED;
6706                 else
6707                         pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6708
6709                 set_bit(__I40E_PORT_SUSPENDED, pf->state);
6710                 /* Reconfiguration needed quiesce all VSIs */
6711                 i40e_pf_quiesce_all_vsi(pf);
6712                 ret = i40e_suspend_port_tx(pf);
6713                 if (ret)
6714                         goto err;
6715         }
6716
6717         /* Configure Port ETS Tx Scheduler */
6718         ets_data.tc_valid_bits = tc_map;
6719         ets_data.tc_strict_priority_flags = lltc_map;
6720         ret = i40e_aq_config_switch_comp_ets
6721                 (hw, pf->mac_seid, &ets_data,
6722                  i40e_aqc_opc_modify_switching_comp_ets, NULL);
6723         if (ret) {
6724                 dev_info(&pf->pdev->dev,
6725                          "Modify Port ETS failed, err %s aq_err %s\n",
6726                          i40e_stat_str(&pf->hw, ret),
6727                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6728                 goto out;
6729         }
6730
6731         /* Configure Rx ETS HW */
6732         memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
6733         i40e_dcb_hw_set_num_tc(hw, new_numtc);
6734         i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,
6735                                    I40E_DCB_ARB_MODE_STRICT_PRIORITY,
6736                                    I40E_DCB_DEFAULT_MAX_EXPONENT,
6737                                    lltc_map);
6738         i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);
6739         i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,
6740                                      prio_type);
6741         i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,
6742                                new_cfg->etscfg.prioritytable);
6743         i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);
6744
6745         /* Configure Rx Packet Buffers in HW */
6746         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6747                 mfs_tc[i] = pf->vsi[pf->lan_vsi]->netdev->mtu;
6748                 mfs_tc[i] += I40E_PACKET_HDR_PAD;
6749         }
6750
6751         i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
6752                                          false, new_cfg->pfc.pfcenable,
6753                                          mfs_tc, &pb_cfg);
6754         i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);
6755
6756         /* Update the local Rx Packet buffer config */
6757         pf->pb_cfg = pb_cfg;
6758
6759         /* Inform the FW about changes to DCB configuration */
6760         ret = i40e_aq_dcb_updated(&pf->hw, NULL);
6761         if (ret) {
6762                 dev_info(&pf->pdev->dev,
6763                          "DCB Updated failed, err %s aq_err %s\n",
6764                          i40e_stat_str(&pf->hw, ret),
6765                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6766                 goto out;
6767         }
6768
6769         /* Update the port DCBx configuration */
6770         *old_cfg = *new_cfg;
6771
6772         /* Changes in configuration update VEB/VSI */
6773         i40e_dcb_reconfigure(pf);
6774 out:
6775         /* Re-start the VSIs if disabled */
6776         if (need_reconfig) {
6777                 ret = i40e_resume_port_tx(pf);
6778
6779                 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
6780                 /* In case of error no point in resuming VSIs */
6781                 if (ret)
6782                         goto err;
6783
6784                 /* Wait for the PF's queues to be disabled */
6785                 ret = i40e_pf_wait_queues_disabled(pf);
6786                 if (ret) {
6787                         /* Schedule PF reset to recover */
6788                         set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6789                         i40e_service_event_schedule(pf);
6790                         goto err;
6791                 } else {
6792                         i40e_pf_unquiesce_all_vsi(pf);
6793                         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
6794                         set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
6795                 }
6796                 /* registers are set, lets apply */
6797                 if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB)
6798                         ret = i40e_hw_set_dcb_config(pf, new_cfg);
6799         }
6800
6801 err:
6802         return ret;
6803 }
6804
6805 /**
6806  * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
6807  * @pf: PF being queried
6808  *
6809  * Set default DCB configuration in case DCB is to be done in SW.
6810  **/
6811 int i40e_dcb_sw_default_config(struct i40e_pf *pf)
6812 {
6813         struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
6814         struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6815         struct i40e_hw *hw = &pf->hw;
6816         int err;
6817
6818         if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB) {
6819                 /* Update the local cached instance with TC0 ETS */
6820                 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
6821                 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
6822                 pf->tmp_cfg.etscfg.maxtcs = 0;
6823                 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
6824                 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
6825                 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
6826                 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
6827                 /* FW needs one App to configure HW */
6828                 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
6829                 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
6830                 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
6831                 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
6832
6833                 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);
6834         }
6835
6836         memset(&ets_data, 0, sizeof(ets_data));
6837         ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
6838         ets_data.tc_strict_priority_flags = 0; /* ETS */
6839         ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
6840
6841         /* Enable ETS on the Physical port */
6842         err = i40e_aq_config_switch_comp_ets
6843                 (hw, pf->mac_seid, &ets_data,
6844                  i40e_aqc_opc_enable_switching_comp_ets, NULL);
6845         if (err) {
6846                 dev_info(&pf->pdev->dev,
6847                          "Enable Port ETS failed, err %s aq_err %s\n",
6848                          i40e_stat_str(&pf->hw, err),
6849                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6850                 err = -ENOENT;
6851                 goto out;
6852         }
6853
6854         /* Update the local cached instance with TC0 ETS */
6855         dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
6856         dcb_cfg->etscfg.cbs = 0;
6857         dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
6858         dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
6859
6860 out:
6861         return err;
6862 }
6863
6864 /**
6865  * i40e_init_pf_dcb - Initialize DCB configuration
6866  * @pf: PF being configured
6867  *
6868  * Query the current DCB configuration and cache it
6869  * in the hardware structure
6870  **/
6871 static int i40e_init_pf_dcb(struct i40e_pf *pf)
6872 {
6873         struct i40e_hw *hw = &pf->hw;
6874         int err;
6875
6876         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable
6877          * Also do not enable DCBx if FW LLDP agent is disabled
6878          */
6879         if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) {
6880                 dev_info(&pf->pdev->dev, "DCB is not supported.\n");
6881                 err = I40E_NOT_SUPPORTED;
6882                 goto out;
6883         }
6884         if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) {
6885                 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
6886                 err = i40e_dcb_sw_default_config(pf);
6887                 if (err) {
6888                         dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
6889                         goto out;
6890                 }
6891                 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
6892                 pf->dcbx_cap = DCB_CAP_DCBX_HOST |
6893                                DCB_CAP_DCBX_VER_IEEE;
6894                 /* at init capable but disabled */
6895                 pf->flags |= I40E_FLAG_DCB_CAPABLE;
6896                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6897                 goto out;
6898         }
6899         err = i40e_init_dcb(hw, true);
6900         if (!err) {
6901                 /* Device/Function is not DCBX capable */
6902                 if ((!hw->func_caps.dcb) ||
6903                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
6904                         dev_info(&pf->pdev->dev,
6905                                  "DCBX offload is not supported or is disabled for this PF.\n");
6906                 } else {
6907                         /* When status is not DISABLED then DCBX in FW */
6908                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
6909                                        DCB_CAP_DCBX_VER_IEEE;
6910
6911                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
6912                         /* Enable DCB tagging only when more than one TC
6913                          * or explicitly disable if only one TC
6914                          */
6915                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6916                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
6917                         else
6918                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6919                         dev_dbg(&pf->pdev->dev,
6920                                 "DCBX offload is supported for this PF.\n");
6921                 }
6922         } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
6923                 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
6924                 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
6925         } else {
6926                 dev_info(&pf->pdev->dev,
6927                          "Query for DCB configuration failed, err %s aq_err %s\n",
6928                          i40e_stat_str(&pf->hw, err),
6929                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6930         }
6931
6932 out:
6933         return err;
6934 }
6935 #endif /* CONFIG_I40E_DCB */
6936
6937 /**
6938  * i40e_print_link_message - print link up or down
6939  * @vsi: the VSI for which link needs a message
6940  * @isup: true of link is up, false otherwise
6941  */
6942 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
6943 {
6944         enum i40e_aq_link_speed new_speed;
6945         struct i40e_pf *pf = vsi->back;
6946         char *speed = "Unknown";
6947         char *fc = "Unknown";
6948         char *fec = "";
6949         char *req_fec = "";
6950         char *an = "";
6951
6952         if (isup)
6953                 new_speed = pf->hw.phy.link_info.link_speed;
6954         else
6955                 new_speed = I40E_LINK_SPEED_UNKNOWN;
6956
6957         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
6958                 return;
6959         vsi->current_isup = isup;
6960         vsi->current_speed = new_speed;
6961         if (!isup) {
6962                 netdev_info(vsi->netdev, "NIC Link is Down\n");
6963                 return;
6964         }
6965
6966         /* Warn user if link speed on NPAR enabled partition is not at
6967          * least 10GB
6968          */
6969         if (pf->hw.func_caps.npar_enable &&
6970             (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
6971              pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
6972                 netdev_warn(vsi->netdev,
6973                             "The partition detected link speed that is less than 10Gbps\n");
6974
6975         switch (pf->hw.phy.link_info.link_speed) {
6976         case I40E_LINK_SPEED_40GB:
6977                 speed = "40 G";
6978                 break;
6979         case I40E_LINK_SPEED_20GB:
6980                 speed = "20 G";
6981                 break;
6982         case I40E_LINK_SPEED_25GB:
6983                 speed = "25 G";
6984                 break;
6985         case I40E_LINK_SPEED_10GB:
6986                 speed = "10 G";
6987                 break;
6988         case I40E_LINK_SPEED_5GB:
6989                 speed = "5 G";
6990                 break;
6991         case I40E_LINK_SPEED_2_5GB:
6992                 speed = "2.5 G";
6993                 break;
6994         case I40E_LINK_SPEED_1GB:
6995                 speed = "1000 M";
6996                 break;
6997         case I40E_LINK_SPEED_100MB:
6998                 speed = "100 M";
6999                 break;
7000         default:
7001                 break;
7002         }
7003
7004         switch (pf->hw.fc.current_mode) {
7005         case I40E_FC_FULL:
7006                 fc = "RX/TX";
7007                 break;
7008         case I40E_FC_TX_PAUSE:
7009                 fc = "TX";
7010                 break;
7011         case I40E_FC_RX_PAUSE:
7012                 fc = "RX";
7013                 break;
7014         default:
7015                 fc = "None";
7016                 break;
7017         }
7018
7019         if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
7020                 req_fec = "None";
7021                 fec = "None";
7022                 an = "False";
7023
7024                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7025                         an = "True";
7026
7027                 if (pf->hw.phy.link_info.fec_info &
7028                     I40E_AQ_CONFIG_FEC_KR_ENA)
7029                         fec = "CL74 FC-FEC/BASE-R";
7030                 else if (pf->hw.phy.link_info.fec_info &
7031                          I40E_AQ_CONFIG_FEC_RS_ENA)
7032                         fec = "CL108 RS-FEC";
7033
7034                 /* 'CL108 RS-FEC' should be displayed when RS is requested, or
7035                  * both RS and FC are requested
7036                  */
7037                 if (vsi->back->hw.phy.link_info.req_fec_info &
7038                     (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7039                         if (vsi->back->hw.phy.link_info.req_fec_info &
7040                             I40E_AQ_REQUEST_FEC_RS)
7041                                 req_fec = "CL108 RS-FEC";
7042                         else
7043                                 req_fec = "CL74 FC-FEC/BASE-R";
7044                 }
7045                 netdev_info(vsi->netdev,
7046                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7047                             speed, req_fec, fec, an, fc);
7048         } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7049                 req_fec = "None";
7050                 fec = "None";
7051                 an = "False";
7052
7053                 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7054                         an = "True";
7055
7056                 if (pf->hw.phy.link_info.fec_info &
7057                     I40E_AQ_CONFIG_FEC_KR_ENA)
7058                         fec = "CL74 FC-FEC/BASE-R";
7059
7060                 if (pf->hw.phy.link_info.req_fec_info &
7061                     I40E_AQ_REQUEST_FEC_KR)
7062                         req_fec = "CL74 FC-FEC/BASE-R";
7063
7064                 netdev_info(vsi->netdev,
7065                             "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7066                             speed, req_fec, fec, an, fc);
7067         } else {
7068                 netdev_info(vsi->netdev,
7069                             "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7070                             speed, fc);
7071         }
7072
7073 }
7074
7075 /**
7076  * i40e_up_complete - Finish the last steps of bringing up a connection
7077  * @vsi: the VSI being configured
7078  **/
7079 static int i40e_up_complete(struct i40e_vsi *vsi)
7080 {
7081         struct i40e_pf *pf = vsi->back;
7082         int err;
7083
7084         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7085                 i40e_vsi_configure_msix(vsi);
7086         else
7087                 i40e_configure_msi_and_legacy(vsi);
7088
7089         /* start rings */
7090         err = i40e_vsi_start_rings(vsi);
7091         if (err)
7092                 return err;
7093
7094         clear_bit(__I40E_VSI_DOWN, vsi->state);
7095         i40e_napi_enable_all(vsi);
7096         i40e_vsi_enable_irq(vsi);
7097
7098         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7099             (vsi->netdev)) {
7100                 i40e_print_link_message(vsi, true);
7101                 netif_tx_start_all_queues(vsi->netdev);
7102                 netif_carrier_on(vsi->netdev);
7103         }
7104
7105         /* replay FDIR SB filters */
7106         if (vsi->type == I40E_VSI_FDIR) {
7107                 /* reset fd counters */
7108                 pf->fd_add_err = 0;
7109                 pf->fd_atr_cnt = 0;
7110                 i40e_fdir_filter_restore(vsi);
7111         }
7112
7113         /* On the next run of the service_task, notify any clients of the new
7114          * opened netdev
7115          */
7116         set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7117         i40e_service_event_schedule(pf);
7118
7119         return 0;
7120 }
7121
7122 /**
7123  * i40e_vsi_reinit_locked - Reset the VSI
7124  * @vsi: the VSI being configured
7125  *
7126  * Rebuild the ring structs after some configuration
7127  * has changed, e.g. MTU size.
7128  **/
7129 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7130 {
7131         struct i40e_pf *pf = vsi->back;
7132
7133         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
7134                 usleep_range(1000, 2000);
7135         i40e_down(vsi);
7136
7137         i40e_up(vsi);
7138         clear_bit(__I40E_CONFIG_BUSY, pf->state);
7139 }
7140
7141 /**
7142  * i40e_force_link_state - Force the link status
7143  * @pf: board private structure
7144  * @is_up: whether the link state should be forced up or down
7145  **/
7146 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7147 {
7148         struct i40e_aq_get_phy_abilities_resp abilities;
7149         struct i40e_aq_set_phy_config config = {0};
7150         bool non_zero_phy_type = is_up;
7151         struct i40e_hw *hw = &pf->hw;
7152         i40e_status err;
7153         u64 mask;
7154         u8 speed;
7155
7156         /* Card might've been put in an unstable state by other drivers
7157          * and applications, which causes incorrect speed values being
7158          * set on startup. In order to clear speed registers, we call
7159          * get_phy_capabilities twice, once to get initial state of
7160          * available speeds, and once to get current PHY config.
7161          */
7162         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
7163                                            NULL);
7164         if (err) {
7165                 dev_err(&pf->pdev->dev,
7166                         "failed to get phy cap., ret =  %s last_status =  %s\n",
7167                         i40e_stat_str(hw, err),
7168                         i40e_aq_str(hw, hw->aq.asq_last_status));
7169                 return err;
7170         }
7171         speed = abilities.link_speed;
7172
7173         /* Get the current phy config */
7174         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
7175                                            NULL);
7176         if (err) {
7177                 dev_err(&pf->pdev->dev,
7178                         "failed to get phy cap., ret =  %s last_status =  %s\n",
7179                         i40e_stat_str(hw, err),
7180                         i40e_aq_str(hw, hw->aq.asq_last_status));
7181                 return err;
7182         }
7183
7184         /* If link needs to go up, but was not forced to go down,
7185          * and its speed values are OK, no need for a flap
7186          * if non_zero_phy_type was set, still need to force up
7187          */
7188         if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)
7189                 non_zero_phy_type = true;
7190         else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7191                 return I40E_SUCCESS;
7192
7193         /* To force link we need to set bits for all supported PHY types,
7194          * but there are now more than 32, so we need to split the bitmap
7195          * across two fields.
7196          */
7197         mask = I40E_PHY_TYPES_BITMASK;
7198         config.phy_type =
7199                 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7200         config.phy_type_ext =
7201                 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7202         /* Copy the old settings, except of phy_type */
7203         config.abilities = abilities.abilities;
7204         if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) {
7205                 if (is_up)
7206                         config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7207                 else
7208                         config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7209         }
7210         if (abilities.link_speed != 0)
7211                 config.link_speed = abilities.link_speed;
7212         else
7213                 config.link_speed = speed;
7214         config.eee_capability = abilities.eee_capability;
7215         config.eeer = abilities.eeer_val;
7216         config.low_power_ctrl = abilities.d3_lpan;
7217         config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7218                             I40E_AQ_PHY_FEC_CONFIG_MASK;
7219         err = i40e_aq_set_phy_config(hw, &config, NULL);
7220
7221         if (err) {
7222                 dev_err(&pf->pdev->dev,
7223                         "set phy config ret =  %s last_status =  %s\n",
7224                         i40e_stat_str(&pf->hw, err),
7225                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7226                 return err;
7227         }
7228
7229         /* Update the link info */
7230         err = i40e_update_link_info(hw);
7231         if (err) {
7232                 /* Wait a little bit (on 40G cards it sometimes takes a really
7233                  * long time for link to come back from the atomic reset)
7234                  * and try once more
7235                  */
7236                 msleep(1000);
7237                 i40e_update_link_info(hw);
7238         }
7239
7240         i40e_aq_set_link_restart_an(hw, is_up, NULL);
7241
7242         return I40E_SUCCESS;
7243 }
7244
7245 /**
7246  * i40e_up - Bring the connection back up after being down
7247  * @vsi: the VSI being configured
7248  **/
7249 int i40e_up(struct i40e_vsi *vsi)
7250 {
7251         int err;
7252
7253         if (vsi->type == I40E_VSI_MAIN &&
7254             (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7255              vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7256                 i40e_force_link_state(vsi->back, true);
7257
7258         err = i40e_vsi_configure(vsi);
7259         if (!err)
7260                 err = i40e_up_complete(vsi);
7261
7262         return err;
7263 }
7264
7265 /**
7266  * i40e_down - Shutdown the connection processing
7267  * @vsi: the VSI being stopped
7268  **/
7269 void i40e_down(struct i40e_vsi *vsi)
7270 {
7271         int i;
7272
7273         /* It is assumed that the caller of this function
7274          * sets the vsi->state __I40E_VSI_DOWN bit.
7275          */
7276         if (vsi->netdev) {
7277                 netif_carrier_off(vsi->netdev);
7278                 netif_tx_disable(vsi->netdev);
7279         }
7280         i40e_vsi_disable_irq(vsi);
7281         i40e_vsi_stop_rings(vsi);
7282         if (vsi->type == I40E_VSI_MAIN &&
7283            (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7284             vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7285                 i40e_force_link_state(vsi->back, false);
7286         i40e_napi_disable_all(vsi);
7287
7288         for (i = 0; i < vsi->num_queue_pairs; i++) {
7289                 i40e_clean_tx_ring(vsi->tx_rings[i]);
7290                 if (i40e_enabled_xdp_vsi(vsi)) {
7291                         /* Make sure that in-progress ndo_xdp_xmit and
7292                          * ndo_xsk_wakeup calls are completed.
7293                          */
7294                         synchronize_rcu();
7295                         i40e_clean_tx_ring(vsi->xdp_rings[i]);
7296                 }
7297                 i40e_clean_rx_ring(vsi->rx_rings[i]);
7298         }
7299
7300 }
7301
7302 /**
7303  * i40e_validate_mqprio_qopt- validate queue mapping info
7304  * @vsi: the VSI being configured
7305  * @mqprio_qopt: queue parametrs
7306  **/
7307 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7308                                      struct tc_mqprio_qopt_offload *mqprio_qopt)
7309 {
7310         u64 sum_max_rate = 0;
7311         u64 max_rate = 0;
7312         int i;
7313
7314         if (mqprio_qopt->qopt.offset[0] != 0 ||
7315             mqprio_qopt->qopt.num_tc < 1 ||
7316             mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7317                 return -EINVAL;
7318         for (i = 0; ; i++) {
7319                 if (!mqprio_qopt->qopt.count[i])
7320                         return -EINVAL;
7321                 if (mqprio_qopt->min_rate[i]) {
7322                         dev_err(&vsi->back->pdev->dev,
7323                                 "Invalid min tx rate (greater than 0) specified\n");
7324                         return -EINVAL;
7325                 }
7326                 max_rate = mqprio_qopt->max_rate[i];
7327                 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7328                 sum_max_rate += max_rate;
7329
7330                 if (i >= mqprio_qopt->qopt.num_tc - 1)
7331                         break;
7332                 if (mqprio_qopt->qopt.offset[i + 1] !=
7333                     (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7334                         return -EINVAL;
7335         }
7336         if (vsi->num_queue_pairs <
7337             (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7338                 dev_err(&vsi->back->pdev->dev,
7339                         "Failed to create traffic channel, insufficient number of queues.\n");
7340                 return -EINVAL;
7341         }
7342         if (sum_max_rate > i40e_get_link_speed(vsi)) {
7343                 dev_err(&vsi->back->pdev->dev,
7344                         "Invalid max tx rate specified\n");
7345                 return -EINVAL;
7346         }
7347         return 0;
7348 }
7349
7350 /**
7351  * i40e_vsi_set_default_tc_config - set default values for tc configuration
7352  * @vsi: the VSI being configured
7353  **/
7354 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7355 {
7356         u16 qcount;
7357         int i;
7358
7359         /* Only TC0 is enabled */
7360         vsi->tc_config.numtc = 1;
7361         vsi->tc_config.enabled_tc = 1;
7362         qcount = min_t(int, vsi->alloc_queue_pairs,
7363                        i40e_pf_get_max_q_per_tc(vsi->back));
7364         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7365                 /* For the TC that is not enabled set the offset to default
7366                  * queue and allocate one queue for the given TC.
7367                  */
7368                 vsi->tc_config.tc_info[i].qoffset = 0;
7369                 if (i == 0)
7370                         vsi->tc_config.tc_info[i].qcount = qcount;
7371                 else
7372                         vsi->tc_config.tc_info[i].qcount = 1;
7373                 vsi->tc_config.tc_info[i].netdev_tc = 0;
7374         }
7375 }
7376
7377 /**
7378  * i40e_del_macvlan_filter
7379  * @hw: pointer to the HW structure
7380  * @seid: seid of the channel VSI
7381  * @macaddr: the mac address to apply as a filter
7382  * @aq_err: store the admin Q error
7383  *
7384  * This function deletes a mac filter on the channel VSI which serves as the
7385  * macvlan. Returns 0 on success.
7386  **/
7387 static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7388                                            const u8 *macaddr, int *aq_err)
7389 {
7390         struct i40e_aqc_remove_macvlan_element_data element;
7391         i40e_status status;
7392
7393         memset(&element, 0, sizeof(element));
7394         ether_addr_copy(element.mac_addr, macaddr);
7395         element.vlan_tag = 0;
7396         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7397         status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7398         *aq_err = hw->aq.asq_last_status;
7399
7400         return status;
7401 }
7402
7403 /**
7404  * i40e_add_macvlan_filter
7405  * @hw: pointer to the HW structure
7406  * @seid: seid of the channel VSI
7407  * @macaddr: the mac address to apply as a filter
7408  * @aq_err: store the admin Q error
7409  *
7410  * This function adds a mac filter on the channel VSI which serves as the
7411  * macvlan. Returns 0 on success.
7412  **/
7413 static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7414                                            const u8 *macaddr, int *aq_err)
7415 {
7416         struct i40e_aqc_add_macvlan_element_data element;
7417         i40e_status status;
7418         u16 cmd_flags = 0;
7419
7420         ether_addr_copy(element.mac_addr, macaddr);
7421         element.vlan_tag = 0;
7422         element.queue_number = 0;
7423         element.match_method = I40E_AQC_MM_ERR_NO_RES;
7424         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7425         element.flags = cpu_to_le16(cmd_flags);
7426         status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7427         *aq_err = hw->aq.asq_last_status;
7428
7429         return status;
7430 }
7431
7432 /**
7433  * i40e_reset_ch_rings - Reset the queue contexts in a channel
7434  * @vsi: the VSI we want to access
7435  * @ch: the channel we want to access
7436  */
7437 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7438 {
7439         struct i40e_ring *tx_ring, *rx_ring;
7440         u16 pf_q;
7441         int i;
7442
7443         for (i = 0; i < ch->num_queue_pairs; i++) {
7444                 pf_q = ch->base_queue + i;
7445                 tx_ring = vsi->tx_rings[pf_q];
7446                 tx_ring->ch = NULL;
7447                 rx_ring = vsi->rx_rings[pf_q];
7448                 rx_ring->ch = NULL;
7449         }
7450 }
7451
7452 /**
7453  * i40e_free_macvlan_channels
7454  * @vsi: the VSI we want to access
7455  *
7456  * This function frees the Qs of the channel VSI from
7457  * the stack and also deletes the channel VSIs which
7458  * serve as macvlans.
7459  */
7460 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7461 {
7462         struct i40e_channel *ch, *ch_tmp;
7463         int ret;
7464
7465         if (list_empty(&vsi->macvlan_list))
7466                 return;
7467
7468         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7469                 struct i40e_vsi *parent_vsi;
7470
7471                 if (i40e_is_channel_macvlan(ch)) {
7472                         i40e_reset_ch_rings(vsi, ch);
7473                         clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7474                         netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7475                         netdev_set_sb_channel(ch->fwd->netdev, 0);
7476                         kfree(ch->fwd);
7477                         ch->fwd = NULL;
7478                 }
7479
7480                 list_del(&ch->list);
7481                 parent_vsi = ch->parent_vsi;
7482                 if (!parent_vsi || !ch->initialized) {
7483                         kfree(ch);
7484                         continue;
7485                 }
7486
7487                 /* remove the VSI */
7488                 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7489                                              NULL);
7490                 if (ret)
7491                         dev_err(&vsi->back->pdev->dev,
7492                                 "unable to remove channel (%d) for parent VSI(%d)\n",
7493                                 ch->seid, parent_vsi->seid);
7494                 kfree(ch);
7495         }
7496         vsi->macvlan_cnt = 0;
7497 }
7498
7499 /**
7500  * i40e_fwd_ring_up - bring the macvlan device up
7501  * @vsi: the VSI we want to access
7502  * @vdev: macvlan netdevice
7503  * @fwd: the private fwd structure
7504  */
7505 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7506                             struct i40e_fwd_adapter *fwd)
7507 {
7508         int ret = 0, num_tc = 1,  i, aq_err;
7509         struct i40e_channel *ch, *ch_tmp;
7510         struct i40e_pf *pf = vsi->back;
7511         struct i40e_hw *hw = &pf->hw;
7512
7513         if (list_empty(&vsi->macvlan_list))
7514                 return -EINVAL;
7515
7516         /* Go through the list and find an available channel */
7517         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7518                 if (!i40e_is_channel_macvlan(ch)) {
7519                         ch->fwd = fwd;
7520                         /* record configuration for macvlan interface in vdev */
7521                         for (i = 0; i < num_tc; i++)
7522                                 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7523                                                              i,
7524                                                              ch->num_queue_pairs,
7525                                                              ch->base_queue);
7526                         for (i = 0; i < ch->num_queue_pairs; i++) {
7527                                 struct i40e_ring *tx_ring, *rx_ring;
7528                                 u16 pf_q;
7529
7530                                 pf_q = ch->base_queue + i;
7531
7532                                 /* Get to TX ring ptr */
7533                                 tx_ring = vsi->tx_rings[pf_q];
7534                                 tx_ring->ch = ch;
7535
7536                                 /* Get the RX ring ptr */
7537                                 rx_ring = vsi->rx_rings[pf_q];
7538                                 rx_ring->ch = ch;
7539                         }
7540                         break;
7541                 }
7542         }
7543
7544         /* Guarantee all rings are updated before we update the
7545          * MAC address filter.
7546          */
7547         wmb();
7548
7549         /* Add a mac filter */
7550         ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7551         if (ret) {
7552                 /* if we cannot add the MAC rule then disable the offload */
7553                 macvlan_release_l2fw_offload(vdev);
7554                 for (i = 0; i < ch->num_queue_pairs; i++) {
7555                         struct i40e_ring *rx_ring;
7556                         u16 pf_q;
7557
7558                         pf_q = ch->base_queue + i;
7559                         rx_ring = vsi->rx_rings[pf_q];
7560                         rx_ring->netdev = NULL;
7561                 }
7562                 dev_info(&pf->pdev->dev,
7563                          "Error adding mac filter on macvlan err %s, aq_err %s\n",
7564                           i40e_stat_str(hw, ret),
7565                           i40e_aq_str(hw, aq_err));
7566                 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7567         }
7568
7569         return ret;
7570 }
7571
7572 /**
7573  * i40e_setup_macvlans - create the channels which will be macvlans
7574  * @vsi: the VSI we want to access
7575  * @macvlan_cnt: no. of macvlans to be setup
7576  * @qcnt: no. of Qs per macvlan
7577  * @vdev: macvlan netdevice
7578  */
7579 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7580                                struct net_device *vdev)
7581 {
7582         struct i40e_pf *pf = vsi->back;
7583         struct i40e_hw *hw = &pf->hw;
7584         struct i40e_vsi_context ctxt;
7585         u16 sections, qmap, num_qps;
7586         struct i40e_channel *ch;
7587         int i, pow, ret = 0;
7588         u8 offset = 0;
7589
7590         if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7591                 return -EINVAL;
7592
7593         num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7594
7595         /* find the next higher power-of-2 of num queue pairs */
7596         pow = fls(roundup_pow_of_two(num_qps) - 1);
7597
7598         qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7599                 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7600
7601         /* Setup context bits for the main VSI */
7602         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7603         sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7604         memset(&ctxt, 0, sizeof(ctxt));
7605         ctxt.seid = vsi->seid;
7606         ctxt.pf_num = vsi->back->hw.pf_id;
7607         ctxt.vf_num = 0;
7608         ctxt.uplink_seid = vsi->uplink_seid;
7609         ctxt.info = vsi->info;
7610         ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7611         ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7612         ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7613         ctxt.info.valid_sections |= cpu_to_le16(sections);
7614
7615         /* Reconfigure RSS for main VSI with new max queue count */
7616         vsi->rss_size = max_t(u16, num_qps, qcnt);
7617         ret = i40e_vsi_config_rss(vsi);
7618         if (ret) {
7619                 dev_info(&pf->pdev->dev,
7620                          "Failed to reconfig RSS for num_queues (%u)\n",
7621                          vsi->rss_size);
7622                 return ret;
7623         }
7624         vsi->reconfig_rss = true;
7625         dev_dbg(&vsi->back->pdev->dev,
7626                 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7627         vsi->next_base_queue = num_qps;
7628         vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7629
7630         /* Update the VSI after updating the VSI queue-mapping
7631          * information
7632          */
7633         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7634         if (ret) {
7635                 dev_info(&pf->pdev->dev,
7636                          "Update vsi tc config failed, err %s aq_err %s\n",
7637                          i40e_stat_str(hw, ret),
7638                          i40e_aq_str(hw, hw->aq.asq_last_status));
7639                 return ret;
7640         }
7641         /* update the local VSI info with updated queue map */
7642         i40e_vsi_update_queue_map(vsi, &ctxt);
7643         vsi->info.valid_sections = 0;
7644
7645         /* Create channels for macvlans */
7646         INIT_LIST_HEAD(&vsi->macvlan_list);
7647         for (i = 0; i < macvlan_cnt; i++) {
7648                 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7649                 if (!ch) {
7650                         ret = -ENOMEM;
7651                         goto err_free;
7652                 }
7653                 INIT_LIST_HEAD(&ch->list);
7654                 ch->num_queue_pairs = qcnt;
7655                 if (!i40e_setup_channel(pf, vsi, ch)) {
7656                         ret = -EINVAL;
7657                         kfree(ch);
7658                         goto err_free;
7659                 }
7660                 ch->parent_vsi = vsi;
7661                 vsi->cnt_q_avail -= ch->num_queue_pairs;
7662                 vsi->macvlan_cnt++;
7663                 list_add_tail(&ch->list, &vsi->macvlan_list);
7664         }
7665
7666         return ret;
7667
7668 err_free:
7669         dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7670         i40e_free_macvlan_channels(vsi);
7671
7672         return ret;
7673 }
7674
7675 /**
7676  * i40e_fwd_add - configure macvlans
7677  * @netdev: net device to configure
7678  * @vdev: macvlan netdevice
7679  **/
7680 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7681 {
7682         struct i40e_netdev_priv *np = netdev_priv(netdev);
7683         u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7684         struct i40e_vsi *vsi = np->vsi;
7685         struct i40e_pf *pf = vsi->back;
7686         struct i40e_fwd_adapter *fwd;
7687         int avail_macvlan, ret;
7688
7689         if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
7690                 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
7691                 return ERR_PTR(-EINVAL);
7692         }
7693         if ((pf->flags & I40E_FLAG_TC_MQPRIO)) {
7694                 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
7695                 return ERR_PTR(-EINVAL);
7696         }
7697         if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
7698                 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
7699                 return ERR_PTR(-EINVAL);
7700         }
7701
7702         /* The macvlan device has to be a single Q device so that the
7703          * tc_to_txq field can be reused to pick the tx queue.
7704          */
7705         if (netif_is_multiqueue(vdev))
7706                 return ERR_PTR(-ERANGE);
7707
7708         if (!vsi->macvlan_cnt) {
7709                 /* reserve bit 0 for the pf device */
7710                 set_bit(0, vsi->fwd_bitmask);
7711
7712                 /* Try to reserve as many queues as possible for macvlans. First
7713                  * reserve 3/4th of max vectors, then half, then quarter and
7714                  * calculate Qs per macvlan as you go
7715                  */
7716                 vectors = pf->num_lan_msix;
7717                 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
7718                         /* allocate 4 Qs per macvlan and 32 Qs to the PF*/
7719                         q_per_macvlan = 4;
7720                         macvlan_cnt = (vectors - 32) / 4;
7721                 } else if (vectors <= 64 && vectors > 32) {
7722                         /* allocate 2 Qs per macvlan and 16 Qs to the PF*/
7723                         q_per_macvlan = 2;
7724                         macvlan_cnt = (vectors - 16) / 2;
7725                 } else if (vectors <= 32 && vectors > 16) {
7726                         /* allocate 1 Q per macvlan and 16 Qs to the PF*/
7727                         q_per_macvlan = 1;
7728                         macvlan_cnt = vectors - 16;
7729                 } else if (vectors <= 16 && vectors > 8) {
7730                         /* allocate 1 Q per macvlan and 8 Qs to the PF */
7731                         q_per_macvlan = 1;
7732                         macvlan_cnt = vectors - 8;
7733                 } else {
7734                         /* allocate 1 Q per macvlan and 1 Q to the PF */
7735                         q_per_macvlan = 1;
7736                         macvlan_cnt = vectors - 1;
7737                 }
7738
7739                 if (macvlan_cnt == 0)
7740                         return ERR_PTR(-EBUSY);
7741
7742                 /* Quiesce VSI queues */
7743                 i40e_quiesce_vsi(vsi);
7744
7745                 /* sets up the macvlans but does not "enable" them */
7746                 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
7747                                           vdev);
7748                 if (ret)
7749                         return ERR_PTR(ret);
7750
7751                 /* Unquiesce VSI */
7752                 i40e_unquiesce_vsi(vsi);
7753         }
7754         avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
7755                                             vsi->macvlan_cnt);
7756         if (avail_macvlan >= I40E_MAX_MACVLANS)
7757                 return ERR_PTR(-EBUSY);
7758
7759         /* create the fwd struct */
7760         fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
7761         if (!fwd)
7762                 return ERR_PTR(-ENOMEM);
7763
7764         set_bit(avail_macvlan, vsi->fwd_bitmask);
7765         fwd->bit_no = avail_macvlan;
7766         netdev_set_sb_channel(vdev, avail_macvlan);
7767         fwd->netdev = vdev;
7768
7769         if (!netif_running(netdev))
7770                 return fwd;
7771
7772         /* Set fwd ring up */
7773         ret = i40e_fwd_ring_up(vsi, vdev, fwd);
7774         if (ret) {
7775                 /* unbind the queues and drop the subordinate channel config */
7776                 netdev_unbind_sb_channel(netdev, vdev);
7777                 netdev_set_sb_channel(vdev, 0);
7778
7779                 kfree(fwd);
7780                 return ERR_PTR(-EINVAL);
7781         }
7782
7783         return fwd;
7784 }
7785
7786 /**
7787  * i40e_del_all_macvlans - Delete all the mac filters on the channels
7788  * @vsi: the VSI we want to access
7789  */
7790 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
7791 {
7792         struct i40e_channel *ch, *ch_tmp;
7793         struct i40e_pf *pf = vsi->back;
7794         struct i40e_hw *hw = &pf->hw;
7795         int aq_err, ret = 0;
7796
7797         if (list_empty(&vsi->macvlan_list))
7798                 return;
7799
7800         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7801                 if (i40e_is_channel_macvlan(ch)) {
7802                         ret = i40e_del_macvlan_filter(hw, ch->seid,
7803                                                       i40e_channel_mac(ch),
7804                                                       &aq_err);
7805                         if (!ret) {
7806                                 /* Reset queue contexts */
7807                                 i40e_reset_ch_rings(vsi, ch);
7808                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7809                                 netdev_unbind_sb_channel(vsi->netdev,
7810                                                          ch->fwd->netdev);
7811                                 netdev_set_sb_channel(ch->fwd->netdev, 0);
7812                                 kfree(ch->fwd);
7813                                 ch->fwd = NULL;
7814                         }
7815                 }
7816         }
7817 }
7818
7819 /**
7820  * i40e_fwd_del - delete macvlan interfaces
7821  * @netdev: net device to configure
7822  * @vdev: macvlan netdevice
7823  */
7824 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
7825 {
7826         struct i40e_netdev_priv *np = netdev_priv(netdev);
7827         struct i40e_fwd_adapter *fwd = vdev;
7828         struct i40e_channel *ch, *ch_tmp;
7829         struct i40e_vsi *vsi = np->vsi;
7830         struct i40e_pf *pf = vsi->back;
7831         struct i40e_hw *hw = &pf->hw;
7832         int aq_err, ret = 0;
7833
7834         /* Find the channel associated with the macvlan and del mac filter */
7835         list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7836                 if (i40e_is_channel_macvlan(ch) &&
7837                     ether_addr_equal(i40e_channel_mac(ch),
7838                                      fwd->netdev->dev_addr)) {
7839                         ret = i40e_del_macvlan_filter(hw, ch->seid,
7840                                                       i40e_channel_mac(ch),
7841                                                       &aq_err);
7842                         if (!ret) {
7843                                 /* Reset queue contexts */
7844                                 i40e_reset_ch_rings(vsi, ch);
7845                                 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7846                                 netdev_unbind_sb_channel(netdev, fwd->netdev);
7847                                 netdev_set_sb_channel(fwd->netdev, 0);
7848                                 kfree(ch->fwd);
7849                                 ch->fwd = NULL;
7850                         } else {
7851                                 dev_info(&pf->pdev->dev,
7852                                          "Error deleting mac filter on macvlan err %s, aq_err %s\n",
7853                                           i40e_stat_str(hw, ret),
7854                                           i40e_aq_str(hw, aq_err));
7855                         }
7856                         break;
7857                 }
7858         }
7859 }
7860
7861 /**
7862  * i40e_setup_tc - configure multiple traffic classes
7863  * @netdev: net device to configure
7864  * @type_data: tc offload data
7865  **/
7866 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
7867 {
7868         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
7869         struct i40e_netdev_priv *np = netdev_priv(netdev);
7870         struct i40e_vsi *vsi = np->vsi;
7871         struct i40e_pf *pf = vsi->back;
7872         u8 enabled_tc = 0, num_tc, hw;
7873         bool need_reset = false;
7874         int old_queue_pairs;
7875         int ret = -EINVAL;
7876         u16 mode;
7877         int i;
7878
7879         old_queue_pairs = vsi->num_queue_pairs;
7880         num_tc = mqprio_qopt->qopt.num_tc;
7881         hw = mqprio_qopt->qopt.hw;
7882         mode = mqprio_qopt->mode;
7883         if (!hw) {
7884                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7885                 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
7886                 goto config_tc;
7887         }
7888
7889         /* Check if MFP enabled */
7890         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
7891                 netdev_info(netdev,
7892                             "Configuring TC not supported in MFP mode\n");
7893                 return ret;
7894         }
7895         switch (mode) {
7896         case TC_MQPRIO_MODE_DCB:
7897                 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7898
7899                 /* Check if DCB enabled to continue */
7900                 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7901                         netdev_info(netdev,
7902                                     "DCB is not enabled for adapter\n");
7903                         return ret;
7904                 }
7905
7906                 /* Check whether tc count is within enabled limit */
7907                 if (num_tc > i40e_pf_get_num_tc(pf)) {
7908                         netdev_info(netdev,
7909                                     "TC count greater than enabled on link for adapter\n");
7910                         return ret;
7911                 }
7912                 break;
7913         case TC_MQPRIO_MODE_CHANNEL:
7914                 if (pf->flags & I40E_FLAG_DCB_ENABLED) {
7915                         netdev_info(netdev,
7916                                     "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
7917                         return ret;
7918                 }
7919                 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7920                         return ret;
7921                 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
7922                 if (ret)
7923                         return ret;
7924                 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
7925                        sizeof(*mqprio_qopt));
7926                 pf->flags |= I40E_FLAG_TC_MQPRIO;
7927                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7928                 break;
7929         default:
7930                 return -EINVAL;
7931         }
7932
7933 config_tc:
7934         /* Generate TC map for number of tc requested */
7935         for (i = 0; i < num_tc; i++)
7936                 enabled_tc |= BIT(i);
7937
7938         /* Requesting same TC configuration as already enabled */
7939         if (enabled_tc == vsi->tc_config.enabled_tc &&
7940             mode != TC_MQPRIO_MODE_CHANNEL)
7941                 return 0;
7942
7943         /* Quiesce VSI queues */
7944         i40e_quiesce_vsi(vsi);
7945
7946         if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO))
7947                 i40e_remove_queue_channels(vsi);
7948
7949         /* Configure VSI for enabled TCs */
7950         ret = i40e_vsi_config_tc(vsi, enabled_tc);
7951         if (ret) {
7952                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
7953                             vsi->seid);
7954                 need_reset = true;
7955                 goto exit;
7956         } else if (enabled_tc &&
7957                    (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
7958                 netdev_info(netdev,
7959                             "Failed to create channel. Override queues (%u) not power of 2\n",
7960                             vsi->tc_config.tc_info[0].qcount);
7961                 ret = -EINVAL;
7962                 need_reset = true;
7963                 goto exit;
7964         }
7965
7966         dev_info(&vsi->back->pdev->dev,
7967                  "Setup channel (id:%u) utilizing num_queues %d\n",
7968                  vsi->seid, vsi->tc_config.tc_info[0].qcount);
7969
7970         if (pf->flags & I40E_FLAG_TC_MQPRIO) {
7971                 if (vsi->mqprio_qopt.max_rate[0]) {
7972                         u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
7973
7974                         do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
7975                         ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
7976                         if (!ret) {
7977                                 u64 credits = max_tx_rate;
7978
7979                                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
7980                                 dev_dbg(&vsi->back->pdev->dev,
7981                                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
7982                                         max_tx_rate,
7983                                         credits,
7984                                         vsi->seid);
7985                         } else {
7986                                 need_reset = true;
7987                                 goto exit;
7988                         }
7989                 }
7990                 ret = i40e_configure_queue_channels(vsi);
7991                 if (ret) {
7992                         vsi->num_queue_pairs = old_queue_pairs;
7993                         netdev_info(netdev,
7994                                     "Failed configuring queue channels\n");
7995                         need_reset = true;
7996                         goto exit;
7997                 }
7998         }
7999
8000 exit:
8001         /* Reset the configuration data to defaults, only TC0 is enabled */
8002         if (need_reset) {
8003                 i40e_vsi_set_default_tc_config(vsi);
8004                 need_reset = false;
8005         }
8006
8007         /* Unquiesce VSI */
8008         i40e_unquiesce_vsi(vsi);
8009         return ret;
8010 }
8011
8012 /**
8013  * i40e_set_cld_element - sets cloud filter element data
8014  * @filter: cloud filter rule
8015  * @cld: ptr to cloud filter element data
8016  *
8017  * This is helper function to copy data into cloud filter element
8018  **/
8019 static inline void
8020 i40e_set_cld_element(struct i40e_cloud_filter *filter,
8021                      struct i40e_aqc_cloud_filters_element_data *cld)
8022 {
8023         u32 ipa;
8024         int i;
8025
8026         memset(cld, 0, sizeof(*cld));
8027         ether_addr_copy(cld->outer_mac, filter->dst_mac);
8028         ether_addr_copy(cld->inner_mac, filter->src_mac);
8029
8030         if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
8031                 return;
8032
8033         if (filter->n_proto == ETH_P_IPV6) {
8034 #define IPV6_MAX_INDEX  (ARRAY_SIZE(filter->dst_ipv6) - 1)
8035                 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
8036                         ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
8037
8038                         *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
8039                 }
8040         } else {
8041                 ipa = be32_to_cpu(filter->dst_ipv4);
8042
8043                 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8044         }
8045
8046         cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8047
8048         /* tenant_id is not supported by FW now, once the support is enabled
8049          * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8050          */
8051         if (filter->tenant_id)
8052                 return;
8053 }
8054
8055 /**
8056  * i40e_add_del_cloud_filter - Add/del cloud filter
8057  * @vsi: pointer to VSI
8058  * @filter: cloud filter rule
8059  * @add: if true, add, if false, delete
8060  *
8061  * Add or delete a cloud filter for a specific flow spec.
8062  * Returns 0 if the filter were successfully added.
8063  **/
8064 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8065                               struct i40e_cloud_filter *filter, bool add)
8066 {
8067         struct i40e_aqc_cloud_filters_element_data cld_filter;
8068         struct i40e_pf *pf = vsi->back;
8069         int ret;
8070         static const u16 flag_table[128] = {
8071                 [I40E_CLOUD_FILTER_FLAGS_OMAC]  =
8072                         I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8073                 [I40E_CLOUD_FILTER_FLAGS_IMAC]  =
8074                         I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8075                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
8076                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8077                 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8078                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8079                 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8080                         I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8081                 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8082                         I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8083                 [I40E_CLOUD_FILTER_FLAGS_IIP] =
8084                         I40E_AQC_ADD_CLOUD_FILTER_IIP,
8085         };
8086
8087         if (filter->flags >= ARRAY_SIZE(flag_table))
8088                 return I40E_ERR_CONFIG;
8089
8090         memset(&cld_filter, 0, sizeof(cld_filter));
8091
8092         /* copy element needed to add cloud filter from filter */
8093         i40e_set_cld_element(filter, &cld_filter);
8094
8095         if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8096                 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8097                                              I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8098
8099         if (filter->n_proto == ETH_P_IPV6)
8100                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8101                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8102         else
8103                 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8104                                                 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8105
8106         if (add)
8107                 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
8108                                                 &cld_filter, 1);
8109         else
8110                 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
8111                                                 &cld_filter, 1);
8112         if (ret)
8113                 dev_dbg(&pf->pdev->dev,
8114                         "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8115                         add ? "add" : "delete", filter->dst_port, ret,
8116                         pf->hw.aq.asq_last_status);
8117         else
8118                 dev_info(&pf->pdev->dev,
8119                          "%s cloud filter for VSI: %d\n",
8120                          add ? "Added" : "Deleted", filter->seid);
8121         return ret;
8122 }
8123
8124 /**
8125  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8126  * @vsi: pointer to VSI
8127  * @filter: cloud filter rule
8128  * @add: if true, add, if false, delete
8129  *
8130  * Add or delete a cloud filter for a specific flow spec using big buffer.
8131  * Returns 0 if the filter were successfully added.
8132  **/
8133 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8134                                       struct i40e_cloud_filter *filter,
8135                                       bool add)
8136 {
8137         struct i40e_aqc_cloud_filters_element_bb cld_filter;
8138         struct i40e_pf *pf = vsi->back;
8139         int ret;
8140
8141         /* Both (src/dst) valid mac_addr are not supported */
8142         if ((is_valid_ether_addr(filter->dst_mac) &&
8143              is_valid_ether_addr(filter->src_mac)) ||
8144             (is_multicast_ether_addr(filter->dst_mac) &&
8145              is_multicast_ether_addr(filter->src_mac)))
8146                 return -EOPNOTSUPP;
8147
8148         /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8149          * ports are not supported via big buffer now.
8150          */
8151         if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8152                 return -EOPNOTSUPP;
8153
8154         /* adding filter using src_port/src_ip is not supported at this stage */
8155         if (filter->src_port ||
8156             (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8157             !ipv6_addr_any(&filter->ip.v6.src_ip6))
8158                 return -EOPNOTSUPP;
8159
8160         memset(&cld_filter, 0, sizeof(cld_filter));
8161
8162         /* copy element needed to add cloud filter from filter */
8163         i40e_set_cld_element(filter, &cld_filter.element);
8164
8165         if (is_valid_ether_addr(filter->dst_mac) ||
8166             is_valid_ether_addr(filter->src_mac) ||
8167             is_multicast_ether_addr(filter->dst_mac) ||
8168             is_multicast_ether_addr(filter->src_mac)) {
8169                 /* MAC + IP : unsupported mode */
8170                 if (filter->dst_ipv4)
8171                         return -EOPNOTSUPP;
8172
8173                 /* since we validated that L4 port must be valid before
8174                  * we get here, start with respective "flags" value
8175                  * and update if vlan is present or not
8176                  */
8177                 cld_filter.element.flags =
8178                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8179
8180                 if (filter->vlan_id) {
8181                         cld_filter.element.flags =
8182                         cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8183                 }
8184
8185         } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8186                    !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
8187                 cld_filter.element.flags =
8188                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8189                 if (filter->n_proto == ETH_P_IPV6)
8190                         cld_filter.element.flags |=
8191                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8192                 else
8193                         cld_filter.element.flags |=
8194                                 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8195         } else {
8196                 dev_err(&pf->pdev->dev,
8197                         "either mac or ip has to be valid for cloud filter\n");
8198                 return -EINVAL;
8199         }
8200
8201         /* Now copy L4 port in Byte 6..7 in general fields */
8202         cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8203                                                 be16_to_cpu(filter->dst_port);
8204
8205         if (add) {
8206                 /* Validate current device switch mode, change if necessary */
8207                 ret = i40e_validate_and_set_switch_mode(vsi);
8208                 if (ret) {
8209                         dev_err(&pf->pdev->dev,
8210                                 "failed to set switch mode, ret %d\n",
8211                                 ret);
8212                         return ret;
8213                 }
8214
8215                 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
8216                                                    &cld_filter, 1);
8217         } else {
8218                 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
8219                                                    &cld_filter, 1);
8220         }
8221
8222         if (ret)
8223                 dev_dbg(&pf->pdev->dev,
8224                         "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8225                         add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8226         else
8227                 dev_info(&pf->pdev->dev,
8228                          "%s cloud filter for VSI: %d, L4 port: %d\n",
8229                          add ? "add" : "delete", filter->seid,
8230                          ntohs(filter->dst_port));
8231         return ret;
8232 }
8233
8234 /**
8235  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8236  * @vsi: Pointer to VSI
8237  * @f: Pointer to struct flow_cls_offload
8238  * @filter: Pointer to cloud filter structure
8239  *
8240  **/
8241 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8242                                  struct flow_cls_offload *f,
8243                                  struct i40e_cloud_filter *filter)
8244 {
8245         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8246         struct flow_dissector *dissector = rule->match.dissector;
8247         u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8248         struct i40e_pf *pf = vsi->back;
8249         u8 field_flags = 0;
8250
8251         if (dissector->used_keys &
8252             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
8253               BIT(FLOW_DISSECTOR_KEY_BASIC) |
8254               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8255               BIT(FLOW_DISSECTOR_KEY_VLAN) |
8256               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8257               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8258               BIT(FLOW_DISSECTOR_KEY_PORTS) |
8259               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8260                 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
8261                         dissector->used_keys);
8262                 return -EOPNOTSUPP;
8263         }
8264
8265         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8266                 struct flow_match_enc_keyid match;
8267
8268                 flow_rule_match_enc_keyid(rule, &match);
8269                 if (match.mask->keyid != 0)
8270                         field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8271
8272                 filter->tenant_id = be32_to_cpu(match.key->keyid);
8273         }
8274
8275         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8276                 struct flow_match_basic match;
8277
8278                 flow_rule_match_basic(rule, &match);
8279                 n_proto_key = ntohs(match.key->n_proto);
8280                 n_proto_mask = ntohs(match.mask->n_proto);
8281
8282                 if (n_proto_key == ETH_P_ALL) {
8283                         n_proto_key = 0;
8284                         n_proto_mask = 0;
8285                 }
8286                 filter->n_proto = n_proto_key & n_proto_mask;
8287                 filter->ip_proto = match.key->ip_proto;
8288         }
8289
8290         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8291                 struct flow_match_eth_addrs match;
8292
8293                 flow_rule_match_eth_addrs(rule, &match);
8294
8295                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
8296                 if (!is_zero_ether_addr(match.mask->dst)) {
8297                         if (is_broadcast_ether_addr(match.mask->dst)) {
8298                                 field_flags |= I40E_CLOUD_FIELD_OMAC;
8299                         } else {
8300                                 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8301                                         match.mask->dst);
8302                                 return I40E_ERR_CONFIG;
8303                         }
8304                 }
8305
8306                 if (!is_zero_ether_addr(match.mask->src)) {
8307                         if (is_broadcast_ether_addr(match.mask->src)) {
8308                                 field_flags |= I40E_CLOUD_FIELD_IMAC;
8309                         } else {
8310                                 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8311                                         match.mask->src);
8312                                 return I40E_ERR_CONFIG;
8313                         }
8314                 }
8315                 ether_addr_copy(filter->dst_mac, match.key->dst);
8316                 ether_addr_copy(filter->src_mac, match.key->src);
8317         }
8318
8319         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8320                 struct flow_match_vlan match;
8321
8322                 flow_rule_match_vlan(rule, &match);
8323                 if (match.mask->vlan_id) {
8324                         if (match.mask->vlan_id == VLAN_VID_MASK) {
8325                                 field_flags |= I40E_CLOUD_FIELD_IVLAN;
8326
8327                         } else {
8328                                 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8329                                         match.mask->vlan_id);
8330                                 return I40E_ERR_CONFIG;
8331                         }
8332                 }
8333
8334                 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8335         }
8336
8337         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8338                 struct flow_match_control match;
8339
8340                 flow_rule_match_control(rule, &match);
8341                 addr_type = match.key->addr_type;
8342         }
8343
8344         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8345                 struct flow_match_ipv4_addrs match;
8346
8347                 flow_rule_match_ipv4_addrs(rule, &match);
8348                 if (match.mask->dst) {
8349                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8350                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8351                         } else {
8352                                 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8353                                         &match.mask->dst);
8354                                 return I40E_ERR_CONFIG;
8355                         }
8356                 }
8357
8358                 if (match.mask->src) {
8359                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
8360                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8361                         } else {
8362                                 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8363                                         &match.mask->src);
8364                                 return I40E_ERR_CONFIG;
8365                         }
8366                 }
8367
8368                 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8369                         dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8370                         return I40E_ERR_CONFIG;
8371                 }
8372                 filter->dst_ipv4 = match.key->dst;
8373                 filter->src_ipv4 = match.key->src;
8374         }
8375
8376         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8377                 struct flow_match_ipv6_addrs match;
8378
8379                 flow_rule_match_ipv6_addrs(rule, &match);
8380
8381                 /* src and dest IPV6 address should not be LOOPBACK
8382                  * (0:0:0:0:0:0:0:1), which can be represented as ::1
8383                  */
8384                 if (ipv6_addr_loopback(&match.key->dst) ||
8385                     ipv6_addr_loopback(&match.key->src)) {
8386                         dev_err(&pf->pdev->dev,
8387                                 "Bad ipv6, addr is LOOPBACK\n");
8388                         return I40E_ERR_CONFIG;
8389                 }
8390                 if (!ipv6_addr_any(&match.mask->dst) ||
8391                     !ipv6_addr_any(&match.mask->src))
8392                         field_flags |= I40E_CLOUD_FIELD_IIP;
8393
8394                 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8395                        sizeof(filter->src_ipv6));
8396                 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8397                        sizeof(filter->dst_ipv6));
8398         }
8399
8400         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8401                 struct flow_match_ports match;
8402
8403                 flow_rule_match_ports(rule, &match);
8404                 if (match.mask->src) {
8405                         if (match.mask->src == cpu_to_be16(0xffff)) {
8406                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8407                         } else {
8408                                 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8409                                         be16_to_cpu(match.mask->src));
8410                                 return I40E_ERR_CONFIG;
8411                         }
8412                 }
8413
8414                 if (match.mask->dst) {
8415                         if (match.mask->dst == cpu_to_be16(0xffff)) {
8416                                 field_flags |= I40E_CLOUD_FIELD_IIP;
8417                         } else {
8418                                 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8419                                         be16_to_cpu(match.mask->dst));
8420                                 return I40E_ERR_CONFIG;
8421                         }
8422                 }
8423
8424                 filter->dst_port = match.key->dst;
8425                 filter->src_port = match.key->src;
8426
8427                 switch (filter->ip_proto) {
8428                 case IPPROTO_TCP:
8429                 case IPPROTO_UDP:
8430                         break;
8431                 default:
8432                         dev_err(&pf->pdev->dev,
8433                                 "Only UDP and TCP transport are supported\n");
8434                         return -EINVAL;
8435                 }
8436         }
8437         filter->flags = field_flags;
8438         return 0;
8439 }
8440
8441 /**
8442  * i40e_handle_tclass: Forward to a traffic class on the device
8443  * @vsi: Pointer to VSI
8444  * @tc: traffic class index on the device
8445  * @filter: Pointer to cloud filter structure
8446  *
8447  **/
8448 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8449                               struct i40e_cloud_filter *filter)
8450 {
8451         struct i40e_channel *ch, *ch_tmp;
8452
8453         /* direct to a traffic class on the same device */
8454         if (tc == 0) {
8455                 filter->seid = vsi->seid;
8456                 return 0;
8457         } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8458                 if (!filter->dst_port) {
8459                         dev_err(&vsi->back->pdev->dev,
8460                                 "Specify destination port to direct to traffic class that is not default\n");
8461                         return -EINVAL;
8462                 }
8463                 if (list_empty(&vsi->ch_list))
8464                         return -EINVAL;
8465                 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8466                                          list) {
8467                         if (ch->seid == vsi->tc_seid_map[tc])
8468                                 filter->seid = ch->seid;
8469                 }
8470                 return 0;
8471         }
8472         dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8473         return -EINVAL;
8474 }
8475
8476 /**
8477  * i40e_configure_clsflower - Configure tc flower filters
8478  * @vsi: Pointer to VSI
8479  * @cls_flower: Pointer to struct flow_cls_offload
8480  *
8481  **/
8482 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8483                                     struct flow_cls_offload *cls_flower)
8484 {
8485         int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8486         struct i40e_cloud_filter *filter = NULL;
8487         struct i40e_pf *pf = vsi->back;
8488         int err = 0;
8489
8490         if (tc < 0) {
8491                 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8492                 return -EOPNOTSUPP;
8493         }
8494
8495         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8496             test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8497                 return -EBUSY;
8498
8499         if (pf->fdir_pf_active_filters ||
8500             (!hlist_empty(&pf->fdir_filter_list))) {
8501                 dev_err(&vsi->back->pdev->dev,
8502                         "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8503                 return -EINVAL;
8504         }
8505
8506         if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
8507                 dev_err(&vsi->back->pdev->dev,
8508                         "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8509                 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8510                 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8511         }
8512
8513         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8514         if (!filter)
8515                 return -ENOMEM;
8516
8517         filter->cookie = cls_flower->cookie;
8518
8519         err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8520         if (err < 0)
8521                 goto err;
8522
8523         err = i40e_handle_tclass(vsi, tc, filter);
8524         if (err < 0)
8525                 goto err;
8526
8527         /* Add cloud filter */
8528         if (filter->dst_port)
8529                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8530         else
8531                 err = i40e_add_del_cloud_filter(vsi, filter, true);
8532
8533         if (err) {
8534                 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8535                         err);
8536                 goto err;
8537         }
8538
8539         /* add filter to the ordered list */
8540         INIT_HLIST_NODE(&filter->cloud_node);
8541
8542         hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8543
8544         pf->num_cloud_filters++;
8545
8546         return err;
8547 err:
8548         kfree(filter);
8549         return err;
8550 }
8551
8552 /**
8553  * i40e_find_cloud_filter - Find the could filter in the list
8554  * @vsi: Pointer to VSI
8555  * @cookie: filter specific cookie
8556  *
8557  **/
8558 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8559                                                         unsigned long *cookie)
8560 {
8561         struct i40e_cloud_filter *filter = NULL;
8562         struct hlist_node *node2;
8563
8564         hlist_for_each_entry_safe(filter, node2,
8565                                   &vsi->back->cloud_filter_list, cloud_node)
8566                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8567                         return filter;
8568         return NULL;
8569 }
8570
8571 /**
8572  * i40e_delete_clsflower - Remove tc flower filters
8573  * @vsi: Pointer to VSI
8574  * @cls_flower: Pointer to struct flow_cls_offload
8575  *
8576  **/
8577 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8578                                  struct flow_cls_offload *cls_flower)
8579 {
8580         struct i40e_cloud_filter *filter = NULL;
8581         struct i40e_pf *pf = vsi->back;
8582         int err = 0;
8583
8584         filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8585
8586         if (!filter)
8587                 return -EINVAL;
8588
8589         hash_del(&filter->cloud_node);
8590
8591         if (filter->dst_port)
8592                 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8593         else
8594                 err = i40e_add_del_cloud_filter(vsi, filter, false);
8595
8596         kfree(filter);
8597         if (err) {
8598                 dev_err(&pf->pdev->dev,
8599                         "Failed to delete cloud filter, err %s\n",
8600                         i40e_stat_str(&pf->hw, err));
8601                 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8602         }
8603
8604         pf->num_cloud_filters--;
8605         if (!pf->num_cloud_filters)
8606                 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8607                     !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8608                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8609                         pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8610                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8611                 }
8612         return 0;
8613 }
8614
8615 /**
8616  * i40e_setup_tc_cls_flower - flower classifier offloads
8617  * @np: net device to configure
8618  * @cls_flower: offload data
8619  **/
8620 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8621                                     struct flow_cls_offload *cls_flower)
8622 {
8623         struct i40e_vsi *vsi = np->vsi;
8624
8625         switch (cls_flower->command) {
8626         case FLOW_CLS_REPLACE:
8627                 return i40e_configure_clsflower(vsi, cls_flower);
8628         case FLOW_CLS_DESTROY:
8629                 return i40e_delete_clsflower(vsi, cls_flower);
8630         case FLOW_CLS_STATS:
8631                 return -EOPNOTSUPP;
8632         default:
8633                 return -EOPNOTSUPP;
8634         }
8635 }
8636
8637 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8638                                   void *cb_priv)
8639 {
8640         struct i40e_netdev_priv *np = cb_priv;
8641
8642         if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8643                 return -EOPNOTSUPP;
8644
8645         switch (type) {
8646         case TC_SETUP_CLSFLOWER:
8647                 return i40e_setup_tc_cls_flower(np, type_data);
8648
8649         default:
8650                 return -EOPNOTSUPP;
8651         }
8652 }
8653
8654 static LIST_HEAD(i40e_block_cb_list);
8655
8656 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8657                            void *type_data)
8658 {
8659         struct i40e_netdev_priv *np = netdev_priv(netdev);
8660
8661         switch (type) {
8662         case TC_SETUP_QDISC_MQPRIO:
8663                 return i40e_setup_tc(netdev, type_data);
8664         case TC_SETUP_BLOCK:
8665                 return flow_block_cb_setup_simple(type_data,
8666                                                   &i40e_block_cb_list,
8667                                                   i40e_setup_tc_block_cb,
8668                                                   np, np, true);
8669         default:
8670                 return -EOPNOTSUPP;
8671         }
8672 }
8673
8674 /**
8675  * i40e_open - Called when a network interface is made active
8676  * @netdev: network interface device structure
8677  *
8678  * The open entry point is called when a network interface is made
8679  * active by the system (IFF_UP).  At this point all resources needed
8680  * for transmit and receive operations are allocated, the interrupt
8681  * handler is registered with the OS, the netdev watchdog subtask is
8682  * enabled, and the stack is notified that the interface is ready.
8683  *
8684  * Returns 0 on success, negative value on failure
8685  **/
8686 int i40e_open(struct net_device *netdev)
8687 {
8688         struct i40e_netdev_priv *np = netdev_priv(netdev);
8689         struct i40e_vsi *vsi = np->vsi;
8690         struct i40e_pf *pf = vsi->back;
8691         int err;
8692
8693         /* disallow open during test or if eeprom is broken */
8694         if (test_bit(__I40E_TESTING, pf->state) ||
8695             test_bit(__I40E_BAD_EEPROM, pf->state))
8696                 return -EBUSY;
8697
8698         netif_carrier_off(netdev);
8699
8700         if (i40e_force_link_state(pf, true))
8701                 return -EAGAIN;
8702
8703         err = i40e_vsi_open(vsi);
8704         if (err)
8705                 return err;
8706
8707         /* configure global TSO hardware offload settings */
8708         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
8709                                                        TCP_FLAG_FIN) >> 16);
8710         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
8711                                                        TCP_FLAG_FIN |
8712                                                        TCP_FLAG_CWR) >> 16);
8713         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
8714         udp_tunnel_get_rx_info(netdev);
8715
8716         return 0;
8717 }
8718
8719 /**
8720  * i40e_vsi_open -
8721  * @vsi: the VSI to open
8722  *
8723  * Finish initialization of the VSI.
8724  *
8725  * Returns 0 on success, negative value on failure
8726  *
8727  * Note: expects to be called while under rtnl_lock()
8728  **/
8729 int i40e_vsi_open(struct i40e_vsi *vsi)
8730 {
8731         struct i40e_pf *pf = vsi->back;
8732         char int_name[I40E_INT_NAME_STR_LEN];
8733         int err;
8734
8735         /* allocate descriptors */
8736         err = i40e_vsi_setup_tx_resources(vsi);
8737         if (err)
8738                 goto err_setup_tx;
8739         err = i40e_vsi_setup_rx_resources(vsi);
8740         if (err)
8741                 goto err_setup_rx;
8742
8743         err = i40e_vsi_configure(vsi);
8744         if (err)
8745                 goto err_setup_rx;
8746
8747         if (vsi->netdev) {
8748                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
8749                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
8750                 err = i40e_vsi_request_irq(vsi, int_name);
8751                 if (err)
8752                         goto err_setup_rx;
8753
8754                 /* Notify the stack of the actual queue counts. */
8755                 err = netif_set_real_num_tx_queues(vsi->netdev,
8756                                                    vsi->num_queue_pairs);
8757                 if (err)
8758                         goto err_set_queues;
8759
8760                 err = netif_set_real_num_rx_queues(vsi->netdev,
8761                                                    vsi->num_queue_pairs);
8762                 if (err)
8763                         goto err_set_queues;
8764
8765         } else if (vsi->type == I40E_VSI_FDIR) {
8766                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
8767                          dev_driver_string(&pf->pdev->dev),
8768                          dev_name(&pf->pdev->dev));
8769                 err = i40e_vsi_request_irq(vsi, int_name);
8770                 if (err)
8771                         goto err_setup_rx;
8772
8773         } else {
8774                 err = -EINVAL;
8775                 goto err_setup_rx;
8776         }
8777
8778         err = i40e_up_complete(vsi);
8779         if (err)
8780                 goto err_up_complete;
8781
8782         return 0;
8783
8784 err_up_complete:
8785         i40e_down(vsi);
8786 err_set_queues:
8787         i40e_vsi_free_irq(vsi);
8788 err_setup_rx:
8789         i40e_vsi_free_rx_resources(vsi);
8790 err_setup_tx:
8791         i40e_vsi_free_tx_resources(vsi);
8792         if (vsi == pf->vsi[pf->lan_vsi])
8793                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
8794
8795         return err;
8796 }
8797
8798 /**
8799  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
8800  * @pf: Pointer to PF
8801  *
8802  * This function destroys the hlist where all the Flow Director
8803  * filters were saved.
8804  **/
8805 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
8806 {
8807         struct i40e_fdir_filter *filter;
8808         struct i40e_flex_pit *pit_entry, *tmp;
8809         struct hlist_node *node2;
8810
8811         hlist_for_each_entry_safe(filter, node2,
8812                                   &pf->fdir_filter_list, fdir_node) {
8813                 hlist_del(&filter->fdir_node);
8814                 kfree(filter);
8815         }
8816
8817         list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
8818                 list_del(&pit_entry->list);
8819                 kfree(pit_entry);
8820         }
8821         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
8822
8823         list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
8824                 list_del(&pit_entry->list);
8825                 kfree(pit_entry);
8826         }
8827         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
8828
8829         pf->fdir_pf_active_filters = 0;
8830         i40e_reset_fdir_filter_cnt(pf);
8831
8832         /* Reprogram the default input set for TCP/IPv4 */
8833         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8834                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8835                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8836
8837         /* Reprogram the default input set for TCP/IPv6 */
8838         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
8839                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
8840                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8841
8842         /* Reprogram the default input set for UDP/IPv4 */
8843         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
8844                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8845                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8846
8847         /* Reprogram the default input set for UDP/IPv6 */
8848         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
8849                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
8850                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8851
8852         /* Reprogram the default input set for SCTP/IPv4 */
8853         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
8854                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8855                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8856
8857         /* Reprogram the default input set for SCTP/IPv6 */
8858         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
8859                                 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
8860                                 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8861
8862         /* Reprogram the default input set for Other/IPv4 */
8863         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
8864                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8865
8866         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
8867                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8868
8869         /* Reprogram the default input set for Other/IPv6 */
8870         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
8871                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8872
8873         i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,
8874                                 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8875 }
8876
8877 /**
8878  * i40e_cloud_filter_exit - Cleans up the cloud filters
8879  * @pf: Pointer to PF
8880  *
8881  * This function destroys the hlist where all the cloud filters
8882  * were saved.
8883  **/
8884 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
8885 {
8886         struct i40e_cloud_filter *cfilter;
8887         struct hlist_node *node;
8888
8889         hlist_for_each_entry_safe(cfilter, node,
8890                                   &pf->cloud_filter_list, cloud_node) {
8891                 hlist_del(&cfilter->cloud_node);
8892                 kfree(cfilter);
8893         }
8894         pf->num_cloud_filters = 0;
8895
8896         if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8897             !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8898                 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8899                 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8900                 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8901         }
8902 }
8903
8904 /**
8905  * i40e_close - Disables a network interface
8906  * @netdev: network interface device structure
8907  *
8908  * The close entry point is called when an interface is de-activated
8909  * by the OS.  The hardware is still under the driver's control, but
8910  * this netdev interface is disabled.
8911  *
8912  * Returns 0, this is not allowed to fail
8913  **/
8914 int i40e_close(struct net_device *netdev)
8915 {
8916         struct i40e_netdev_priv *np = netdev_priv(netdev);
8917         struct i40e_vsi *vsi = np->vsi;
8918
8919         i40e_vsi_close(vsi);
8920
8921         return 0;
8922 }
8923
8924 /**
8925  * i40e_do_reset - Start a PF or Core Reset sequence
8926  * @pf: board private structure
8927  * @reset_flags: which reset is requested
8928  * @lock_acquired: indicates whether or not the lock has been acquired
8929  * before this function was called.
8930  *
8931  * The essential difference in resets is that the PF Reset
8932  * doesn't clear the packet buffers, doesn't reset the PE
8933  * firmware, and doesn't bother the other PFs on the chip.
8934  **/
8935 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
8936 {
8937         u32 val;
8938
8939         /* do the biggest reset indicated */
8940         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
8941
8942                 /* Request a Global Reset
8943                  *
8944                  * This will start the chip's countdown to the actual full
8945                  * chip reset event, and a warning interrupt to be sent
8946                  * to all PFs, including the requestor.  Our handler
8947                  * for the warning interrupt will deal with the shutdown
8948                  * and recovery of the switch setup.
8949                  */
8950                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
8951                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8952                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
8953                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8954
8955         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
8956
8957                 /* Request a Core Reset
8958                  *
8959                  * Same as Global Reset, except does *not* include the MAC/PHY
8960                  */
8961                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
8962                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8963                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
8964                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8965                 i40e_flush(&pf->hw);
8966
8967         } else if (reset_flags & I40E_PF_RESET_FLAG) {
8968
8969                 /* Request a PF Reset
8970                  *
8971                  * Resets only the PF-specific registers
8972                  *
8973                  * This goes directly to the tear-down and rebuild of
8974                  * the switch, since we need to do all the recovery as
8975                  * for the Core Reset.
8976                  */
8977                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
8978                 i40e_handle_reset_warning(pf, lock_acquired);
8979
8980         } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
8981                 /* Request a PF Reset
8982                  *
8983                  * Resets PF and reinitializes PFs VSI.
8984                  */
8985                 i40e_prep_for_reset(pf);
8986                 i40e_reset_and_rebuild(pf, true, lock_acquired);
8987                 dev_info(&pf->pdev->dev,
8988                          pf->flags & I40E_FLAG_DISABLE_FW_LLDP ?
8989                          "FW LLDP is disabled\n" :
8990                          "FW LLDP is enabled\n");
8991
8992         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
8993                 int v;
8994
8995                 /* Find the VSI(s) that requested a re-init */
8996                 dev_info(&pf->pdev->dev,
8997                          "VSI reinit requested\n");
8998                 for (v = 0; v < pf->num_alloc_vsi; v++) {
8999                         struct i40e_vsi *vsi = pf->vsi[v];
9000
9001                         if (vsi != NULL &&
9002                             test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
9003                                                vsi->state))
9004                                 i40e_vsi_reinit_locked(pf->vsi[v]);
9005                 }
9006         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
9007                 int v;
9008
9009                 /* Find the VSI(s) that needs to be brought down */
9010                 dev_info(&pf->pdev->dev, "VSI down requested\n");
9011                 for (v = 0; v < pf->num_alloc_vsi; v++) {
9012                         struct i40e_vsi *vsi = pf->vsi[v];
9013
9014                         if (vsi != NULL &&
9015                             test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
9016                                                vsi->state)) {
9017                                 set_bit(__I40E_VSI_DOWN, vsi->state);
9018                                 i40e_down(vsi);
9019                         }
9020                 }
9021         } else {
9022                 dev_info(&pf->pdev->dev,
9023                          "bad reset request 0x%08x\n", reset_flags);
9024         }
9025 }
9026
9027 #ifdef CONFIG_I40E_DCB
9028 /**
9029  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
9030  * @pf: board private structure
9031  * @old_cfg: current DCB config
9032  * @new_cfg: new DCB config
9033  **/
9034 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
9035                             struct i40e_dcbx_config *old_cfg,
9036                             struct i40e_dcbx_config *new_cfg)
9037 {
9038         bool need_reconfig = false;
9039
9040         /* Check if ETS configuration has changed */
9041         if (memcmp(&new_cfg->etscfg,
9042                    &old_cfg->etscfg,
9043                    sizeof(new_cfg->etscfg))) {
9044                 /* If Priority Table has changed reconfig is needed */
9045                 if (memcmp(&new_cfg->etscfg.prioritytable,
9046                            &old_cfg->etscfg.prioritytable,
9047                            sizeof(new_cfg->etscfg.prioritytable))) {
9048                         need_reconfig = true;
9049                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9050                 }
9051
9052                 if (memcmp(&new_cfg->etscfg.tcbwtable,
9053                            &old_cfg->etscfg.tcbwtable,
9054                            sizeof(new_cfg->etscfg.tcbwtable)))
9055                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9056
9057                 if (memcmp(&new_cfg->etscfg.tsatable,
9058                            &old_cfg->etscfg.tsatable,
9059                            sizeof(new_cfg->etscfg.tsatable)))
9060                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9061         }
9062
9063         /* Check if PFC configuration has changed */
9064         if (memcmp(&new_cfg->pfc,
9065                    &old_cfg->pfc,
9066                    sizeof(new_cfg->pfc))) {
9067                 need_reconfig = true;
9068                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9069         }
9070
9071         /* Check if APP Table has changed */
9072         if (memcmp(&new_cfg->app,
9073                    &old_cfg->app,
9074                    sizeof(new_cfg->app))) {
9075                 need_reconfig = true;
9076                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9077         }
9078
9079         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9080         return need_reconfig;
9081 }
9082
9083 /**
9084  * i40e_handle_lldp_event - Handle LLDP Change MIB event
9085  * @pf: board private structure
9086  * @e: event info posted on ARQ
9087  **/
9088 static int i40e_handle_lldp_event(struct i40e_pf *pf,
9089                                   struct i40e_arq_event_info *e)
9090 {
9091         struct i40e_aqc_lldp_get_mib *mib =
9092                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9093         struct i40e_hw *hw = &pf->hw;
9094         struct i40e_dcbx_config tmp_dcbx_cfg;
9095         bool need_reconfig = false;
9096         int ret = 0;
9097         u8 type;
9098
9099         /* X710-T*L 2.5G and 5G speeds don't support DCB */
9100         if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9101             (hw->phy.link_info.link_speed &
9102              ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9103              !(pf->flags & I40E_FLAG_DCB_CAPABLE))
9104                 /* let firmware decide if the DCB should be disabled */
9105                 pf->flags |= I40E_FLAG_DCB_CAPABLE;
9106
9107         /* Not DCB capable or capability disabled */
9108         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
9109                 return ret;
9110
9111         /* Ignore if event is not for Nearest Bridge */
9112         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9113                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9114         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9115         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9116                 return ret;
9117
9118         /* Check MIB Type and return if event for Remote MIB update */
9119         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9120         dev_dbg(&pf->pdev->dev,
9121                 "LLDP event mib type %s\n", type ? "remote" : "local");
9122         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9123                 /* Update the remote cached instance and return */
9124                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9125                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9126                                 &hw->remote_dcbx_config);
9127                 goto exit;
9128         }
9129
9130         /* Store the old configuration */
9131         tmp_dcbx_cfg = hw->local_dcbx_config;
9132
9133         /* Reset the old DCBx configuration data */
9134         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9135         /* Get updated DCBX data from firmware */
9136         ret = i40e_get_dcb_config(&pf->hw);
9137         if (ret) {
9138                 /* X710-T*L 2.5G and 5G speeds don't support DCB */
9139                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9140                     (hw->phy.link_info.link_speed &
9141                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9142                         dev_warn(&pf->pdev->dev,
9143                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
9144                         pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
9145                 } else {
9146                         dev_info(&pf->pdev->dev,
9147                                  "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
9148                                  i40e_stat_str(&pf->hw, ret),
9149                                  i40e_aq_str(&pf->hw,
9150                                              pf->hw.aq.asq_last_status));
9151                 }
9152                 goto exit;
9153         }
9154
9155         /* No change detected in DCBX configs */
9156         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
9157                     sizeof(tmp_dcbx_cfg))) {
9158                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9159                 goto exit;
9160         }
9161
9162         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
9163                                                &hw->local_dcbx_config);
9164
9165         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
9166
9167         if (!need_reconfig)
9168                 goto exit;
9169
9170         /* Enable DCB tagging only when more than one TC */
9171         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
9172                 pf->flags |= I40E_FLAG_DCB_ENABLED;
9173         else
9174                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9175
9176         set_bit(__I40E_PORT_SUSPENDED, pf->state);
9177         /* Reconfiguration needed quiesce all VSIs */
9178         i40e_pf_quiesce_all_vsi(pf);
9179
9180         /* Changes in configuration update VEB/VSI */
9181         i40e_dcb_reconfigure(pf);
9182
9183         ret = i40e_resume_port_tx(pf);
9184
9185         clear_bit(__I40E_PORT_SUSPENDED, pf->state);
9186         /* In case of error no point in resuming VSIs */
9187         if (ret)
9188                 goto exit;
9189
9190         /* Wait for the PF's queues to be disabled */
9191         ret = i40e_pf_wait_queues_disabled(pf);
9192         if (ret) {
9193                 /* Schedule PF reset to recover */
9194                 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9195                 i40e_service_event_schedule(pf);
9196         } else {
9197                 i40e_pf_unquiesce_all_vsi(pf);
9198                 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
9199                 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
9200         }
9201
9202 exit:
9203         return ret;
9204 }
9205 #endif /* CONFIG_I40E_DCB */
9206
9207 /**
9208  * i40e_do_reset_safe - Protected reset path for userland calls.
9209  * @pf: board private structure
9210  * @reset_flags: which reset is requested
9211  *
9212  **/
9213 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9214 {
9215         rtnl_lock();
9216         i40e_do_reset(pf, reset_flags, true);
9217         rtnl_unlock();
9218 }
9219
9220 /**
9221  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9222  * @pf: board private structure
9223  * @e: event info posted on ARQ
9224  *
9225  * Handler for LAN Queue Overflow Event generated by the firmware for PF
9226  * and VF queues
9227  **/
9228 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9229                                            struct i40e_arq_event_info *e)
9230 {
9231         struct i40e_aqc_lan_overflow *data =
9232                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9233         u32 queue = le32_to_cpu(data->prtdcb_rupto);
9234         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9235         struct i40e_hw *hw = &pf->hw;
9236         struct i40e_vf *vf;
9237         u16 vf_id;
9238
9239         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9240                 queue, qtx_ctl);
9241
9242         /* Queue belongs to VF, find the VF and issue VF reset */
9243         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
9244             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
9245                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
9246                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
9247                 vf_id -= hw->func_caps.vf_base_id;
9248                 vf = &pf->vf[vf_id];
9249                 i40e_vc_notify_vf_reset(vf);
9250                 /* Allow VF to process pending reset notification */
9251                 msleep(20);
9252                 i40e_reset_vf(vf, false);
9253         }
9254 }
9255
9256 /**
9257  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9258  * @pf: board private structure
9259  **/
9260 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9261 {
9262         u32 val, fcnt_prog;
9263
9264         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9265         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9266         return fcnt_prog;
9267 }
9268
9269 /**
9270  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9271  * @pf: board private structure
9272  **/
9273 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9274 {
9275         u32 val, fcnt_prog;
9276
9277         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9278         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9279                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
9280                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
9281         return fcnt_prog;
9282 }
9283
9284 /**
9285  * i40e_get_global_fd_count - Get total FD filters programmed on device
9286  * @pf: board private structure
9287  **/
9288 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9289 {
9290         u32 val, fcnt_prog;
9291
9292         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9293         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9294                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
9295                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
9296         return fcnt_prog;
9297 }
9298
9299 /**
9300  * i40e_reenable_fdir_sb - Restore FDir SB capability
9301  * @pf: board private structure
9302  **/
9303 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9304 {
9305         if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9306                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
9307                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9308                         dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9309 }
9310
9311 /**
9312  * i40e_reenable_fdir_atr - Restore FDir ATR capability
9313  * @pf: board private structure
9314  **/
9315 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9316 {
9317         if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9318                 /* ATR uses the same filtering logic as SB rules. It only
9319                  * functions properly if the input set mask is at the default
9320                  * settings. It is safe to restore the default input set
9321                  * because there are no active TCPv4 filter rules.
9322                  */
9323                 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9324                                         I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9325                                         I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9326
9327                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9328                     (I40E_DEBUG_FD & pf->hw.debug_mask))
9329                         dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9330         }
9331 }
9332
9333 /**
9334  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9335  * @pf: board private structure
9336  * @filter: FDir filter to remove
9337  */
9338 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9339                                        struct i40e_fdir_filter *filter)
9340 {
9341         /* Update counters */
9342         pf->fdir_pf_active_filters--;
9343         pf->fd_inv = 0;
9344
9345         switch (filter->flow_type) {
9346         case TCP_V4_FLOW:
9347                 pf->fd_tcp4_filter_cnt--;
9348                 break;
9349         case UDP_V4_FLOW:
9350                 pf->fd_udp4_filter_cnt--;
9351                 break;
9352         case SCTP_V4_FLOW:
9353                 pf->fd_sctp4_filter_cnt--;
9354                 break;
9355         case TCP_V6_FLOW:
9356                 pf->fd_tcp6_filter_cnt--;
9357                 break;
9358         case UDP_V6_FLOW:
9359                 pf->fd_udp6_filter_cnt--;
9360                 break;
9361         case SCTP_V6_FLOW:
9362                 pf->fd_udp6_filter_cnt--;
9363                 break;
9364         case IP_USER_FLOW:
9365                 switch (filter->ipl4_proto) {
9366                 case IPPROTO_TCP:
9367                         pf->fd_tcp4_filter_cnt--;
9368                         break;
9369                 case IPPROTO_UDP:
9370                         pf->fd_udp4_filter_cnt--;
9371                         break;
9372                 case IPPROTO_SCTP:
9373                         pf->fd_sctp4_filter_cnt--;
9374                         break;
9375                 case IPPROTO_IP:
9376                         pf->fd_ip4_filter_cnt--;
9377                         break;
9378                 }
9379                 break;
9380         case IPV6_USER_FLOW:
9381                 switch (filter->ipl4_proto) {
9382                 case IPPROTO_TCP:
9383                         pf->fd_tcp6_filter_cnt--;
9384                         break;
9385                 case IPPROTO_UDP:
9386                         pf->fd_udp6_filter_cnt--;
9387                         break;
9388                 case IPPROTO_SCTP:
9389                         pf->fd_sctp6_filter_cnt--;
9390                         break;
9391                 case IPPROTO_IP:
9392                         pf->fd_ip6_filter_cnt--;
9393                         break;
9394                 }
9395                 break;
9396         }
9397
9398         /* Remove the filter from the list and free memory */
9399         hlist_del(&filter->fdir_node);
9400         kfree(filter);
9401 }
9402
9403 /**
9404  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9405  * @pf: board private structure
9406  **/
9407 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9408 {
9409         struct i40e_fdir_filter *filter;
9410         u32 fcnt_prog, fcnt_avail;
9411         struct hlist_node *node;
9412
9413         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9414                 return;
9415
9416         /* Check if we have enough room to re-enable FDir SB capability. */
9417         fcnt_prog = i40e_get_global_fd_count(pf);
9418         fcnt_avail = pf->fdir_pf_filter_count;
9419         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9420             (pf->fd_add_err == 0) ||
9421             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9422                 i40e_reenable_fdir_sb(pf);
9423
9424         /* We should wait for even more space before re-enabling ATR.
9425          * Additionally, we cannot enable ATR as long as we still have TCP SB
9426          * rules active.
9427          */
9428         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9429             pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9430                 i40e_reenable_fdir_atr(pf);
9431
9432         /* if hw had a problem adding a filter, delete it */
9433         if (pf->fd_inv > 0) {
9434                 hlist_for_each_entry_safe(filter, node,
9435                                           &pf->fdir_filter_list, fdir_node)
9436                         if (filter->fd_id == pf->fd_inv)
9437                                 i40e_delete_invalid_filter(pf, filter);
9438         }
9439 }
9440
9441 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9442 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9443 /**
9444  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9445  * @pf: board private structure
9446  **/
9447 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9448 {
9449         unsigned long min_flush_time;
9450         int flush_wait_retry = 50;
9451         bool disable_atr = false;
9452         int fd_room;
9453         int reg;
9454
9455         if (!time_after(jiffies, pf->fd_flush_timestamp +
9456                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9457                 return;
9458
9459         /* If the flush is happening too quick and we have mostly SB rules we
9460          * should not re-enable ATR for some time.
9461          */
9462         min_flush_time = pf->fd_flush_timestamp +
9463                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9464         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9465
9466         if (!(time_after(jiffies, min_flush_time)) &&
9467             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9468                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9469                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9470                 disable_atr = true;
9471         }
9472
9473         pf->fd_flush_timestamp = jiffies;
9474         set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9475         /* flush all filters */
9476         wr32(&pf->hw, I40E_PFQF_CTL_1,
9477              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9478         i40e_flush(&pf->hw);
9479         pf->fd_flush_cnt++;
9480         pf->fd_add_err = 0;
9481         do {
9482                 /* Check FD flush status every 5-6msec */
9483                 usleep_range(5000, 6000);
9484                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9485                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9486                         break;
9487         } while (flush_wait_retry--);
9488         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9489                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9490         } else {
9491                 /* replay sideband filters */
9492                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
9493                 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9494                         clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9495                 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9496                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
9497                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9498         }
9499 }
9500
9501 /**
9502  * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9503  * @pf: board private structure
9504  **/
9505 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9506 {
9507         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9508 }
9509
9510 /**
9511  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9512  * @pf: board private structure
9513  **/
9514 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9515 {
9516
9517         /* if interface is down do nothing */
9518         if (test_bit(__I40E_DOWN, pf->state))
9519                 return;
9520
9521         if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9522                 i40e_fdir_flush_and_replay(pf);
9523
9524         i40e_fdir_check_and_reenable(pf);
9525
9526 }
9527
9528 /**
9529  * i40e_vsi_link_event - notify VSI of a link event
9530  * @vsi: vsi to be notified
9531  * @link_up: link up or down
9532  **/
9533 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9534 {
9535         if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9536                 return;
9537
9538         switch (vsi->type) {
9539         case I40E_VSI_MAIN:
9540                 if (!vsi->netdev || !vsi->netdev_registered)
9541                         break;
9542
9543                 if (link_up) {
9544                         netif_carrier_on(vsi->netdev);
9545                         netif_tx_wake_all_queues(vsi->netdev);
9546                 } else {
9547                         netif_carrier_off(vsi->netdev);
9548                         netif_tx_stop_all_queues(vsi->netdev);
9549                 }
9550                 break;
9551
9552         case I40E_VSI_SRIOV:
9553         case I40E_VSI_VMDQ2:
9554         case I40E_VSI_CTRL:
9555         case I40E_VSI_IWARP:
9556         case I40E_VSI_MIRROR:
9557         default:
9558                 /* there is no notification for other VSIs */
9559                 break;
9560         }
9561 }
9562
9563 /**
9564  * i40e_veb_link_event - notify elements on the veb of a link event
9565  * @veb: veb to be notified
9566  * @link_up: link up or down
9567  **/
9568 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9569 {
9570         struct i40e_pf *pf;
9571         int i;
9572
9573         if (!veb || !veb->pf)
9574                 return;
9575         pf = veb->pf;
9576
9577         /* depth first... */
9578         for (i = 0; i < I40E_MAX_VEB; i++)
9579                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
9580                         i40e_veb_link_event(pf->veb[i], link_up);
9581
9582         /* ... now the local VSIs */
9583         for (i = 0; i < pf->num_alloc_vsi; i++)
9584                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
9585                         i40e_vsi_link_event(pf->vsi[i], link_up);
9586 }
9587
9588 /**
9589  * i40e_link_event - Update netif_carrier status
9590  * @pf: board private structure
9591  **/
9592 static void i40e_link_event(struct i40e_pf *pf)
9593 {
9594         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9595         u8 new_link_speed, old_link_speed;
9596         i40e_status status;
9597         bool new_link, old_link;
9598 #ifdef CONFIG_I40E_DCB
9599         int err;
9600 #endif /* CONFIG_I40E_DCB */
9601
9602         /* set this to force the get_link_status call to refresh state */
9603         pf->hw.phy.get_link_info = true;
9604         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9605         status = i40e_get_link_status(&pf->hw, &new_link);
9606
9607         /* On success, disable temp link polling */
9608         if (status == I40E_SUCCESS) {
9609                 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9610         } else {
9611                 /* Enable link polling temporarily until i40e_get_link_status
9612                  * returns I40E_SUCCESS
9613                  */
9614                 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9615                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9616                         status);
9617                 return;
9618         }
9619
9620         old_link_speed = pf->hw.phy.link_info_old.link_speed;
9621         new_link_speed = pf->hw.phy.link_info.link_speed;
9622
9623         if (new_link == old_link &&
9624             new_link_speed == old_link_speed &&
9625             (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9626              new_link == netif_carrier_ok(vsi->netdev)))
9627                 return;
9628
9629         i40e_print_link_message(vsi, new_link);
9630
9631         /* Notify the base of the switch tree connected to
9632          * the link.  Floating VEBs are not notified.
9633          */
9634         if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9635                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
9636         else
9637                 i40e_vsi_link_event(vsi, new_link);
9638
9639         if (pf->vf)
9640                 i40e_vc_notify_link_state(pf);
9641
9642         if (pf->flags & I40E_FLAG_PTP)
9643                 i40e_ptp_set_increment(pf);
9644 #ifdef CONFIG_I40E_DCB
9645         if (new_link == old_link)
9646                 return;
9647         /* Not SW DCB so firmware will take care of default settings */
9648         if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
9649                 return;
9650
9651         /* We cover here only link down, as after link up in case of SW DCB
9652          * SW LLDP agent will take care of setting it up
9653          */
9654         if (!new_link) {
9655                 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
9656                 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
9657                 err = i40e_dcb_sw_default_config(pf);
9658                 if (err) {
9659                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
9660                                        I40E_FLAG_DCB_ENABLED);
9661                 } else {
9662                         pf->dcbx_cap = DCB_CAP_DCBX_HOST |
9663                                        DCB_CAP_DCBX_VER_IEEE;
9664                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
9665                         pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9666                 }
9667         }
9668 #endif /* CONFIG_I40E_DCB */
9669 }
9670
9671 /**
9672  * i40e_watchdog_subtask - periodic checks not using event driven response
9673  * @pf: board private structure
9674  **/
9675 static void i40e_watchdog_subtask(struct i40e_pf *pf)
9676 {
9677         int i;
9678
9679         /* if interface is down do nothing */
9680         if (test_bit(__I40E_DOWN, pf->state) ||
9681             test_bit(__I40E_CONFIG_BUSY, pf->state))
9682                 return;
9683
9684         /* make sure we don't do these things too often */
9685         if (time_before(jiffies, (pf->service_timer_previous +
9686                                   pf->service_timer_period)))
9687                 return;
9688         pf->service_timer_previous = jiffies;
9689
9690         if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
9691             test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
9692                 i40e_link_event(pf);
9693
9694         /* Update the stats for active netdevs so the network stack
9695          * can look at updated numbers whenever it cares to
9696          */
9697         for (i = 0; i < pf->num_alloc_vsi; i++)
9698                 if (pf->vsi[i] && pf->vsi[i]->netdev)
9699                         i40e_update_stats(pf->vsi[i]);
9700
9701         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
9702                 /* Update the stats for the active switching components */
9703                 for (i = 0; i < I40E_MAX_VEB; i++)
9704                         if (pf->veb[i])
9705                                 i40e_update_veb_stats(pf->veb[i]);
9706         }
9707
9708         i40e_ptp_rx_hang(pf);
9709         i40e_ptp_tx_hang(pf);
9710 }
9711
9712 /**
9713  * i40e_reset_subtask - Set up for resetting the device and driver
9714  * @pf: board private structure
9715  **/
9716 static void i40e_reset_subtask(struct i40e_pf *pf)
9717 {
9718         u32 reset_flags = 0;
9719
9720         if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
9721                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
9722                 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
9723         }
9724         if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
9725                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
9726                 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9727         }
9728         if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
9729                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
9730                 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
9731         }
9732         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
9733                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
9734                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
9735         }
9736         if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
9737                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
9738                 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
9739         }
9740
9741         /* If there's a recovery already waiting, it takes
9742          * precedence before starting a new reset sequence.
9743          */
9744         if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
9745                 i40e_prep_for_reset(pf);
9746                 i40e_reset(pf);
9747                 i40e_rebuild(pf, false, false);
9748         }
9749
9750         /* If we're already down or resetting, just bail */
9751         if (reset_flags &&
9752             !test_bit(__I40E_DOWN, pf->state) &&
9753             !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
9754                 i40e_do_reset(pf, reset_flags, false);
9755         }
9756 }
9757
9758 /**
9759  * i40e_handle_link_event - Handle link event
9760  * @pf: board private structure
9761  * @e: event info posted on ARQ
9762  **/
9763 static void i40e_handle_link_event(struct i40e_pf *pf,
9764                                    struct i40e_arq_event_info *e)
9765 {
9766         struct i40e_aqc_get_link_status *status =
9767                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
9768
9769         /* Do a new status request to re-enable LSE reporting
9770          * and load new status information into the hw struct
9771          * This completely ignores any state information
9772          * in the ARQ event info, instead choosing to always
9773          * issue the AQ update link status command.
9774          */
9775         i40e_link_event(pf);
9776
9777         /* Check if module meets thermal requirements */
9778         if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
9779                 dev_err(&pf->pdev->dev,
9780                         "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
9781                 dev_err(&pf->pdev->dev,
9782                         "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9783         } else {
9784                 /* check for unqualified module, if link is down, suppress
9785                  * the message if link was forced to be down.
9786                  */
9787                 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
9788                     (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
9789                     (!(status->link_info & I40E_AQ_LINK_UP)) &&
9790                     (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
9791                         dev_err(&pf->pdev->dev,
9792                                 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
9793                         dev_err(&pf->pdev->dev,
9794                                 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9795                 }
9796         }
9797 }
9798
9799 /**
9800  * i40e_clean_adminq_subtask - Clean the AdminQ rings
9801  * @pf: board private structure
9802  **/
9803 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
9804 {
9805         struct i40e_arq_event_info event;
9806         struct i40e_hw *hw = &pf->hw;
9807         u16 pending, i = 0;
9808         i40e_status ret;
9809         u16 opcode;
9810         u32 oldval;
9811         u32 val;
9812
9813         /* Do not run clean AQ when PF reset fails */
9814         if (test_bit(__I40E_RESET_FAILED, pf->state))
9815                 return;
9816
9817         /* check for error indications */
9818         val = rd32(&pf->hw, pf->hw.aq.arq.len);
9819         oldval = val;
9820         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
9821                 if (hw->debug_mask & I40E_DEBUG_AQ)
9822                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
9823                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
9824         }
9825         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
9826                 if (hw->debug_mask & I40E_DEBUG_AQ)
9827                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
9828                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
9829                 pf->arq_overflows++;
9830         }
9831         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
9832                 if (hw->debug_mask & I40E_DEBUG_AQ)
9833                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
9834                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
9835         }
9836         if (oldval != val)
9837                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
9838
9839         val = rd32(&pf->hw, pf->hw.aq.asq.len);
9840         oldval = val;
9841         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
9842                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9843                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
9844                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
9845         }
9846         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
9847                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9848                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
9849                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
9850         }
9851         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
9852                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9853                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
9854                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
9855         }
9856         if (oldval != val)
9857                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
9858
9859         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
9860         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
9861         if (!event.msg_buf)
9862                 return;
9863
9864         do {
9865                 ret = i40e_clean_arq_element(hw, &event, &pending);
9866                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
9867                         break;
9868                 else if (ret) {
9869                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
9870                         break;
9871                 }
9872
9873                 opcode = le16_to_cpu(event.desc.opcode);
9874                 switch (opcode) {
9875
9876                 case i40e_aqc_opc_get_link_status:
9877                         rtnl_lock();
9878                         i40e_handle_link_event(pf, &event);
9879                         rtnl_unlock();
9880                         break;
9881                 case i40e_aqc_opc_send_msg_to_pf:
9882                         ret = i40e_vc_process_vf_msg(pf,
9883                                         le16_to_cpu(event.desc.retval),
9884                                         le32_to_cpu(event.desc.cookie_high),
9885                                         le32_to_cpu(event.desc.cookie_low),
9886                                         event.msg_buf,
9887                                         event.msg_len);
9888                         break;
9889                 case i40e_aqc_opc_lldp_update_mib:
9890                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
9891 #ifdef CONFIG_I40E_DCB
9892                         rtnl_lock();
9893                         i40e_handle_lldp_event(pf, &event);
9894                         rtnl_unlock();
9895 #endif /* CONFIG_I40E_DCB */
9896                         break;
9897                 case i40e_aqc_opc_event_lan_overflow:
9898                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
9899                         i40e_handle_lan_overflow_event(pf, &event);
9900                         break;
9901                 case i40e_aqc_opc_send_msg_to_peer:
9902                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
9903                         break;
9904                 case i40e_aqc_opc_nvm_erase:
9905                 case i40e_aqc_opc_nvm_update:
9906                 case i40e_aqc_opc_oem_post_update:
9907                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
9908                                    "ARQ NVM operation 0x%04x completed\n",
9909                                    opcode);
9910                         break;
9911                 default:
9912                         dev_info(&pf->pdev->dev,
9913                                  "ARQ: Unknown event 0x%04x ignored\n",
9914                                  opcode);
9915                         break;
9916                 }
9917         } while (i++ < pf->adminq_work_limit);
9918
9919         if (i < pf->adminq_work_limit)
9920                 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
9921
9922         /* re-enable Admin queue interrupt cause */
9923         val = rd32(hw, I40E_PFINT_ICR0_ENA);
9924         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
9925         wr32(hw, I40E_PFINT_ICR0_ENA, val);
9926         i40e_flush(hw);
9927
9928         kfree(event.msg_buf);
9929 }
9930
9931 /**
9932  * i40e_verify_eeprom - make sure eeprom is good to use
9933  * @pf: board private structure
9934  **/
9935 static void i40e_verify_eeprom(struct i40e_pf *pf)
9936 {
9937         int err;
9938
9939         err = i40e_diag_eeprom_test(&pf->hw);
9940         if (err) {
9941                 /* retry in case of garbage read */
9942                 err = i40e_diag_eeprom_test(&pf->hw);
9943                 if (err) {
9944                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
9945                                  err);
9946                         set_bit(__I40E_BAD_EEPROM, pf->state);
9947                 }
9948         }
9949
9950         if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
9951                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
9952                 clear_bit(__I40E_BAD_EEPROM, pf->state);
9953         }
9954 }
9955
9956 /**
9957  * i40e_enable_pf_switch_lb
9958  * @pf: pointer to the PF structure
9959  *
9960  * enable switch loop back or die - no point in a return value
9961  **/
9962 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
9963 {
9964         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9965         struct i40e_vsi_context ctxt;
9966         int ret;
9967
9968         ctxt.seid = pf->main_vsi_seid;
9969         ctxt.pf_num = pf->hw.pf_id;
9970         ctxt.vf_num = 0;
9971         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9972         if (ret) {
9973                 dev_info(&pf->pdev->dev,
9974                          "couldn't get PF vsi config, err %s aq_err %s\n",
9975                          i40e_stat_str(&pf->hw, ret),
9976                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9977                 return;
9978         }
9979         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9980         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9981         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9982
9983         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9984         if (ret) {
9985                 dev_info(&pf->pdev->dev,
9986                          "update vsi switch failed, err %s aq_err %s\n",
9987                          i40e_stat_str(&pf->hw, ret),
9988                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9989         }
9990 }
9991
9992 /**
9993  * i40e_disable_pf_switch_lb
9994  * @pf: pointer to the PF structure
9995  *
9996  * disable switch loop back or die - no point in a return value
9997  **/
9998 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
9999 {
10000         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10001         struct i40e_vsi_context ctxt;
10002         int ret;
10003
10004         ctxt.seid = pf->main_vsi_seid;
10005         ctxt.pf_num = pf->hw.pf_id;
10006         ctxt.vf_num = 0;
10007         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10008         if (ret) {
10009                 dev_info(&pf->pdev->dev,
10010                          "couldn't get PF vsi config, err %s aq_err %s\n",
10011                          i40e_stat_str(&pf->hw, ret),
10012                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10013                 return;
10014         }
10015         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10016         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10017         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10018
10019         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10020         if (ret) {
10021                 dev_info(&pf->pdev->dev,
10022                          "update vsi switch failed, err %s aq_err %s\n",
10023                          i40e_stat_str(&pf->hw, ret),
10024                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10025         }
10026 }
10027
10028 /**
10029  * i40e_config_bridge_mode - Configure the HW bridge mode
10030  * @veb: pointer to the bridge instance
10031  *
10032  * Configure the loop back mode for the LAN VSI that is downlink to the
10033  * specified HW bridge instance. It is expected this function is called
10034  * when a new HW bridge is instantiated.
10035  **/
10036 static void i40e_config_bridge_mode(struct i40e_veb *veb)
10037 {
10038         struct i40e_pf *pf = veb->pf;
10039
10040         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
10041                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
10042                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10043         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
10044                 i40e_disable_pf_switch_lb(pf);
10045         else
10046                 i40e_enable_pf_switch_lb(pf);
10047 }
10048
10049 /**
10050  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
10051  * @veb: pointer to the VEB instance
10052  *
10053  * This is a recursive function that first builds the attached VSIs then
10054  * recurses in to build the next layer of VEB.  We track the connections
10055  * through our own index numbers because the seid's from the HW could
10056  * change across the reset.
10057  **/
10058 static int i40e_reconstitute_veb(struct i40e_veb *veb)
10059 {
10060         struct i40e_vsi *ctl_vsi = NULL;
10061         struct i40e_pf *pf = veb->pf;
10062         int v, veb_idx;
10063         int ret;
10064
10065         /* build VSI that owns this VEB, temporarily attached to base VEB */
10066         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
10067                 if (pf->vsi[v] &&
10068                     pf->vsi[v]->veb_idx == veb->idx &&
10069                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
10070                         ctl_vsi = pf->vsi[v];
10071                         break;
10072                 }
10073         }
10074         if (!ctl_vsi) {
10075                 dev_info(&pf->pdev->dev,
10076                          "missing owner VSI for veb_idx %d\n", veb->idx);
10077                 ret = -ENOENT;
10078                 goto end_reconstitute;
10079         }
10080         if (ctl_vsi != pf->vsi[pf->lan_vsi])
10081                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10082         ret = i40e_add_vsi(ctl_vsi);
10083         if (ret) {
10084                 dev_info(&pf->pdev->dev,
10085                          "rebuild of veb_idx %d owner VSI failed: %d\n",
10086                          veb->idx, ret);
10087                 goto end_reconstitute;
10088         }
10089         i40e_vsi_reset_stats(ctl_vsi);
10090
10091         /* create the VEB in the switch and move the VSI onto the VEB */
10092         ret = i40e_add_veb(veb, ctl_vsi);
10093         if (ret)
10094                 goto end_reconstitute;
10095
10096         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
10097                 veb->bridge_mode = BRIDGE_MODE_VEB;
10098         else
10099                 veb->bridge_mode = BRIDGE_MODE_VEPA;
10100         i40e_config_bridge_mode(veb);
10101
10102         /* create the remaining VSIs attached to this VEB */
10103         for (v = 0; v < pf->num_alloc_vsi; v++) {
10104                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
10105                         continue;
10106
10107                 if (pf->vsi[v]->veb_idx == veb->idx) {
10108                         struct i40e_vsi *vsi = pf->vsi[v];
10109
10110                         vsi->uplink_seid = veb->seid;
10111                         ret = i40e_add_vsi(vsi);
10112                         if (ret) {
10113                                 dev_info(&pf->pdev->dev,
10114                                          "rebuild of vsi_idx %d failed: %d\n",
10115                                          v, ret);
10116                                 goto end_reconstitute;
10117                         }
10118                         i40e_vsi_reset_stats(vsi);
10119                 }
10120         }
10121
10122         /* create any VEBs attached to this VEB - RECURSION */
10123         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10124                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
10125                         pf->veb[veb_idx]->uplink_seid = veb->seid;
10126                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
10127                         if (ret)
10128                                 break;
10129                 }
10130         }
10131
10132 end_reconstitute:
10133         return ret;
10134 }
10135
10136 /**
10137  * i40e_get_capabilities - get info about the HW
10138  * @pf: the PF struct
10139  * @list_type: AQ capability to be queried
10140  **/
10141 static int i40e_get_capabilities(struct i40e_pf *pf,
10142                                  enum i40e_admin_queue_opc list_type)
10143 {
10144         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10145         u16 data_size;
10146         int buf_len;
10147         int err;
10148
10149         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10150         do {
10151                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
10152                 if (!cap_buf)
10153                         return -ENOMEM;
10154
10155                 /* this loads the data into the hw struct for us */
10156                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
10157                                                     &data_size, list_type,
10158                                                     NULL);
10159                 /* data loaded, buffer no longer needed */
10160                 kfree(cap_buf);
10161
10162                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10163                         /* retry with a larger buffer */
10164                         buf_len = data_size;
10165                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
10166                         dev_info(&pf->pdev->dev,
10167                                  "capability discovery failed, err %s aq_err %s\n",
10168                                  i40e_stat_str(&pf->hw, err),
10169                                  i40e_aq_str(&pf->hw,
10170                                              pf->hw.aq.asq_last_status));
10171                         return -ENODEV;
10172                 }
10173         } while (err);
10174
10175         if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10176                 if (list_type == i40e_aqc_opc_list_func_capabilities) {
10177                         dev_info(&pf->pdev->dev,
10178                                  "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10179                                  pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10180                                  pf->hw.func_caps.num_msix_vectors,
10181                                  pf->hw.func_caps.num_msix_vectors_vf,
10182                                  pf->hw.func_caps.fd_filters_guaranteed,
10183                                  pf->hw.func_caps.fd_filters_best_effort,
10184                                  pf->hw.func_caps.num_tx_qp,
10185                                  pf->hw.func_caps.num_vsis);
10186                 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10187                         dev_info(&pf->pdev->dev,
10188                                  "switch_mode=0x%04x, function_valid=0x%08x\n",
10189                                  pf->hw.dev_caps.switch_mode,
10190                                  pf->hw.dev_caps.valid_functions);
10191                         dev_info(&pf->pdev->dev,
10192                                  "SR-IOV=%d, num_vfs for all function=%u\n",
10193                                  pf->hw.dev_caps.sr_iov_1_1,
10194                                  pf->hw.dev_caps.num_vfs);
10195                         dev_info(&pf->pdev->dev,
10196                                  "num_vsis=%u, num_rx:%u, num_tx=%u\n",
10197                                  pf->hw.dev_caps.num_vsis,
10198                                  pf->hw.dev_caps.num_rx_qp,
10199                                  pf->hw.dev_caps.num_tx_qp);
10200                 }
10201         }
10202         if (list_type == i40e_aqc_opc_list_func_capabilities) {
10203 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10204                        + pf->hw.func_caps.num_vfs)
10205                 if (pf->hw.revision_id == 0 &&
10206                     pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10207                         dev_info(&pf->pdev->dev,
10208                                  "got num_vsis %d, setting num_vsis to %d\n",
10209                                  pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10210                         pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10211                 }
10212         }
10213         return 0;
10214 }
10215
10216 static int i40e_vsi_clear(struct i40e_vsi *vsi);
10217
10218 /**
10219  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10220  * @pf: board private structure
10221  **/
10222 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10223 {
10224         struct i40e_vsi *vsi;
10225
10226         /* quick workaround for an NVM issue that leaves a critical register
10227          * uninitialized
10228          */
10229         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10230                 static const u32 hkey[] = {
10231                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10232                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10233                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10234                         0x95b3a76d};
10235                 int i;
10236
10237                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10238                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10239         }
10240
10241         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
10242                 return;
10243
10244         /* find existing VSI and see if it needs configuring */
10245         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10246
10247         /* create a new VSI if none exists */
10248         if (!vsi) {
10249                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
10250                                      pf->vsi[pf->lan_vsi]->seid, 0);
10251                 if (!vsi) {
10252                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10253                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10254                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
10255                         return;
10256                 }
10257         }
10258
10259         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
10260 }
10261
10262 /**
10263  * i40e_fdir_teardown - release the Flow Director resources
10264  * @pf: board private structure
10265  **/
10266 static void i40e_fdir_teardown(struct i40e_pf *pf)
10267 {
10268         struct i40e_vsi *vsi;
10269
10270         i40e_fdir_filter_exit(pf);
10271         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10272         if (vsi)
10273                 i40e_vsi_release(vsi);
10274 }
10275
10276 /**
10277  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10278  * @vsi: PF main vsi
10279  * @seid: seid of main or channel VSIs
10280  *
10281  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
10282  * existed before reset
10283  **/
10284 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10285 {
10286         struct i40e_cloud_filter *cfilter;
10287         struct i40e_pf *pf = vsi->back;
10288         struct hlist_node *node;
10289         i40e_status ret;
10290
10291         /* Add cloud filters back if they exist */
10292         hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10293                                   cloud_node) {
10294                 if (cfilter->seid != seid)
10295                         continue;
10296
10297                 if (cfilter->dst_port)
10298                         ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10299                                                                 true);
10300                 else
10301                         ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10302
10303                 if (ret) {
10304                         dev_dbg(&pf->pdev->dev,
10305                                 "Failed to rebuild cloud filter, err %s aq_err %s\n",
10306                                 i40e_stat_str(&pf->hw, ret),
10307                                 i40e_aq_str(&pf->hw,
10308                                             pf->hw.aq.asq_last_status));
10309                         return ret;
10310                 }
10311         }
10312         return 0;
10313 }
10314
10315 /**
10316  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10317  * @vsi: PF main vsi
10318  *
10319  * Rebuilds channel VSIs if they existed before reset
10320  **/
10321 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10322 {
10323         struct i40e_channel *ch, *ch_tmp;
10324         i40e_status ret;
10325
10326         if (list_empty(&vsi->ch_list))
10327                 return 0;
10328
10329         list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10330                 if (!ch->initialized)
10331                         break;
10332                 /* Proceed with creation of channel (VMDq2) VSI */
10333                 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10334                 if (ret) {
10335                         dev_info(&vsi->back->pdev->dev,
10336                                  "failed to rebuild channels using uplink_seid %u\n",
10337                                  vsi->uplink_seid);
10338                         return ret;
10339                 }
10340                 /* Reconfigure TX queues using QTX_CTL register */
10341                 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10342                 if (ret) {
10343                         dev_info(&vsi->back->pdev->dev,
10344                                  "failed to configure TX rings for channel %u\n",
10345                                  ch->seid);
10346                         return ret;
10347                 }
10348                 /* update 'next_base_queue' */
10349                 vsi->next_base_queue = vsi->next_base_queue +
10350                                                         ch->num_queue_pairs;
10351                 if (ch->max_tx_rate) {
10352                         u64 credits = ch->max_tx_rate;
10353
10354                         if (i40e_set_bw_limit(vsi, ch->seid,
10355                                               ch->max_tx_rate))
10356                                 return -EINVAL;
10357
10358                         do_div(credits, I40E_BW_CREDIT_DIVISOR);
10359                         dev_dbg(&vsi->back->pdev->dev,
10360                                 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10361                                 ch->max_tx_rate,
10362                                 credits,
10363                                 ch->seid);
10364                 }
10365                 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10366                 if (ret) {
10367                         dev_dbg(&vsi->back->pdev->dev,
10368                                 "Failed to rebuild cloud filters for channel VSI %u\n",
10369                                 ch->seid);
10370                         return ret;
10371                 }
10372         }
10373         return 0;
10374 }
10375
10376 /**
10377  * i40e_prep_for_reset - prep for the core to reset
10378  * @pf: board private structure
10379  *
10380  * Close up the VFs and other things in prep for PF Reset.
10381   **/
10382 static void i40e_prep_for_reset(struct i40e_pf *pf)
10383 {
10384         struct i40e_hw *hw = &pf->hw;
10385         i40e_status ret = 0;
10386         u32 v;
10387
10388         clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10389         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10390                 return;
10391         if (i40e_check_asq_alive(&pf->hw))
10392                 i40e_vc_notify_reset(pf);
10393
10394         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10395
10396         /* quiesce the VSIs and their queues that are not already DOWN */
10397         i40e_pf_quiesce_all_vsi(pf);
10398
10399         for (v = 0; v < pf->num_alloc_vsi; v++) {
10400                 if (pf->vsi[v])
10401                         pf->vsi[v]->seid = 0;
10402         }
10403
10404         i40e_shutdown_adminq(&pf->hw);
10405
10406         /* call shutdown HMC */
10407         if (hw->hmc.hmc_obj) {
10408                 ret = i40e_shutdown_lan_hmc(hw);
10409                 if (ret)
10410                         dev_warn(&pf->pdev->dev,
10411                                  "shutdown_lan_hmc failed: %d\n", ret);
10412         }
10413
10414         /* Save the current PTP time so that we can restore the time after the
10415          * reset completes.
10416          */
10417         i40e_ptp_save_hw_time(pf);
10418 }
10419
10420 /**
10421  * i40e_send_version - update firmware with driver version
10422  * @pf: PF struct
10423  */
10424 static void i40e_send_version(struct i40e_pf *pf)
10425 {
10426         struct i40e_driver_version dv;
10427
10428         dv.major_version = 0xff;
10429         dv.minor_version = 0xff;
10430         dv.build_version = 0xff;
10431         dv.subbuild_version = 0;
10432         strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10433         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10434 }
10435
10436 /**
10437  * i40e_get_oem_version - get OEM specific version information
10438  * @hw: pointer to the hardware structure
10439  **/
10440 static void i40e_get_oem_version(struct i40e_hw *hw)
10441 {
10442         u16 block_offset = 0xffff;
10443         u16 block_length = 0;
10444         u16 capabilities = 0;
10445         u16 gen_snap = 0;
10446         u16 release = 0;
10447
10448 #define I40E_SR_NVM_OEM_VERSION_PTR             0x1B
10449 #define I40E_NVM_OEM_LENGTH_OFFSET              0x00
10450 #define I40E_NVM_OEM_CAPABILITIES_OFFSET        0x01
10451 #define I40E_NVM_OEM_GEN_OFFSET                 0x02
10452 #define I40E_NVM_OEM_RELEASE_OFFSET             0x03
10453 #define I40E_NVM_OEM_CAPABILITIES_MASK          0x000F
10454 #define I40E_NVM_OEM_LENGTH                     3
10455
10456         /* Check if pointer to OEM version block is valid. */
10457         i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10458         if (block_offset == 0xffff)
10459                 return;
10460
10461         /* Check if OEM version block has correct length. */
10462         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10463                            &block_length);
10464         if (block_length < I40E_NVM_OEM_LENGTH)
10465                 return;
10466
10467         /* Check if OEM version format is as expected. */
10468         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10469                            &capabilities);
10470         if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10471                 return;
10472
10473         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10474                            &gen_snap);
10475         i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10476                            &release);
10477         hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
10478         hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10479 }
10480
10481 /**
10482  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10483  * @pf: board private structure
10484  **/
10485 static int i40e_reset(struct i40e_pf *pf)
10486 {
10487         struct i40e_hw *hw = &pf->hw;
10488         i40e_status ret;
10489
10490         ret = i40e_pf_reset(hw);
10491         if (ret) {
10492                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10493                 set_bit(__I40E_RESET_FAILED, pf->state);
10494                 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10495         } else {
10496                 pf->pfr_count++;
10497         }
10498         return ret;
10499 }
10500
10501 /**
10502  * i40e_rebuild - rebuild using a saved config
10503  * @pf: board private structure
10504  * @reinit: if the Main VSI needs to re-initialized.
10505  * @lock_acquired: indicates whether or not the lock has been acquired
10506  * before this function was called.
10507  **/
10508 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10509 {
10510         int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state);
10511         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10512         struct i40e_hw *hw = &pf->hw;
10513         i40e_status ret;
10514         u32 val;
10515         int v;
10516
10517         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10518             i40e_check_recovery_mode(pf)) {
10519                 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev);
10520         }
10521
10522         if (test_bit(__I40E_DOWN, pf->state) &&
10523             !test_bit(__I40E_RECOVERY_MODE, pf->state) &&
10524             !old_recovery_mode_bit)
10525                 goto clear_recovery;
10526         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10527
10528         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10529         ret = i40e_init_adminq(&pf->hw);
10530         if (ret) {
10531                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
10532                          i40e_stat_str(&pf->hw, ret),
10533                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10534                 goto clear_recovery;
10535         }
10536         i40e_get_oem_version(&pf->hw);
10537
10538         if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10539             ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) ||
10540              hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) {
10541                 /* The following delay is necessary for 4.33 firmware and older
10542                  * to recover after EMP reset. 200 ms should suffice but we
10543                  * put here 300 ms to be sure that FW is ready to operate
10544                  * after reset.
10545                  */
10546                 mdelay(300);
10547         }
10548
10549         /* re-verify the eeprom if we just had an EMP reset */
10550         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10551                 i40e_verify_eeprom(pf);
10552
10553         /* if we are going out of or into recovery mode we have to act
10554          * accordingly with regard to resources initialization
10555          * and deinitialization
10556          */
10557         if (test_bit(__I40E_RECOVERY_MODE, pf->state) ||
10558             old_recovery_mode_bit) {
10559                 if (i40e_get_capabilities(pf,
10560                                           i40e_aqc_opc_list_func_capabilities))
10561                         goto end_unlock;
10562
10563                 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10564                         /* we're staying in recovery mode so we'll reinitialize
10565                          * misc vector here
10566                          */
10567                         if (i40e_setup_misc_vector_for_recovery_mode(pf))
10568                                 goto end_unlock;
10569                 } else {
10570                         if (!lock_acquired)
10571                                 rtnl_lock();
10572                         /* we're going out of recovery mode so we'll free
10573                          * the IRQ allocated specifically for recovery mode
10574                          * and restore the interrupt scheme
10575                          */
10576                         free_irq(pf->pdev->irq, pf);
10577                         i40e_clear_interrupt_scheme(pf);
10578                         if (i40e_restore_interrupt_scheme(pf))
10579                                 goto end_unlock;
10580                 }
10581
10582                 /* tell the firmware that we're starting */
10583                 i40e_send_version(pf);
10584
10585                 /* bail out in case recovery mode was detected, as there is
10586                  * no need for further configuration.
10587                  */
10588                 goto end_unlock;
10589         }
10590
10591         i40e_clear_pxe_mode(hw);
10592         ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10593         if (ret)
10594                 goto end_core_reset;
10595
10596         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10597                                 hw->func_caps.num_rx_qp, 0, 0);
10598         if (ret) {
10599                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10600                 goto end_core_reset;
10601         }
10602         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10603         if (ret) {
10604                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10605                 goto end_core_reset;
10606         }
10607
10608 #ifdef CONFIG_I40E_DCB
10609         /* Enable FW to write a default DCB config on link-up
10610          * unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10611          * is not supported with new link speed
10612          */
10613         if (pf->flags & I40E_FLAG_TC_MQPRIO) {
10614                 i40e_aq_set_dcb_parameters(hw, false, NULL);
10615         } else {
10616                 if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10617                     (hw->phy.link_info.link_speed &
10618                      (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10619                         i40e_aq_set_dcb_parameters(hw, false, NULL);
10620                         dev_warn(&pf->pdev->dev,
10621                                  "DCB is not supported for X710-T*L 2.5/5G speeds\n");
10622                         pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10623                 } else {
10624                         i40e_aq_set_dcb_parameters(hw, true, NULL);
10625                         ret = i40e_init_pf_dcb(pf);
10626                         if (ret) {
10627                                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10628                                          ret);
10629                                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10630                                 /* Continue without DCB enabled */
10631                         }
10632                 }
10633         }
10634
10635 #endif /* CONFIG_I40E_DCB */
10636         if (!lock_acquired)
10637                 rtnl_lock();
10638         ret = i40e_setup_pf_switch(pf, reinit, true);
10639         if (ret)
10640                 goto end_unlock;
10641
10642         /* The driver only wants link up/down and module qualification
10643          * reports from firmware.  Note the negative logic.
10644          */
10645         ret = i40e_aq_set_phy_int_mask(&pf->hw,
10646                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
10647                                          I40E_AQ_EVENT_MEDIA_NA |
10648                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10649         if (ret)
10650                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
10651                          i40e_stat_str(&pf->hw, ret),
10652                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10653
10654         /* Rebuild the VSIs and VEBs that existed before reset.
10655          * They are still in our local switch element arrays, so only
10656          * need to rebuild the switch model in the HW.
10657          *
10658          * If there were VEBs but the reconstitution failed, we'll try
10659          * to recover minimal use by getting the basic PF VSI working.
10660          */
10661         if (vsi->uplink_seid != pf->mac_seid) {
10662                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10663                 /* find the one VEB connected to the MAC, and find orphans */
10664                 for (v = 0; v < I40E_MAX_VEB; v++) {
10665                         if (!pf->veb[v])
10666                                 continue;
10667
10668                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
10669                             pf->veb[v]->uplink_seid == 0) {
10670                                 ret = i40e_reconstitute_veb(pf->veb[v]);
10671
10672                                 if (!ret)
10673                                         continue;
10674
10675                                 /* If Main VEB failed, we're in deep doodoo,
10676                                  * so give up rebuilding the switch and set up
10677                                  * for minimal rebuild of PF VSI.
10678                                  * If orphan failed, we'll report the error
10679                                  * but try to keep going.
10680                                  */
10681                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
10682                                         dev_info(&pf->pdev->dev,
10683                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
10684                                                  ret);
10685                                         vsi->uplink_seid = pf->mac_seid;
10686                                         break;
10687                                 } else if (pf->veb[v]->uplink_seid == 0) {
10688                                         dev_info(&pf->pdev->dev,
10689                                                  "rebuild of orphan VEB failed: %d\n",
10690                                                  ret);
10691                                 }
10692                         }
10693                 }
10694         }
10695
10696         if (vsi->uplink_seid == pf->mac_seid) {
10697                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
10698                 /* no VEB, so rebuild only the Main VSI */
10699                 ret = i40e_add_vsi(vsi);
10700                 if (ret) {
10701                         dev_info(&pf->pdev->dev,
10702                                  "rebuild of Main VSI failed: %d\n", ret);
10703                         goto end_unlock;
10704                 }
10705         }
10706
10707         if (vsi->mqprio_qopt.max_rate[0]) {
10708                 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
10709                 u64 credits = 0;
10710
10711                 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
10712                 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
10713                 if (ret)
10714                         goto end_unlock;
10715
10716                 credits = max_tx_rate;
10717                 do_div(credits, I40E_BW_CREDIT_DIVISOR);
10718                 dev_dbg(&vsi->back->pdev->dev,
10719                         "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10720                         max_tx_rate,
10721                         credits,
10722                         vsi->seid);
10723         }
10724
10725         ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
10726         if (ret)
10727                 goto end_unlock;
10728
10729         /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
10730          * for this main VSI if they exist
10731          */
10732         ret = i40e_rebuild_channels(vsi);
10733         if (ret)
10734                 goto end_unlock;
10735
10736         /* Reconfigure hardware for allowing smaller MSS in the case
10737          * of TSO, so that we avoid the MDD being fired and causing
10738          * a reset in the case of small MSS+TSO.
10739          */
10740 #define I40E_REG_MSS          0x000E64DC
10741 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
10742 #define I40E_64BYTE_MSS       0x400000
10743         val = rd32(hw, I40E_REG_MSS);
10744         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
10745                 val &= ~I40E_REG_MSS_MIN_MASK;
10746                 val |= I40E_64BYTE_MSS;
10747                 wr32(hw, I40E_REG_MSS, val);
10748         }
10749
10750         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
10751                 msleep(75);
10752                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
10753                 if (ret)
10754                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
10755                                  i40e_stat_str(&pf->hw, ret),
10756                                  i40e_aq_str(&pf->hw,
10757                                              pf->hw.aq.asq_last_status));
10758         }
10759         /* reinit the misc interrupt */
10760         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10761                 ret = i40e_setup_misc_vector(pf);
10762
10763         /* Add a filter to drop all Flow control frames from any VSI from being
10764          * transmitted. By doing so we stop a malicious VF from sending out
10765          * PAUSE or PFC frames and potentially controlling traffic for other
10766          * PF/VF VSIs.
10767          * The FW can still send Flow control frames if enabled.
10768          */
10769         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
10770                                                        pf->main_vsi_seid);
10771
10772         /* restart the VSIs that were rebuilt and running before the reset */
10773         i40e_pf_unquiesce_all_vsi(pf);
10774
10775         /* Release the RTNL lock before we start resetting VFs */
10776         if (!lock_acquired)
10777                 rtnl_unlock();
10778
10779         /* Restore promiscuous settings */
10780         ret = i40e_set_promiscuous(pf, pf->cur_promisc);
10781         if (ret)
10782                 dev_warn(&pf->pdev->dev,
10783                          "Failed to restore promiscuous setting: %s, err %s aq_err %s\n",
10784                          pf->cur_promisc ? "on" : "off",
10785                          i40e_stat_str(&pf->hw, ret),
10786                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10787
10788         i40e_reset_all_vfs(pf, true);
10789
10790         /* tell the firmware that we're starting */
10791         i40e_send_version(pf);
10792
10793         /* We've already released the lock, so don't do it again */
10794         goto end_core_reset;
10795
10796 end_unlock:
10797         if (!lock_acquired)
10798                 rtnl_unlock();
10799 end_core_reset:
10800         clear_bit(__I40E_RESET_FAILED, pf->state);
10801 clear_recovery:
10802         clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10803         clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
10804 }
10805
10806 /**
10807  * i40e_reset_and_rebuild - reset and rebuild using a saved config
10808  * @pf: board private structure
10809  * @reinit: if the Main VSI needs to re-initialized.
10810  * @lock_acquired: indicates whether or not the lock has been acquired
10811  * before this function was called.
10812  **/
10813 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
10814                                    bool lock_acquired)
10815 {
10816         int ret;
10817         /* Now we wait for GRST to settle out.
10818          * We don't have to delete the VEBs or VSIs from the hw switch
10819          * because the reset will make them disappear.
10820          */
10821         ret = i40e_reset(pf);
10822         if (!ret)
10823                 i40e_rebuild(pf, reinit, lock_acquired);
10824 }
10825
10826 /**
10827  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
10828  * @pf: board private structure
10829  *
10830  * Close up the VFs and other things in prep for a Core Reset,
10831  * then get ready to rebuild the world.
10832  * @lock_acquired: indicates whether or not the lock has been acquired
10833  * before this function was called.
10834  **/
10835 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
10836 {
10837         i40e_prep_for_reset(pf);
10838         i40e_reset_and_rebuild(pf, false, lock_acquired);
10839 }
10840
10841 /**
10842  * i40e_handle_mdd_event
10843  * @pf: pointer to the PF structure
10844  *
10845  * Called from the MDD irq handler to identify possibly malicious vfs
10846  **/
10847 static void i40e_handle_mdd_event(struct i40e_pf *pf)
10848 {
10849         struct i40e_hw *hw = &pf->hw;
10850         bool mdd_detected = false;
10851         struct i40e_vf *vf;
10852         u32 reg;
10853         int i;
10854
10855         if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
10856                 return;
10857
10858         /* find what triggered the MDD event */
10859         reg = rd32(hw, I40E_GL_MDET_TX);
10860         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
10861                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
10862                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
10863                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
10864                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
10865                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
10866                                 I40E_GL_MDET_TX_EVENT_SHIFT;
10867                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
10868                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
10869                                 pf->hw.func_caps.base_queue;
10870                 if (netif_msg_tx_err(pf))
10871                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
10872                                  event, queue, pf_num, vf_num);
10873                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
10874                 mdd_detected = true;
10875         }
10876         reg = rd32(hw, I40E_GL_MDET_RX);
10877         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
10878                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
10879                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
10880                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
10881                                 I40E_GL_MDET_RX_EVENT_SHIFT;
10882                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
10883                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
10884                                 pf->hw.func_caps.base_queue;
10885                 if (netif_msg_rx_err(pf))
10886                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
10887                                  event, queue, func);
10888                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
10889                 mdd_detected = true;
10890         }
10891
10892         if (mdd_detected) {
10893                 reg = rd32(hw, I40E_PF_MDET_TX);
10894                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
10895                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
10896                         dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
10897                 }
10898                 reg = rd32(hw, I40E_PF_MDET_RX);
10899                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
10900                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
10901                         dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
10902                 }
10903         }
10904
10905         /* see if one of the VFs needs its hand slapped */
10906         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
10907                 vf = &(pf->vf[i]);
10908                 reg = rd32(hw, I40E_VP_MDET_TX(i));
10909                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
10910                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
10911                         vf->num_mdd_events++;
10912                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
10913                                  i);
10914                         dev_info(&pf->pdev->dev,
10915                                  "Use PF Control I/F to re-enable the VF\n");
10916                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10917                 }
10918
10919                 reg = rd32(hw, I40E_VP_MDET_RX(i));
10920                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
10921                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
10922                         vf->num_mdd_events++;
10923                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
10924                                  i);
10925                         dev_info(&pf->pdev->dev,
10926                                  "Use PF Control I/F to re-enable the VF\n");
10927                         set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10928                 }
10929         }
10930
10931         /* re-enable mdd interrupt cause */
10932         clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
10933         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
10934         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
10935         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
10936         i40e_flush(hw);
10937 }
10938
10939 /**
10940  * i40e_service_task - Run the driver's async subtasks
10941  * @work: pointer to work_struct containing our data
10942  **/
10943 static void i40e_service_task(struct work_struct *work)
10944 {
10945         struct i40e_pf *pf = container_of(work,
10946                                           struct i40e_pf,
10947                                           service_task);
10948         unsigned long start_time = jiffies;
10949
10950         /* don't bother with service tasks if a reset is in progress */
10951         if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
10952             test_bit(__I40E_SUSPENDED, pf->state))
10953                 return;
10954
10955         if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
10956                 return;
10957
10958         if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10959                 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
10960                 i40e_sync_filters_subtask(pf);
10961                 i40e_reset_subtask(pf);
10962                 i40e_handle_mdd_event(pf);
10963                 i40e_vc_process_vflr_event(pf);
10964                 i40e_watchdog_subtask(pf);
10965                 i40e_fdir_reinit_subtask(pf);
10966                 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
10967                         /* Client subtask will reopen next time through. */
10968                         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi],
10969                                                            true);
10970                 } else {
10971                         i40e_client_subtask(pf);
10972                         if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
10973                                                pf->state))
10974                                 i40e_notify_client_of_l2_param_changes(
10975                                                                 pf->vsi[pf->lan_vsi]);
10976                 }
10977                 i40e_sync_filters_subtask(pf);
10978         } else {
10979                 i40e_reset_subtask(pf);
10980         }
10981
10982         i40e_clean_adminq_subtask(pf);
10983
10984         /* flush memory to make sure state is correct before next watchdog */
10985         smp_mb__before_atomic();
10986         clear_bit(__I40E_SERVICE_SCHED, pf->state);
10987
10988         /* If the tasks have taken longer than one timer cycle or there
10989          * is more work to be done, reschedule the service task now
10990          * rather than wait for the timer to tick again.
10991          */
10992         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
10993             test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)             ||
10994             test_bit(__I40E_MDD_EVENT_PENDING, pf->state)                ||
10995             test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
10996                 i40e_service_event_schedule(pf);
10997 }
10998
10999 /**
11000  * i40e_service_timer - timer callback
11001  * @t: timer list pointer
11002  **/
11003 static void i40e_service_timer(struct timer_list *t)
11004 {
11005         struct i40e_pf *pf = from_timer(pf, t, service_timer);
11006
11007         mod_timer(&pf->service_timer,
11008                   round_jiffies(jiffies + pf->service_timer_period));
11009         i40e_service_event_schedule(pf);
11010 }
11011
11012 /**
11013  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11014  * @vsi: the VSI being configured
11015  **/
11016 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
11017 {
11018         struct i40e_pf *pf = vsi->back;
11019
11020         switch (vsi->type) {
11021         case I40E_VSI_MAIN:
11022                 vsi->alloc_queue_pairs = pf->num_lan_qps;
11023                 if (!vsi->num_tx_desc)
11024                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11025                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11026                 if (!vsi->num_rx_desc)
11027                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11028                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11029                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11030                         vsi->num_q_vectors = pf->num_lan_msix;
11031                 else
11032                         vsi->num_q_vectors = 1;
11033
11034                 break;
11035
11036         case I40E_VSI_FDIR:
11037                 vsi->alloc_queue_pairs = 1;
11038                 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11039                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11040                 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11041                                          I40E_REQ_DESCRIPTOR_MULTIPLE);
11042                 vsi->num_q_vectors = pf->num_fdsb_msix;
11043                 break;
11044
11045         case I40E_VSI_VMDQ2:
11046                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11047                 if (!vsi->num_tx_desc)
11048                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11049                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11050                 if (!vsi->num_rx_desc)
11051                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11052                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11053                 vsi->num_q_vectors = pf->num_vmdq_msix;
11054                 break;
11055
11056         case I40E_VSI_SRIOV:
11057                 vsi->alloc_queue_pairs = pf->num_vf_qps;
11058                 if (!vsi->num_tx_desc)
11059                         vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11060                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11061                 if (!vsi->num_rx_desc)
11062                         vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11063                                                  I40E_REQ_DESCRIPTOR_MULTIPLE);
11064                 break;
11065
11066         default:
11067                 WARN_ON(1);
11068                 return -ENODATA;
11069         }
11070
11071         if (is_kdump_kernel()) {
11072                 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;
11073                 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;
11074         }
11075
11076         return 0;
11077 }
11078
11079 /**
11080  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11081  * @vsi: VSI pointer
11082  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11083  *
11084  * On error: returns error code (negative)
11085  * On success: returns 0
11086  **/
11087 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11088 {
11089         struct i40e_ring **next_rings;
11090         int size;
11091         int ret = 0;
11092
11093         /* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11094         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11095                (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11096         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11097         if (!vsi->tx_rings)
11098                 return -ENOMEM;
11099         next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11100         if (i40e_enabled_xdp_vsi(vsi)) {
11101                 vsi->xdp_rings = next_rings;
11102                 next_rings += vsi->alloc_queue_pairs;
11103         }
11104         vsi->rx_rings = next_rings;
11105
11106         if (alloc_qvectors) {
11107                 /* allocate memory for q_vector pointers */
11108                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11109                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11110                 if (!vsi->q_vectors) {
11111                         ret = -ENOMEM;
11112                         goto err_vectors;
11113                 }
11114         }
11115         return ret;
11116
11117 err_vectors:
11118         kfree(vsi->tx_rings);
11119         return ret;
11120 }
11121
11122 /**
11123  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11124  * @pf: board private structure
11125  * @type: type of VSI
11126  *
11127  * On error: returns error code (negative)
11128  * On success: returns vsi index in PF (positive)
11129  **/
11130 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11131 {
11132         int ret = -ENODEV;
11133         struct i40e_vsi *vsi;
11134         int vsi_idx;
11135         int i;
11136
11137         /* Need to protect the allocation of the VSIs at the PF level */
11138         mutex_lock(&pf->switch_mutex);
11139
11140         /* VSI list may be fragmented if VSI creation/destruction has
11141          * been happening.  We can afford to do a quick scan to look
11142          * for any free VSIs in the list.
11143          *
11144          * find next empty vsi slot, looping back around if necessary
11145          */
11146         i = pf->next_vsi;
11147         while (i < pf->num_alloc_vsi && pf->vsi[i])
11148                 i++;
11149         if (i >= pf->num_alloc_vsi) {
11150                 i = 0;
11151                 while (i < pf->next_vsi && pf->vsi[i])
11152                         i++;
11153         }
11154
11155         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11156                 vsi_idx = i;             /* Found one! */
11157         } else {
11158                 ret = -ENODEV;
11159                 goto unlock_pf;  /* out of VSI slots! */
11160         }
11161         pf->next_vsi = ++i;
11162
11163         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
11164         if (!vsi) {
11165                 ret = -ENOMEM;
11166                 goto unlock_pf;
11167         }
11168         vsi->type = type;
11169         vsi->back = pf;
11170         set_bit(__I40E_VSI_DOWN, vsi->state);
11171         vsi->flags = 0;
11172         vsi->idx = vsi_idx;
11173         vsi->int_rate_limit = 0;
11174         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11175                                 pf->rss_table_size : 64;
11176         vsi->netdev_registered = false;
11177         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11178         hash_init(vsi->mac_filter_hash);
11179         vsi->irqs_ready = false;
11180
11181         if (type == I40E_VSI_MAIN) {
11182                 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
11183                 if (!vsi->af_xdp_zc_qps)
11184                         goto err_rings;
11185         }
11186
11187         ret = i40e_set_num_rings_in_vsi(vsi);
11188         if (ret)
11189                 goto err_rings;
11190
11191         ret = i40e_vsi_alloc_arrays(vsi, true);
11192         if (ret)
11193                 goto err_rings;
11194
11195         /* Setup default MSIX irq handler for VSI */
11196         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
11197
11198         /* Initialize VSI lock */
11199         spin_lock_init(&vsi->mac_filter_hash_lock);
11200         pf->vsi[vsi_idx] = vsi;
11201         ret = vsi_idx;
11202         goto unlock_pf;
11203
11204 err_rings:
11205         bitmap_free(vsi->af_xdp_zc_qps);
11206         pf->next_vsi = i - 1;
11207         kfree(vsi);
11208 unlock_pf:
11209         mutex_unlock(&pf->switch_mutex);
11210         return ret;
11211 }
11212
11213 /**
11214  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11215  * @vsi: VSI pointer
11216  * @free_qvectors: a bool to specify if q_vectors need to be freed.
11217  *
11218  * On error: returns error code (negative)
11219  * On success: returns 0
11220  **/
11221 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11222 {
11223         /* free the ring and vector containers */
11224         if (free_qvectors) {
11225                 kfree(vsi->q_vectors);
11226                 vsi->q_vectors = NULL;
11227         }
11228         kfree(vsi->tx_rings);
11229         vsi->tx_rings = NULL;
11230         vsi->rx_rings = NULL;
11231         vsi->xdp_rings = NULL;
11232 }
11233
11234 /**
11235  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11236  * and lookup table
11237  * @vsi: Pointer to VSI structure
11238  */
11239 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11240 {
11241         if (!vsi)
11242                 return;
11243
11244         kfree(vsi->rss_hkey_user);
11245         vsi->rss_hkey_user = NULL;
11246
11247         kfree(vsi->rss_lut_user);
11248         vsi->rss_lut_user = NULL;
11249 }
11250
11251 /**
11252  * i40e_vsi_clear - Deallocate the VSI provided
11253  * @vsi: the VSI being un-configured
11254  **/
11255 static int i40e_vsi_clear(struct i40e_vsi *vsi)
11256 {
11257         struct i40e_pf *pf;
11258
11259         if (!vsi)
11260                 return 0;
11261
11262         if (!vsi->back)
11263                 goto free_vsi;
11264         pf = vsi->back;
11265
11266         mutex_lock(&pf->switch_mutex);
11267         if (!pf->vsi[vsi->idx]) {
11268                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11269                         vsi->idx, vsi->idx, vsi->type);
11270                 goto unlock_vsi;
11271         }
11272
11273         if (pf->vsi[vsi->idx] != vsi) {
11274                 dev_err(&pf->pdev->dev,
11275                         "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11276                         pf->vsi[vsi->idx]->idx,
11277                         pf->vsi[vsi->idx]->type,
11278                         vsi->idx, vsi->type);
11279                 goto unlock_vsi;
11280         }
11281
11282         /* updates the PF for this cleared vsi */
11283         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
11284         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
11285
11286         bitmap_free(vsi->af_xdp_zc_qps);
11287         i40e_vsi_free_arrays(vsi, true);
11288         i40e_clear_rss_config_user(vsi);
11289
11290         pf->vsi[vsi->idx] = NULL;
11291         if (vsi->idx < pf->next_vsi)
11292                 pf->next_vsi = vsi->idx;
11293
11294 unlock_vsi:
11295         mutex_unlock(&pf->switch_mutex);
11296 free_vsi:
11297         kfree(vsi);
11298
11299         return 0;
11300 }
11301
11302 /**
11303  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11304  * @vsi: the VSI being cleaned
11305  **/
11306 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11307 {
11308         int i;
11309
11310         if (vsi->tx_rings && vsi->tx_rings[0]) {
11311                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11312                         kfree_rcu(vsi->tx_rings[i], rcu);
11313                         WRITE_ONCE(vsi->tx_rings[i], NULL);
11314                         WRITE_ONCE(vsi->rx_rings[i], NULL);
11315                         if (vsi->xdp_rings)
11316                                 WRITE_ONCE(vsi->xdp_rings[i], NULL);
11317                 }
11318         }
11319 }
11320
11321 /**
11322  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11323  * @vsi: the VSI being configured
11324  **/
11325 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11326 {
11327         int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11328         struct i40e_pf *pf = vsi->back;
11329         struct i40e_ring *ring;
11330
11331         /* Set basic values in the rings to be used later during open() */
11332         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11333                 /* allocate space for both Tx and Rx in one shot */
11334                 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11335                 if (!ring)
11336                         goto err_out;
11337
11338                 ring->queue_index = i;
11339                 ring->reg_idx = vsi->base_queue + i;
11340                 ring->ring_active = false;
11341                 ring->vsi = vsi;
11342                 ring->netdev = vsi->netdev;
11343                 ring->dev = &pf->pdev->dev;
11344                 ring->count = vsi->num_tx_desc;
11345                 ring->size = 0;
11346                 ring->dcb_tc = 0;
11347                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11348                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11349                 ring->itr_setting = pf->tx_itr_default;
11350                 WRITE_ONCE(vsi->tx_rings[i], ring++);
11351
11352                 if (!i40e_enabled_xdp_vsi(vsi))
11353                         goto setup_rx;
11354
11355                 ring->queue_index = vsi->alloc_queue_pairs + i;
11356                 ring->reg_idx = vsi->base_queue + ring->queue_index;
11357                 ring->ring_active = false;
11358                 ring->vsi = vsi;
11359                 ring->netdev = NULL;
11360                 ring->dev = &pf->pdev->dev;
11361                 ring->count = vsi->num_tx_desc;
11362                 ring->size = 0;
11363                 ring->dcb_tc = 0;
11364                 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11365                         ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11366                 set_ring_xdp(ring);
11367                 ring->itr_setting = pf->tx_itr_default;
11368                 WRITE_ONCE(vsi->xdp_rings[i], ring++);
11369
11370 setup_rx:
11371                 ring->queue_index = i;
11372                 ring->reg_idx = vsi->base_queue + i;
11373                 ring->ring_active = false;
11374                 ring->vsi = vsi;
11375                 ring->netdev = vsi->netdev;
11376                 ring->dev = &pf->pdev->dev;
11377                 ring->count = vsi->num_rx_desc;
11378                 ring->size = 0;
11379                 ring->dcb_tc = 0;
11380                 ring->itr_setting = pf->rx_itr_default;
11381                 WRITE_ONCE(vsi->rx_rings[i], ring);
11382         }
11383
11384         return 0;
11385
11386 err_out:
11387         i40e_vsi_clear_rings(vsi);
11388         return -ENOMEM;
11389 }
11390
11391 /**
11392  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11393  * @pf: board private structure
11394  * @vectors: the number of MSI-X vectors to request
11395  *
11396  * Returns the number of vectors reserved, or error
11397  **/
11398 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11399 {
11400         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11401                                         I40E_MIN_MSIX, vectors);
11402         if (vectors < 0) {
11403                 dev_info(&pf->pdev->dev,
11404                          "MSI-X vector reservation failed: %d\n", vectors);
11405                 vectors = 0;
11406         }
11407
11408         return vectors;
11409 }
11410
11411 /**
11412  * i40e_init_msix - Setup the MSIX capability
11413  * @pf: board private structure
11414  *
11415  * Work with the OS to set up the MSIX vectors needed.
11416  *
11417  * Returns the number of vectors reserved or negative on failure
11418  **/
11419 static int i40e_init_msix(struct i40e_pf *pf)
11420 {
11421         struct i40e_hw *hw = &pf->hw;
11422         int cpus, extra_vectors;
11423         int vectors_left;
11424         int v_budget, i;
11425         int v_actual;
11426         int iwarp_requested = 0;
11427
11428         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
11429                 return -ENODEV;
11430
11431         /* The number of vectors we'll request will be comprised of:
11432          *   - Add 1 for "other" cause for Admin Queue events, etc.
11433          *   - The number of LAN queue pairs
11434          *      - Queues being used for RSS.
11435          *              We don't need as many as max_rss_size vectors.
11436          *              use rss_size instead in the calculation since that
11437          *              is governed by number of cpus in the system.
11438          *      - assumes symmetric Tx/Rx pairing
11439          *   - The number of VMDq pairs
11440          *   - The CPU count within the NUMA node if iWARP is enabled
11441          * Once we count this up, try the request.
11442          *
11443          * If we can't get what we want, we'll simplify to nearly nothing
11444          * and try again.  If that still fails, we punt.
11445          */
11446         vectors_left = hw->func_caps.num_msix_vectors;
11447         v_budget = 0;
11448
11449         /* reserve one vector for miscellaneous handler */
11450         if (vectors_left) {
11451                 v_budget++;
11452                 vectors_left--;
11453         }
11454
11455         /* reserve some vectors for the main PF traffic queues. Initially we
11456          * only reserve at most 50% of the available vectors, in the case that
11457          * the number of online CPUs is large. This ensures that we can enable
11458          * extra features as well. Once we've enabled the other features, we
11459          * will use any remaining vectors to reach as close as we can to the
11460          * number of online CPUs.
11461          */
11462         cpus = num_online_cpus();
11463         pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11464         vectors_left -= pf->num_lan_msix;
11465
11466         /* reserve one vector for sideband flow director */
11467         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11468                 if (vectors_left) {
11469                         pf->num_fdsb_msix = 1;
11470                         v_budget++;
11471                         vectors_left--;
11472                 } else {
11473                         pf->num_fdsb_msix = 0;
11474                 }
11475         }
11476
11477         /* can we reserve enough for iWARP? */
11478         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11479                 iwarp_requested = pf->num_iwarp_msix;
11480
11481                 if (!vectors_left)
11482                         pf->num_iwarp_msix = 0;
11483                 else if (vectors_left < pf->num_iwarp_msix)
11484                         pf->num_iwarp_msix = 1;
11485                 v_budget += pf->num_iwarp_msix;
11486                 vectors_left -= pf->num_iwarp_msix;
11487         }
11488
11489         /* any vectors left over go for VMDq support */
11490         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
11491                 if (!vectors_left) {
11492                         pf->num_vmdq_msix = 0;
11493                         pf->num_vmdq_qps = 0;
11494                 } else {
11495                         int vmdq_vecs_wanted =
11496                                 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11497                         int vmdq_vecs =
11498                                 min_t(int, vectors_left, vmdq_vecs_wanted);
11499
11500                         /* if we're short on vectors for what's desired, we limit
11501                          * the queues per vmdq.  If this is still more than are
11502                          * available, the user will need to change the number of
11503                          * queues/vectors used by the PF later with the ethtool
11504                          * channels command
11505                          */
11506                         if (vectors_left < vmdq_vecs_wanted) {
11507                                 pf->num_vmdq_qps = 1;
11508                                 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11509                                 vmdq_vecs = min_t(int,
11510                                                   vectors_left,
11511                                                   vmdq_vecs_wanted);
11512                         }
11513                         pf->num_vmdq_msix = pf->num_vmdq_qps;
11514
11515                         v_budget += vmdq_vecs;
11516                         vectors_left -= vmdq_vecs;
11517                 }
11518         }
11519
11520         /* On systems with a large number of SMP cores, we previously limited
11521          * the number of vectors for num_lan_msix to be at most 50% of the
11522          * available vectors, to allow for other features. Now, we add back
11523          * the remaining vectors. However, we ensure that the total
11524          * num_lan_msix will not exceed num_online_cpus(). To do this, we
11525          * calculate the number of vectors we can add without going over the
11526          * cap of CPUs. For systems with a small number of CPUs this will be
11527          * zero.
11528          */
11529         extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11530         pf->num_lan_msix += extra_vectors;
11531         vectors_left -= extra_vectors;
11532
11533         WARN(vectors_left < 0,
11534              "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11535
11536         v_budget += pf->num_lan_msix;
11537         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11538                                    GFP_KERNEL);
11539         if (!pf->msix_entries)
11540                 return -ENOMEM;
11541
11542         for (i = 0; i < v_budget; i++)
11543                 pf->msix_entries[i].entry = i;
11544         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11545
11546         if (v_actual < I40E_MIN_MSIX) {
11547                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
11548                 kfree(pf->msix_entries);
11549                 pf->msix_entries = NULL;
11550                 pci_disable_msix(pf->pdev);
11551                 return -ENODEV;
11552
11553         } else if (v_actual == I40E_MIN_MSIX) {
11554                 /* Adjust for minimal MSIX use */
11555                 pf->num_vmdq_vsis = 0;
11556                 pf->num_vmdq_qps = 0;
11557                 pf->num_lan_qps = 1;
11558                 pf->num_lan_msix = 1;
11559
11560         } else if (v_actual != v_budget) {
11561                 /* If we have limited resources, we will start with no vectors
11562                  * for the special features and then allocate vectors to some
11563                  * of these features based on the policy and at the end disable
11564                  * the features that did not get any vectors.
11565                  */
11566                 int vec;
11567
11568                 dev_info(&pf->pdev->dev,
11569                          "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11570                          v_actual, v_budget);
11571                 /* reserve the misc vector */
11572                 vec = v_actual - 1;
11573
11574                 /* Scale vector usage down */
11575                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
11576                 pf->num_vmdq_vsis = 1;
11577                 pf->num_vmdq_qps = 1;
11578
11579                 /* partition out the remaining vectors */
11580                 switch (vec) {
11581                 case 2:
11582                         pf->num_lan_msix = 1;
11583                         break;
11584                 case 3:
11585                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11586                                 pf->num_lan_msix = 1;
11587                                 pf->num_iwarp_msix = 1;
11588                         } else {
11589                                 pf->num_lan_msix = 2;
11590                         }
11591                         break;
11592                 default:
11593                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11594                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
11595                                                  iwarp_requested);
11596                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11597                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11598                         } else {
11599                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11600                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
11601                         }
11602                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11603                                 pf->num_fdsb_msix = 1;
11604                                 vec--;
11605                         }
11606                         pf->num_lan_msix = min_t(int,
11607                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11608                                                               pf->num_lan_msix);
11609                         pf->num_lan_qps = pf->num_lan_msix;
11610                         break;
11611                 }
11612         }
11613
11614         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
11615             (pf->num_fdsb_msix == 0)) {
11616                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11617                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11618                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11619         }
11620         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11621             (pf->num_vmdq_msix == 0)) {
11622                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11623                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
11624         }
11625
11626         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
11627             (pf->num_iwarp_msix == 0)) {
11628                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11629                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11630         }
11631         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11632                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11633                    pf->num_lan_msix,
11634                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
11635                    pf->num_fdsb_msix,
11636                    pf->num_iwarp_msix);
11637
11638         return v_actual;
11639 }
11640
11641 /**
11642  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11643  * @vsi: the VSI being configured
11644  * @v_idx: index of the vector in the vsi struct
11645  *
11646  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
11647  **/
11648 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
11649 {
11650         struct i40e_q_vector *q_vector;
11651
11652         /* allocate q_vector */
11653         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11654         if (!q_vector)
11655                 return -ENOMEM;
11656
11657         q_vector->vsi = vsi;
11658         q_vector->v_idx = v_idx;
11659         cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
11660
11661         if (vsi->netdev)
11662                 netif_napi_add(vsi->netdev, &q_vector->napi,
11663                                i40e_napi_poll, NAPI_POLL_WEIGHT);
11664
11665         /* tie q_vector and vsi together */
11666         vsi->q_vectors[v_idx] = q_vector;
11667
11668         return 0;
11669 }
11670
11671 /**
11672  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
11673  * @vsi: the VSI being configured
11674  *
11675  * We allocate one q_vector per queue interrupt.  If allocation fails we
11676  * return -ENOMEM.
11677  **/
11678 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11679 {
11680         struct i40e_pf *pf = vsi->back;
11681         int err, v_idx, num_q_vectors;
11682
11683         /* if not MSIX, give the one vector only to the LAN VSI */
11684         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11685                 num_q_vectors = vsi->num_q_vectors;
11686         else if (vsi == pf->vsi[pf->lan_vsi])
11687                 num_q_vectors = 1;
11688         else
11689                 return -EINVAL;
11690
11691         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
11692                 err = i40e_vsi_alloc_q_vector(vsi, v_idx);
11693                 if (err)
11694                         goto err_out;
11695         }
11696
11697         return 0;
11698
11699 err_out:
11700         while (v_idx--)
11701                 i40e_free_q_vector(vsi, v_idx);
11702
11703         return err;
11704 }
11705
11706 /**
11707  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
11708  * @pf: board private structure to initialize
11709  **/
11710 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
11711 {
11712         int vectors = 0;
11713         ssize_t size;
11714
11715         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11716                 vectors = i40e_init_msix(pf);
11717                 if (vectors < 0) {
11718                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
11719                                        I40E_FLAG_IWARP_ENABLED  |
11720                                        I40E_FLAG_RSS_ENABLED    |
11721                                        I40E_FLAG_DCB_CAPABLE    |
11722                                        I40E_FLAG_DCB_ENABLED    |
11723                                        I40E_FLAG_SRIOV_ENABLED  |
11724                                        I40E_FLAG_FD_SB_ENABLED  |
11725                                        I40E_FLAG_FD_ATR_ENABLED |
11726                                        I40E_FLAG_VMDQ_ENABLED);
11727                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11728
11729                         /* rework the queue expectations without MSIX */
11730                         i40e_determine_queue_usage(pf);
11731                 }
11732         }
11733
11734         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11735             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
11736                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
11737                 vectors = pci_enable_msi(pf->pdev);
11738                 if (vectors < 0) {
11739                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
11740                                  vectors);
11741                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
11742                 }
11743                 vectors = 1;  /* one MSI or Legacy vector */
11744         }
11745
11746         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
11747                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
11748
11749         /* set up vector assignment tracking */
11750         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
11751         pf->irq_pile = kzalloc(size, GFP_KERNEL);
11752         if (!pf->irq_pile)
11753                 return -ENOMEM;
11754
11755         pf->irq_pile->num_entries = vectors;
11756         pf->irq_pile->search_hint = 0;
11757
11758         /* track first vector for misc interrupts, ignore return */
11759         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
11760
11761         return 0;
11762 }
11763
11764 /**
11765  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
11766  * @pf: private board data structure
11767  *
11768  * Restore the interrupt scheme that was cleared when we suspended the
11769  * device. This should be called during resume to re-allocate the q_vectors
11770  * and reacquire IRQs.
11771  */
11772 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
11773 {
11774         int err, i;
11775
11776         /* We cleared the MSI and MSI-X flags when disabling the old interrupt
11777          * scheme. We need to re-enabled them here in order to attempt to
11778          * re-acquire the MSI or MSI-X vectors
11779          */
11780         pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
11781
11782         err = i40e_init_interrupt_scheme(pf);
11783         if (err)
11784                 return err;
11785
11786         /* Now that we've re-acquired IRQs, we need to remap the vectors and
11787          * rings together again.
11788          */
11789         for (i = 0; i < pf->num_alloc_vsi; i++) {
11790                 if (pf->vsi[i]) {
11791                         err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
11792                         if (err)
11793                                 goto err_unwind;
11794                         i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
11795                 }
11796         }
11797
11798         err = i40e_setup_misc_vector(pf);
11799         if (err)
11800                 goto err_unwind;
11801
11802         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
11803                 i40e_client_update_msix_info(pf);
11804
11805         return 0;
11806
11807 err_unwind:
11808         while (i--) {
11809                 if (pf->vsi[i])
11810                         i40e_vsi_free_q_vectors(pf->vsi[i]);
11811         }
11812
11813         return err;
11814 }
11815
11816 /**
11817  * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
11818  * non queue events in recovery mode
11819  * @pf: board private structure
11820  *
11821  * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
11822  * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
11823  * This is handled differently than in recovery mode since no Tx/Rx resources
11824  * are being allocated.
11825  **/
11826 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
11827 {
11828         int err;
11829
11830         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11831                 err = i40e_setup_misc_vector(pf);
11832
11833                 if (err) {
11834                         dev_info(&pf->pdev->dev,
11835                                  "MSI-X misc vector request failed, error %d\n",
11836                                  err);
11837                         return err;
11838                 }
11839         } else {
11840                 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED;
11841
11842                 err = request_irq(pf->pdev->irq, i40e_intr, flags,
11843                                   pf->int_name, pf);
11844
11845                 if (err) {
11846                         dev_info(&pf->pdev->dev,
11847                                  "MSI/legacy misc vector request failed, error %d\n",
11848                                  err);
11849                         return err;
11850                 }
11851                 i40e_enable_misc_int_causes(pf);
11852                 i40e_irq_dynamic_enable_icr0(pf);
11853         }
11854
11855         return 0;
11856 }
11857
11858 /**
11859  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
11860  * @pf: board private structure
11861  *
11862  * This sets up the handler for MSIX 0, which is used to manage the
11863  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
11864  * when in MSI or Legacy interrupt mode.
11865  **/
11866 static int i40e_setup_misc_vector(struct i40e_pf *pf)
11867 {
11868         struct i40e_hw *hw = &pf->hw;
11869         int err = 0;
11870
11871         /* Only request the IRQ once, the first time through. */
11872         if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
11873                 err = request_irq(pf->msix_entries[0].vector,
11874                                   i40e_intr, 0, pf->int_name, pf);
11875                 if (err) {
11876                         clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
11877                         dev_info(&pf->pdev->dev,
11878                                  "request_irq for %s failed: %d\n",
11879                                  pf->int_name, err);
11880                         return -EFAULT;
11881                 }
11882         }
11883
11884         i40e_enable_misc_int_causes(pf);
11885
11886         /* associate no queues to the misc vector */
11887         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
11888         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
11889
11890         i40e_flush(hw);
11891
11892         i40e_irq_dynamic_enable_icr0(pf);
11893
11894         return err;
11895 }
11896
11897 /**
11898  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
11899  * @vsi: Pointer to vsi structure
11900  * @seed: Buffter to store the hash keys
11901  * @lut: Buffer to store the lookup table entries
11902  * @lut_size: Size of buffer to store the lookup table entries
11903  *
11904  * Return 0 on success, negative on failure
11905  */
11906 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
11907                            u8 *lut, u16 lut_size)
11908 {
11909         struct i40e_pf *pf = vsi->back;
11910         struct i40e_hw *hw = &pf->hw;
11911         int ret = 0;
11912
11913         if (seed) {
11914                 ret = i40e_aq_get_rss_key(hw, vsi->id,
11915                         (struct i40e_aqc_get_set_rss_key_data *)seed);
11916                 if (ret) {
11917                         dev_info(&pf->pdev->dev,
11918                                  "Cannot get RSS key, err %s aq_err %s\n",
11919                                  i40e_stat_str(&pf->hw, ret),
11920                                  i40e_aq_str(&pf->hw,
11921                                              pf->hw.aq.asq_last_status));
11922                         return ret;
11923                 }
11924         }
11925
11926         if (lut) {
11927                 bool pf_lut = vsi->type == I40E_VSI_MAIN;
11928
11929                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
11930                 if (ret) {
11931                         dev_info(&pf->pdev->dev,
11932                                  "Cannot get RSS lut, err %s aq_err %s\n",
11933                                  i40e_stat_str(&pf->hw, ret),
11934                                  i40e_aq_str(&pf->hw,
11935                                              pf->hw.aq.asq_last_status));
11936                         return ret;
11937                 }
11938         }
11939
11940         return ret;
11941 }
11942
11943 /**
11944  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
11945  * @vsi: Pointer to vsi structure
11946  * @seed: RSS hash seed
11947  * @lut: Lookup table
11948  * @lut_size: Lookup table size
11949  *
11950  * Returns 0 on success, negative on failure
11951  **/
11952 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
11953                                const u8 *lut, u16 lut_size)
11954 {
11955         struct i40e_pf *pf = vsi->back;
11956         struct i40e_hw *hw = &pf->hw;
11957         u16 vf_id = vsi->vf_id;
11958         u8 i;
11959
11960         /* Fill out hash function seed */
11961         if (seed) {
11962                 u32 *seed_dw = (u32 *)seed;
11963
11964                 if (vsi->type == I40E_VSI_MAIN) {
11965                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11966                                 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
11967                 } else if (vsi->type == I40E_VSI_SRIOV) {
11968                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
11969                                 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
11970                 } else {
11971                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
11972                 }
11973         }
11974
11975         if (lut) {
11976                 u32 *lut_dw = (u32 *)lut;
11977
11978                 if (vsi->type == I40E_VSI_MAIN) {
11979                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
11980                                 return -EINVAL;
11981                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11982                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
11983                 } else if (vsi->type == I40E_VSI_SRIOV) {
11984                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
11985                                 return -EINVAL;
11986                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
11987                                 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
11988                 } else {
11989                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
11990                 }
11991         }
11992         i40e_flush(hw);
11993
11994         return 0;
11995 }
11996
11997 /**
11998  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
11999  * @vsi: Pointer to VSI structure
12000  * @seed: Buffer to store the keys
12001  * @lut: Buffer to store the lookup table entries
12002  * @lut_size: Size of buffer to store the lookup table entries
12003  *
12004  * Returns 0 on success, negative on failure
12005  */
12006 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
12007                             u8 *lut, u16 lut_size)
12008 {
12009         struct i40e_pf *pf = vsi->back;
12010         struct i40e_hw *hw = &pf->hw;
12011         u16 i;
12012
12013         if (seed) {
12014                 u32 *seed_dw = (u32 *)seed;
12015
12016                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12017                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
12018         }
12019         if (lut) {
12020                 u32 *lut_dw = (u32 *)lut;
12021
12022                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
12023                         return -EINVAL;
12024                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12025                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
12026         }
12027
12028         return 0;
12029 }
12030
12031 /**
12032  * i40e_config_rss - Configure RSS keys and lut
12033  * @vsi: Pointer to VSI structure
12034  * @seed: RSS hash seed
12035  * @lut: Lookup table
12036  * @lut_size: Lookup table size
12037  *
12038  * Returns 0 on success, negative on failure
12039  */
12040 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12041 {
12042         struct i40e_pf *pf = vsi->back;
12043
12044         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12045                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12046         else
12047                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12048 }
12049
12050 /**
12051  * i40e_get_rss - Get RSS keys and lut
12052  * @vsi: Pointer to VSI structure
12053  * @seed: Buffer to store the keys
12054  * @lut: Buffer to store the lookup table entries
12055  * @lut_size: Size of buffer to store the lookup table entries
12056  *
12057  * Returns 0 on success, negative on failure
12058  */
12059 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12060 {
12061         struct i40e_pf *pf = vsi->back;
12062
12063         if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12064                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12065         else
12066                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12067 }
12068
12069 /**
12070  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12071  * @pf: Pointer to board private structure
12072  * @lut: Lookup table
12073  * @rss_table_size: Lookup table size
12074  * @rss_size: Range of queue number for hashing
12075  */
12076 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12077                        u16 rss_table_size, u16 rss_size)
12078 {
12079         u16 i;
12080
12081         for (i = 0; i < rss_table_size; i++)
12082                 lut[i] = i % rss_size;
12083 }
12084
12085 /**
12086  * i40e_pf_config_rss - Prepare for RSS if used
12087  * @pf: board private structure
12088  **/
12089 static int i40e_pf_config_rss(struct i40e_pf *pf)
12090 {
12091         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12092         u8 seed[I40E_HKEY_ARRAY_SIZE];
12093         u8 *lut;
12094         struct i40e_hw *hw = &pf->hw;
12095         u32 reg_val;
12096         u64 hena;
12097         int ret;
12098
12099         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12100         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12101                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12102         hena |= i40e_pf_get_default_rss_hena(pf);
12103
12104         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
12105         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
12106
12107         /* Determine the RSS table size based on the hardware capabilities */
12108         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12109         reg_val = (pf->rss_table_size == 512) ?
12110                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12111                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12112         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12113
12114         /* Determine the RSS size of the VSI */
12115         if (!vsi->rss_size) {
12116                 u16 qcount;
12117                 /* If the firmware does something weird during VSI init, we
12118                  * could end up with zero TCs. Check for that to avoid
12119                  * divide-by-zero. It probably won't pass traffic, but it also
12120                  * won't panic.
12121                  */
12122                 qcount = vsi->num_queue_pairs /
12123                          (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12124                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12125         }
12126         if (!vsi->rss_size)
12127                 return -EINVAL;
12128
12129         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
12130         if (!lut)
12131                 return -ENOMEM;
12132
12133         /* Use user configured lut if there is one, otherwise use default */
12134         if (vsi->rss_lut_user)
12135                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12136         else
12137                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
12138
12139         /* Use user configured hash key if there is one, otherwise
12140          * use default.
12141          */
12142         if (vsi->rss_hkey_user)
12143                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12144         else
12145                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
12146         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
12147         kfree(lut);
12148
12149         return ret;
12150 }
12151
12152 /**
12153  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12154  * @pf: board private structure
12155  * @queue_count: the requested queue count for rss.
12156  *
12157  * returns 0 if rss is not enabled, if enabled returns the final rss queue
12158  * count which may be different from the requested queue count.
12159  * Note: expects to be called while under rtnl_lock()
12160  **/
12161 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12162 {
12163         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12164         int new_rss_size;
12165
12166         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
12167                 return 0;
12168
12169         queue_count = min_t(int, queue_count, num_online_cpus());
12170         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12171
12172         if (queue_count != vsi->num_queue_pairs) {
12173                 u16 qcount;
12174
12175                 vsi->req_queue_pairs = queue_count;
12176                 i40e_prep_for_reset(pf);
12177
12178                 pf->alloc_rss_size = new_rss_size;
12179
12180                 i40e_reset_and_rebuild(pf, true, true);
12181
12182                 /* Discard the user configured hash keys and lut, if less
12183                  * queues are enabled.
12184                  */
12185                 if (queue_count < vsi->rss_size) {
12186                         i40e_clear_rss_config_user(vsi);
12187                         dev_dbg(&pf->pdev->dev,
12188                                 "discard user configured hash keys and lut\n");
12189                 }
12190
12191                 /* Reset vsi->rss_size, as number of enabled queues changed */
12192                 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12193                 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12194
12195                 i40e_pf_config_rss(pf);
12196         }
12197         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
12198                  vsi->req_queue_pairs, pf->rss_size_max);
12199         return pf->alloc_rss_size;
12200 }
12201
12202 /**
12203  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12204  * @pf: board private structure
12205  **/
12206 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
12207 {
12208         i40e_status status;
12209         bool min_valid, max_valid;
12210         u32 max_bw, min_bw;
12211
12212         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
12213                                            &min_valid, &max_valid);
12214
12215         if (!status) {
12216                 if (min_valid)
12217                         pf->min_bw = min_bw;
12218                 if (max_valid)
12219                         pf->max_bw = max_bw;
12220         }
12221
12222         return status;
12223 }
12224
12225 /**
12226  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12227  * @pf: board private structure
12228  **/
12229 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
12230 {
12231         struct i40e_aqc_configure_partition_bw_data bw_data;
12232         i40e_status status;
12233
12234         memset(&bw_data, 0, sizeof(bw_data));
12235
12236         /* Set the valid bit for this PF */
12237         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12238         bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12239         bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12240
12241         /* Set the new bandwidths */
12242         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
12243
12244         return status;
12245 }
12246
12247 /**
12248  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12249  * @pf: board private structure
12250  **/
12251 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
12252 {
12253         /* Commit temporary BW setting to permanent NVM image */
12254         enum i40e_admin_queue_err last_aq_status;
12255         i40e_status ret;
12256         u16 nvm_word;
12257
12258         if (pf->hw.partition_id != 1) {
12259                 dev_info(&pf->pdev->dev,
12260                          "Commit BW only works on partition 1! This is partition %d",
12261                          pf->hw.partition_id);
12262                 ret = I40E_NOT_SUPPORTED;
12263                 goto bw_commit_out;
12264         }
12265
12266         /* Acquire NVM for read access */
12267         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
12268         last_aq_status = pf->hw.aq.asq_last_status;
12269         if (ret) {
12270                 dev_info(&pf->pdev->dev,
12271                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
12272                          i40e_stat_str(&pf->hw, ret),
12273                          i40e_aq_str(&pf->hw, last_aq_status));
12274                 goto bw_commit_out;
12275         }
12276
12277         /* Read word 0x10 of NVM - SW compatibility word 1 */
12278         ret = i40e_aq_read_nvm(&pf->hw,
12279                                I40E_SR_NVM_CONTROL_WORD,
12280                                0x10, sizeof(nvm_word), &nvm_word,
12281                                false, NULL);
12282         /* Save off last admin queue command status before releasing
12283          * the NVM
12284          */
12285         last_aq_status = pf->hw.aq.asq_last_status;
12286         i40e_release_nvm(&pf->hw);
12287         if (ret) {
12288                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
12289                          i40e_stat_str(&pf->hw, ret),
12290                          i40e_aq_str(&pf->hw, last_aq_status));
12291                 goto bw_commit_out;
12292         }
12293
12294         /* Wait a bit for NVM release to complete */
12295         msleep(50);
12296
12297         /* Acquire NVM for write access */
12298         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
12299         last_aq_status = pf->hw.aq.asq_last_status;
12300         if (ret) {
12301                 dev_info(&pf->pdev->dev,
12302                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
12303                          i40e_stat_str(&pf->hw, ret),
12304                          i40e_aq_str(&pf->hw, last_aq_status));
12305                 goto bw_commit_out;
12306         }
12307         /* Write it back out unchanged to initiate update NVM,
12308          * which will force a write of the shadow (alt) RAM to
12309          * the NVM - thus storing the bandwidth values permanently.
12310          */
12311         ret = i40e_aq_update_nvm(&pf->hw,
12312                                  I40E_SR_NVM_CONTROL_WORD,
12313                                  0x10, sizeof(nvm_word),
12314                                  &nvm_word, true, 0, NULL);
12315         /* Save off last admin queue command status before releasing
12316          * the NVM
12317          */
12318         last_aq_status = pf->hw.aq.asq_last_status;
12319         i40e_release_nvm(&pf->hw);
12320         if (ret)
12321                 dev_info(&pf->pdev->dev,
12322                          "BW settings NOT SAVED, err %s aq_err %s\n",
12323                          i40e_stat_str(&pf->hw, ret),
12324                          i40e_aq_str(&pf->hw, last_aq_status));
12325 bw_commit_out:
12326
12327         return ret;
12328 }
12329
12330 /**
12331  * i40e_is_total_port_shutdown_enabled - read NVM and return value
12332  * if total port shutdown feature is enabled for this PF
12333  * @pf: board private structure
12334  **/
12335 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12336 {
12337 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED        BIT(4)
12338 #define I40E_FEATURES_ENABLE_PTR                0x2A
12339 #define I40E_CURRENT_SETTING_PTR                0x2B
12340 #define I40E_LINK_BEHAVIOR_WORD_OFFSET          0x2D
12341 #define I40E_LINK_BEHAVIOR_WORD_LENGTH          0x1
12342 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED    BIT(0)
12343 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH      4
12344         i40e_status read_status = I40E_SUCCESS;
12345         u16 sr_emp_sr_settings_ptr = 0;
12346         u16 features_enable = 0;
12347         u16 link_behavior = 0;
12348         bool ret = false;
12349
12350         read_status = i40e_read_nvm_word(&pf->hw,
12351                                          I40E_SR_EMP_SR_SETTINGS_PTR,
12352                                          &sr_emp_sr_settings_ptr);
12353         if (read_status)
12354                 goto err_nvm;
12355         read_status = i40e_read_nvm_word(&pf->hw,
12356                                          sr_emp_sr_settings_ptr +
12357                                          I40E_FEATURES_ENABLE_PTR,
12358                                          &features_enable);
12359         if (read_status)
12360                 goto err_nvm;
12361         if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12362                 read_status = i40e_read_nvm_module_data(&pf->hw,
12363                                                         I40E_SR_EMP_SR_SETTINGS_PTR,
12364                                                         I40E_CURRENT_SETTING_PTR,
12365                                                         I40E_LINK_BEHAVIOR_WORD_OFFSET,
12366                                                         I40E_LINK_BEHAVIOR_WORD_LENGTH,
12367                                                         &link_behavior);
12368                 if (read_status)
12369                         goto err_nvm;
12370                 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12371                 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12372         }
12373         return ret;
12374
12375 err_nvm:
12376         dev_warn(&pf->pdev->dev,
12377                  "total-port-shutdown feature is off due to read nvm error: %s\n",
12378                  i40e_stat_str(&pf->hw, read_status));
12379         return ret;
12380 }
12381
12382 /**
12383  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12384  * @pf: board private structure to initialize
12385  *
12386  * i40e_sw_init initializes the Adapter private data structure.
12387  * Fields are initialized based on PCI device information and
12388  * OS network device settings (MTU size).
12389  **/
12390 static int i40e_sw_init(struct i40e_pf *pf)
12391 {
12392         int err = 0;
12393         int size;
12394         u16 pow;
12395
12396         /* Set default capability flags */
12397         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
12398                     I40E_FLAG_MSI_ENABLED     |
12399                     I40E_FLAG_MSIX_ENABLED;
12400
12401         /* Set default ITR */
12402         pf->rx_itr_default = I40E_ITR_RX_DEF;
12403         pf->tx_itr_default = I40E_ITR_TX_DEF;
12404
12405         /* Depending on PF configurations, it is possible that the RSS
12406          * maximum might end up larger than the available queues
12407          */
12408         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12409         pf->alloc_rss_size = 1;
12410         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12411         pf->rss_size_max = min_t(int, pf->rss_size_max,
12412                                  pf->hw.func_caps.num_tx_qp);
12413
12414         /* find the next higher power-of-2 of num cpus */
12415         pow = roundup_pow_of_two(num_online_cpus());
12416         pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12417
12418         if (pf->hw.func_caps.rss) {
12419                 pf->flags |= I40E_FLAG_RSS_ENABLED;
12420                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12421                                            num_online_cpus());
12422         }
12423
12424         /* MFP mode enabled */
12425         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12426                 pf->flags |= I40E_FLAG_MFP_ENABLED;
12427                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12428                 if (i40e_get_partition_bw_setting(pf)) {
12429                         dev_warn(&pf->pdev->dev,
12430                                  "Could not get partition bw settings\n");
12431                 } else {
12432                         dev_info(&pf->pdev->dev,
12433                                  "Partition BW Min = %8.8x, Max = %8.8x\n",
12434                                  pf->min_bw, pf->max_bw);
12435
12436                         /* nudge the Tx scheduler */
12437                         i40e_set_partition_bw_setting(pf);
12438                 }
12439         }
12440
12441         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12442             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12443                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
12444                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
12445                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
12446                     pf->hw.num_partitions > 1)
12447                         dev_info(&pf->pdev->dev,
12448                                  "Flow Director Sideband mode Disabled in MFP mode\n");
12449                 else
12450                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12451                 pf->fdir_pf_filter_count =
12452                                  pf->hw.func_caps.fd_filters_guaranteed;
12453                 pf->hw.fdir_shared_filter_count =
12454                                  pf->hw.func_caps.fd_filters_best_effort;
12455         }
12456
12457         if (pf->hw.mac.type == I40E_MAC_X722) {
12458                 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
12459                                     I40E_HW_128_QP_RSS_CAPABLE |
12460                                     I40E_HW_ATR_EVICT_CAPABLE |
12461                                     I40E_HW_WB_ON_ITR_CAPABLE |
12462                                     I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
12463                                     I40E_HW_NO_PCI_LINK_CHECK |
12464                                     I40E_HW_USE_SET_LLDP_MIB |
12465                                     I40E_HW_GENEVE_OFFLOAD_CAPABLE |
12466                                     I40E_HW_PTP_L4_CAPABLE |
12467                                     I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
12468                                     I40E_HW_OUTER_UDP_CSUM_CAPABLE);
12469
12470 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
12471                 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
12472                     I40E_FDEVICT_PCTYPE_DEFAULT) {
12473                         dev_warn(&pf->pdev->dev,
12474                                  "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
12475                         pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
12476                 }
12477         } else if ((pf->hw.aq.api_maj_ver > 1) ||
12478                    ((pf->hw.aq.api_maj_ver == 1) &&
12479                     (pf->hw.aq.api_min_ver > 4))) {
12480                 /* Supported in FW API version higher than 1.4 */
12481                 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
12482         }
12483
12484         /* Enable HW ATR eviction if possible */
12485         if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
12486                 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
12487
12488         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12489             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
12490             (pf->hw.aq.fw_maj_ver < 4))) {
12491                 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
12492                 /* No DCB support  for FW < v4.33 */
12493                 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
12494         }
12495
12496         /* Disable FW LLDP if FW < v4.3 */
12497         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12498             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
12499             (pf->hw.aq.fw_maj_ver < 4)))
12500                 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
12501
12502         /* Use the FW Set LLDP MIB API if FW > v4.40 */
12503         if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12504             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
12505             (pf->hw.aq.fw_maj_ver >= 5)))
12506                 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
12507
12508         /* Enable PTP L4 if FW > v6.0 */
12509         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12510             pf->hw.aq.fw_maj_ver >= 6)
12511                 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
12512
12513         if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12514                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12515                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
12516                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12517         }
12518
12519         if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12520                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
12521                 /* IWARP needs one extra vector for CQP just like MISC.*/
12522                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12523         }
12524         /* Stopping FW LLDP engine is supported on XL710 and X722
12525          * starting from FW versions determined in i40e_init_adminq.
12526          * Stopping the FW LLDP engine is not supported on XL710
12527          * if NPAR is functioning so unset this hw flag in this case.
12528          */
12529         if (pf->hw.mac.type == I40E_MAC_XL710 &&
12530             pf->hw.func_caps.npar_enable &&
12531             (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
12532                 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
12533
12534 #ifdef CONFIG_PCI_IOV
12535         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12536                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12537                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
12538                 pf->num_req_vfs = min_t(int,
12539                                         pf->hw.func_caps.num_vfs,
12540                                         I40E_MAX_VF_COUNT);
12541         }
12542 #endif /* CONFIG_PCI_IOV */
12543         pf->eeprom_version = 0xDEAD;
12544         pf->lan_veb = I40E_NO_VEB;
12545         pf->lan_vsi = I40E_NO_VSI;
12546
12547         /* By default FW has this off for performance reasons */
12548         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
12549
12550         /* set up queue assignment tracking */
12551         size = sizeof(struct i40e_lump_tracking)
12552                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12553         pf->qp_pile = kzalloc(size, GFP_KERNEL);
12554         if (!pf->qp_pile) {
12555                 err = -ENOMEM;
12556                 goto sw_init_done;
12557         }
12558         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12559         pf->qp_pile->search_hint = 0;
12560
12561         pf->tx_timeout_recovery_level = 1;
12562
12563         if (pf->hw.mac.type != I40E_MAC_X722 &&
12564             i40e_is_total_port_shutdown_enabled(pf)) {
12565                 /* Link down on close must be on when total port shutdown
12566                  * is enabled for a given port
12567                  */
12568                 pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED |
12569                               I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED);
12570                 dev_info(&pf->pdev->dev,
12571                          "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12572         }
12573         mutex_init(&pf->switch_mutex);
12574
12575 sw_init_done:
12576         return err;
12577 }
12578
12579 /**
12580  * i40e_set_ntuple - set the ntuple feature flag and take action
12581  * @pf: board private structure to initialize
12582  * @features: the feature set that the stack is suggesting
12583  *
12584  * returns a bool to indicate if reset needs to happen
12585  **/
12586 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12587 {
12588         bool need_reset = false;
12589
12590         /* Check if Flow Director n-tuple support was enabled or disabled.  If
12591          * the state changed, we need to reset.
12592          */
12593         if (features & NETIF_F_NTUPLE) {
12594                 /* Enable filters and mark for reset */
12595                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
12596                         need_reset = true;
12597                 /* enable FD_SB only if there is MSI-X vector and no cloud
12598                  * filters exist
12599                  */
12600                 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12601                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12602                         pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
12603                 }
12604         } else {
12605                 /* turn off filters, mark for reset and clear SW filter list */
12606                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
12607                         need_reset = true;
12608                         i40e_fdir_filter_exit(pf);
12609                 }
12610                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
12611                 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12612                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12613
12614                 /* reset fd counters */
12615                 pf->fd_add_err = 0;
12616                 pf->fd_atr_cnt = 0;
12617                 /* if ATR was auto disabled it can be re-enabled. */
12618                 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12619                         if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
12620                             (I40E_DEBUG_FD & pf->hw.debug_mask))
12621                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12622         }
12623         return need_reset;
12624 }
12625
12626 /**
12627  * i40e_clear_rss_lut - clear the rx hash lookup table
12628  * @vsi: the VSI being configured
12629  **/
12630 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12631 {
12632         struct i40e_pf *pf = vsi->back;
12633         struct i40e_hw *hw = &pf->hw;
12634         u16 vf_id = vsi->vf_id;
12635         u8 i;
12636
12637         if (vsi->type == I40E_VSI_MAIN) {
12638                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12639                         wr32(hw, I40E_PFQF_HLUT(i), 0);
12640         } else if (vsi->type == I40E_VSI_SRIOV) {
12641                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12642                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12643         } else {
12644                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12645         }
12646 }
12647
12648 /**
12649  * i40e_set_features - set the netdev feature flags
12650  * @netdev: ptr to the netdev being adjusted
12651  * @features: the feature set that the stack is suggesting
12652  * Note: expects to be called while under rtnl_lock()
12653  **/
12654 static int i40e_set_features(struct net_device *netdev,
12655                              netdev_features_t features)
12656 {
12657         struct i40e_netdev_priv *np = netdev_priv(netdev);
12658         struct i40e_vsi *vsi = np->vsi;
12659         struct i40e_pf *pf = vsi->back;
12660         bool need_reset;
12661
12662         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12663                 i40e_pf_config_rss(pf);
12664         else if (!(features & NETIF_F_RXHASH) &&
12665                  netdev->features & NETIF_F_RXHASH)
12666                 i40e_clear_rss_lut(vsi);
12667
12668         if (features & NETIF_F_HW_VLAN_CTAG_RX)
12669                 i40e_vlan_stripping_enable(vsi);
12670         else
12671                 i40e_vlan_stripping_disable(vsi);
12672
12673         if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12674                 dev_err(&pf->pdev->dev,
12675                         "Offloaded tc filters active, can't turn hw_tc_offload off");
12676                 return -EINVAL;
12677         }
12678
12679         if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12680                 i40e_del_all_macvlans(vsi);
12681
12682         need_reset = i40e_set_ntuple(pf, features);
12683
12684         if (need_reset)
12685                 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12686
12687         return 0;
12688 }
12689
12690 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
12691                                     unsigned int table, unsigned int idx,
12692                                     struct udp_tunnel_info *ti)
12693 {
12694         struct i40e_netdev_priv *np = netdev_priv(netdev);
12695         struct i40e_hw *hw = &np->vsi->back->hw;
12696         u8 type, filter_index;
12697         i40e_status ret;
12698
12699         type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
12700                                                    I40E_AQC_TUNNEL_TYPE_NGE;
12701
12702         ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
12703                                      NULL);
12704         if (ret) {
12705                 netdev_info(netdev, "add UDP port failed, err %s aq_err %s\n",
12706                             i40e_stat_str(hw, ret),
12707                             i40e_aq_str(hw, hw->aq.asq_last_status));
12708                 return -EIO;
12709         }
12710
12711         udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
12712         return 0;
12713 }
12714
12715 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
12716                                       unsigned int table, unsigned int idx,
12717                                       struct udp_tunnel_info *ti)
12718 {
12719         struct i40e_netdev_priv *np = netdev_priv(netdev);
12720         struct i40e_hw *hw = &np->vsi->back->hw;
12721         i40e_status ret;
12722
12723         ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
12724         if (ret) {
12725                 netdev_info(netdev, "delete UDP port failed, err %s aq_err %s\n",
12726                             i40e_stat_str(hw, ret),
12727                             i40e_aq_str(hw, hw->aq.asq_last_status));
12728                 return -EIO;
12729         }
12730
12731         return 0;
12732 }
12733
12734 static int i40e_get_phys_port_id(struct net_device *netdev,
12735                                  struct netdev_phys_item_id *ppid)
12736 {
12737         struct i40e_netdev_priv *np = netdev_priv(netdev);
12738         struct i40e_pf *pf = np->vsi->back;
12739         struct i40e_hw *hw = &pf->hw;
12740
12741         if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
12742                 return -EOPNOTSUPP;
12743
12744         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
12745         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
12746
12747         return 0;
12748 }
12749
12750 /**
12751  * i40e_ndo_fdb_add - add an entry to the hardware database
12752  * @ndm: the input from the stack
12753  * @tb: pointer to array of nladdr (unused)
12754  * @dev: the net device pointer
12755  * @addr: the MAC address entry being added
12756  * @vid: VLAN ID
12757  * @flags: instructions from stack about fdb operation
12758  * @extack: netlink extended ack, unused currently
12759  */
12760 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
12761                             struct net_device *dev,
12762                             const unsigned char *addr, u16 vid,
12763                             u16 flags,
12764                             struct netlink_ext_ack *extack)
12765 {
12766         struct i40e_netdev_priv *np = netdev_priv(dev);
12767         struct i40e_pf *pf = np->vsi->back;
12768         int err = 0;
12769
12770         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
12771                 return -EOPNOTSUPP;
12772
12773         if (vid) {
12774                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
12775                 return -EINVAL;
12776         }
12777
12778         /* Hardware does not support aging addresses so if a
12779          * ndm_state is given only allow permanent addresses
12780          */
12781         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
12782                 netdev_info(dev, "FDB only supports static addresses\n");
12783                 return -EINVAL;
12784         }
12785
12786         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
12787                 err = dev_uc_add_excl(dev, addr);
12788         else if (is_multicast_ether_addr(addr))
12789                 err = dev_mc_add_excl(dev, addr);
12790         else
12791                 err = -EINVAL;
12792
12793         /* Only return duplicate errors if NLM_F_EXCL is set */
12794         if (err == -EEXIST && !(flags & NLM_F_EXCL))
12795                 err = 0;
12796
12797         return err;
12798 }
12799
12800 /**
12801  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
12802  * @dev: the netdev being configured
12803  * @nlh: RTNL message
12804  * @flags: bridge flags
12805  * @extack: netlink extended ack
12806  *
12807  * Inserts a new hardware bridge if not already created and
12808  * enables the bridging mode requested (VEB or VEPA). If the
12809  * hardware bridge has already been inserted and the request
12810  * is to change the mode then that requires a PF reset to
12811  * allow rebuild of the components with required hardware
12812  * bridge mode enabled.
12813  *
12814  * Note: expects to be called while under rtnl_lock()
12815  **/
12816 static int i40e_ndo_bridge_setlink(struct net_device *dev,
12817                                    struct nlmsghdr *nlh,
12818                                    u16 flags,
12819                                    struct netlink_ext_ack *extack)
12820 {
12821         struct i40e_netdev_priv *np = netdev_priv(dev);
12822         struct i40e_vsi *vsi = np->vsi;
12823         struct i40e_pf *pf = vsi->back;
12824         struct i40e_veb *veb = NULL;
12825         struct nlattr *attr, *br_spec;
12826         int i, rem;
12827
12828         /* Only for PF VSI for now */
12829         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12830                 return -EOPNOTSUPP;
12831
12832         /* Find the HW bridge for PF VSI */
12833         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12834                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12835                         veb = pf->veb[i];
12836         }
12837
12838         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12839
12840         nla_for_each_nested(attr, br_spec, rem) {
12841                 __u16 mode;
12842
12843                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12844                         continue;
12845
12846                 mode = nla_get_u16(attr);
12847                 if ((mode != BRIDGE_MODE_VEPA) &&
12848                     (mode != BRIDGE_MODE_VEB))
12849                         return -EINVAL;
12850
12851                 /* Insert a new HW bridge */
12852                 if (!veb) {
12853                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
12854                                              vsi->tc_config.enabled_tc);
12855                         if (veb) {
12856                                 veb->bridge_mode = mode;
12857                                 i40e_config_bridge_mode(veb);
12858                         } else {
12859                                 /* No Bridge HW offload available */
12860                                 return -ENOENT;
12861                         }
12862                         break;
12863                 } else if (mode != veb->bridge_mode) {
12864                         /* Existing HW bridge but different mode needs reset */
12865                         veb->bridge_mode = mode;
12866                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
12867                         if (mode == BRIDGE_MODE_VEB)
12868                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
12869                         else
12870                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
12871                         i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12872                         break;
12873                 }
12874         }
12875
12876         return 0;
12877 }
12878
12879 /**
12880  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
12881  * @skb: skb buff
12882  * @pid: process id
12883  * @seq: RTNL message seq #
12884  * @dev: the netdev being configured
12885  * @filter_mask: unused
12886  * @nlflags: netlink flags passed in
12887  *
12888  * Return the mode in which the hardware bridge is operating in
12889  * i.e VEB or VEPA.
12890  **/
12891 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
12892                                    struct net_device *dev,
12893                                    u32 __always_unused filter_mask,
12894                                    int nlflags)
12895 {
12896         struct i40e_netdev_priv *np = netdev_priv(dev);
12897         struct i40e_vsi *vsi = np->vsi;
12898         struct i40e_pf *pf = vsi->back;
12899         struct i40e_veb *veb = NULL;
12900         int i;
12901
12902         /* Only for PF VSI for now */
12903         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12904                 return -EOPNOTSUPP;
12905
12906         /* Find the HW bridge for the PF VSI */
12907         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12908                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12909                         veb = pf->veb[i];
12910         }
12911
12912         if (!veb)
12913                 return 0;
12914
12915         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
12916                                        0, 0, nlflags, filter_mask, NULL);
12917 }
12918
12919 /**
12920  * i40e_features_check - Validate encapsulated packet conforms to limits
12921  * @skb: skb buff
12922  * @dev: This physical port's netdev
12923  * @features: Offload features that the stack believes apply
12924  **/
12925 static netdev_features_t i40e_features_check(struct sk_buff *skb,
12926                                              struct net_device *dev,
12927                                              netdev_features_t features)
12928 {
12929         size_t len;
12930
12931         /* No point in doing any of this if neither checksum nor GSO are
12932          * being requested for this frame.  We can rule out both by just
12933          * checking for CHECKSUM_PARTIAL
12934          */
12935         if (skb->ip_summed != CHECKSUM_PARTIAL)
12936                 return features;
12937
12938         /* We cannot support GSO if the MSS is going to be less than
12939          * 64 bytes.  If it is then we need to drop support for GSO.
12940          */
12941         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
12942                 features &= ~NETIF_F_GSO_MASK;
12943
12944         /* MACLEN can support at most 63 words */
12945         len = skb_network_header(skb) - skb->data;
12946         if (len & ~(63 * 2))
12947                 goto out_err;
12948
12949         /* IPLEN and EIPLEN can support at most 127 dwords */
12950         len = skb_transport_header(skb) - skb_network_header(skb);
12951         if (len & ~(127 * 4))
12952                 goto out_err;
12953
12954         if (skb->encapsulation) {
12955                 /* L4TUNLEN can support 127 words */
12956                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
12957                 if (len & ~(127 * 2))
12958                         goto out_err;
12959
12960                 /* IPLEN can support at most 127 dwords */
12961                 len = skb_inner_transport_header(skb) -
12962                       skb_inner_network_header(skb);
12963                 if (len & ~(127 * 4))
12964                         goto out_err;
12965         }
12966
12967         /* No need to validate L4LEN as TCP is the only protocol with a
12968          * a flexible value and we support all possible values supported
12969          * by TCP, which is at most 15 dwords
12970          */
12971
12972         return features;
12973 out_err:
12974         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
12975 }
12976
12977 /**
12978  * i40e_xdp_setup - add/remove an XDP program
12979  * @vsi: VSI to changed
12980  * @prog: XDP program
12981  * @extack: netlink extended ack
12982  **/
12983 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
12984                           struct netlink_ext_ack *extack)
12985 {
12986         int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
12987         struct i40e_pf *pf = vsi->back;
12988         struct bpf_prog *old_prog;
12989         bool need_reset;
12990         int i;
12991
12992         /* Don't allow frames that span over multiple buffers */
12993         if (frame_size > vsi->rx_buf_len) {
12994                 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP");
12995                 return -EINVAL;
12996         }
12997
12998         /* When turning XDP on->off/off->on we reset and rebuild the rings. */
12999         need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
13000
13001         if (need_reset)
13002                 i40e_prep_for_reset(pf);
13003
13004         old_prog = xchg(&vsi->xdp_prog, prog);
13005
13006         if (need_reset) {
13007                 if (!prog)
13008                         /* Wait until ndo_xsk_wakeup completes. */
13009                         synchronize_rcu();
13010                 i40e_reset_and_rebuild(pf, true, true);
13011         }
13012
13013         for (i = 0; i < vsi->num_queue_pairs; i++)
13014                 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
13015
13016         if (old_prog)
13017                 bpf_prog_put(old_prog);
13018
13019         /* Kick start the NAPI context if there is an AF_XDP socket open
13020          * on that queue id. This so that receiving will start.
13021          */
13022         if (need_reset && prog)
13023                 for (i = 0; i < vsi->num_queue_pairs; i++)
13024                         if (vsi->xdp_rings[i]->xsk_pool)
13025                                 (void)i40e_xsk_wakeup(vsi->netdev, i,
13026                                                       XDP_WAKEUP_RX);
13027
13028         return 0;
13029 }
13030
13031 /**
13032  * i40e_enter_busy_conf - Enters busy config state
13033  * @vsi: vsi
13034  *
13035  * Returns 0 on success, <0 for failure.
13036  **/
13037 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
13038 {
13039         struct i40e_pf *pf = vsi->back;
13040         int timeout = 50;
13041
13042         while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
13043                 timeout--;
13044                 if (!timeout)
13045                         return -EBUSY;
13046                 usleep_range(1000, 2000);
13047         }
13048
13049         return 0;
13050 }
13051
13052 /**
13053  * i40e_exit_busy_conf - Exits busy config state
13054  * @vsi: vsi
13055  **/
13056 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13057 {
13058         struct i40e_pf *pf = vsi->back;
13059
13060         clear_bit(__I40E_CONFIG_BUSY, pf->state);
13061 }
13062
13063 /**
13064  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13065  * @vsi: vsi
13066  * @queue_pair: queue pair
13067  **/
13068 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13069 {
13070         memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13071                sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13072         memset(&vsi->tx_rings[queue_pair]->stats, 0,
13073                sizeof(vsi->tx_rings[queue_pair]->stats));
13074         if (i40e_enabled_xdp_vsi(vsi)) {
13075                 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13076                        sizeof(vsi->xdp_rings[queue_pair]->stats));
13077         }
13078 }
13079
13080 /**
13081  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13082  * @vsi: vsi
13083  * @queue_pair: queue pair
13084  **/
13085 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13086 {
13087         i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
13088         if (i40e_enabled_xdp_vsi(vsi)) {
13089                 /* Make sure that in-progress ndo_xdp_xmit calls are
13090                  * completed.
13091                  */
13092                 synchronize_rcu();
13093                 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
13094         }
13095         i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13096 }
13097
13098 /**
13099  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13100  * @vsi: vsi
13101  * @queue_pair: queue pair
13102  * @enable: true for enable, false for disable
13103  **/
13104 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13105                                         bool enable)
13106 {
13107         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13108         struct i40e_q_vector *q_vector = rxr->q_vector;
13109
13110         if (!vsi->netdev)
13111                 return;
13112
13113         /* All rings in a qp belong to the same qvector. */
13114         if (q_vector->rx.ring || q_vector->tx.ring) {
13115                 if (enable)
13116                         napi_enable(&q_vector->napi);
13117                 else
13118                         napi_disable(&q_vector->napi);
13119         }
13120 }
13121
13122 /**
13123  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13124  * @vsi: vsi
13125  * @queue_pair: queue pair
13126  * @enable: true for enable, false for disable
13127  *
13128  * Returns 0 on success, <0 on failure.
13129  **/
13130 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13131                                         bool enable)
13132 {
13133         struct i40e_pf *pf = vsi->back;
13134         int pf_q, ret = 0;
13135
13136         pf_q = vsi->base_queue + queue_pair;
13137         ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
13138                                      false /*is xdp*/, enable);
13139         if (ret) {
13140                 dev_info(&pf->pdev->dev,
13141                          "VSI seid %d Tx ring %d %sable timeout\n",
13142                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13143                 return ret;
13144         }
13145
13146         i40e_control_rx_q(pf, pf_q, enable);
13147         ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13148         if (ret) {
13149                 dev_info(&pf->pdev->dev,
13150                          "VSI seid %d Rx ring %d %sable timeout\n",
13151                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13152                 return ret;
13153         }
13154
13155         /* Due to HW errata, on Rx disable only, the register can
13156          * indicate done before it really is. Needs 50ms to be sure
13157          */
13158         if (!enable)
13159                 mdelay(50);
13160
13161         if (!i40e_enabled_xdp_vsi(vsi))
13162                 return ret;
13163
13164         ret = i40e_control_wait_tx_q(vsi->seid, pf,
13165                                      pf_q + vsi->alloc_queue_pairs,
13166                                      true /*is xdp*/, enable);
13167         if (ret) {
13168                 dev_info(&pf->pdev->dev,
13169                          "VSI seid %d XDP Tx ring %d %sable timeout\n",
13170                          vsi->seid, pf_q, (enable ? "en" : "dis"));
13171         }
13172
13173         return ret;
13174 }
13175
13176 /**
13177  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13178  * @vsi: vsi
13179  * @queue_pair: queue_pair
13180  **/
13181 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13182 {
13183         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13184         struct i40e_pf *pf = vsi->back;
13185         struct i40e_hw *hw = &pf->hw;
13186
13187         /* All rings in a qp belong to the same qvector. */
13188         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
13189                 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
13190         else
13191                 i40e_irq_dynamic_enable_icr0(pf);
13192
13193         i40e_flush(hw);
13194 }
13195
13196 /**
13197  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13198  * @vsi: vsi
13199  * @queue_pair: queue_pair
13200  **/
13201 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13202 {
13203         struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13204         struct i40e_pf *pf = vsi->back;
13205         struct i40e_hw *hw = &pf->hw;
13206
13207         /* For simplicity, instead of removing the qp interrupt causes
13208          * from the interrupt linked list, we simply disable the interrupt, and
13209          * leave the list intact.
13210          *
13211          * All rings in a qp belong to the same qvector.
13212          */
13213         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
13214                 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13215
13216                 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13217                 i40e_flush(hw);
13218                 synchronize_irq(pf->msix_entries[intpf].vector);
13219         } else {
13220                 /* Legacy and MSI mode - this stops all interrupt handling */
13221                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13222                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13223                 i40e_flush(hw);
13224                 synchronize_irq(pf->pdev->irq);
13225         }
13226 }
13227
13228 /**
13229  * i40e_queue_pair_disable - Disables a queue pair
13230  * @vsi: vsi
13231  * @queue_pair: queue pair
13232  *
13233  * Returns 0 on success, <0 on failure.
13234  **/
13235 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13236 {
13237         int err;
13238
13239         err = i40e_enter_busy_conf(vsi);
13240         if (err)
13241                 return err;
13242
13243         i40e_queue_pair_disable_irq(vsi, queue_pair);
13244         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
13245         i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
13246         i40e_queue_pair_clean_rings(vsi, queue_pair);
13247         i40e_queue_pair_reset_stats(vsi, queue_pair);
13248
13249         return err;
13250 }
13251
13252 /**
13253  * i40e_queue_pair_enable - Enables a queue pair
13254  * @vsi: vsi
13255  * @queue_pair: queue pair
13256  *
13257  * Returns 0 on success, <0 on failure.
13258  **/
13259 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13260 {
13261         int err;
13262
13263         err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
13264         if (err)
13265                 return err;
13266
13267         if (i40e_enabled_xdp_vsi(vsi)) {
13268                 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
13269                 if (err)
13270                         return err;
13271         }
13272
13273         err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
13274         if (err)
13275                 return err;
13276
13277         err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
13278         i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
13279         i40e_queue_pair_enable_irq(vsi, queue_pair);
13280
13281         i40e_exit_busy_conf(vsi);
13282
13283         return err;
13284 }
13285
13286 /**
13287  * i40e_xdp - implements ndo_bpf for i40e
13288  * @dev: netdevice
13289  * @xdp: XDP command
13290  **/
13291 static int i40e_xdp(struct net_device *dev,
13292                     struct netdev_bpf *xdp)
13293 {
13294         struct i40e_netdev_priv *np = netdev_priv(dev);
13295         struct i40e_vsi *vsi = np->vsi;
13296
13297         if (vsi->type != I40E_VSI_MAIN)
13298                 return -EINVAL;
13299
13300         switch (xdp->command) {
13301         case XDP_SETUP_PROG:
13302                 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);
13303         case XDP_SETUP_XSK_POOL:
13304                 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13305                                            xdp->xsk.queue_id);
13306         default:
13307                 return -EINVAL;
13308         }
13309 }
13310
13311 static const struct net_device_ops i40e_netdev_ops = {
13312         .ndo_open               = i40e_open,
13313         .ndo_stop               = i40e_close,
13314         .ndo_start_xmit         = i40e_lan_xmit_frame,
13315         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
13316         .ndo_set_rx_mode        = i40e_set_rx_mode,
13317         .ndo_validate_addr      = eth_validate_addr,
13318         .ndo_set_mac_address    = i40e_set_mac,
13319         .ndo_change_mtu         = i40e_change_mtu,
13320         .ndo_eth_ioctl          = i40e_ioctl,
13321         .ndo_tx_timeout         = i40e_tx_timeout,
13322         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
13323         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
13324 #ifdef CONFIG_NET_POLL_CONTROLLER
13325         .ndo_poll_controller    = i40e_netpoll,
13326 #endif
13327         .ndo_setup_tc           = __i40e_setup_tc,
13328         .ndo_select_queue       = i40e_lan_select_queue,
13329         .ndo_set_features       = i40e_set_features,
13330         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
13331         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
13332         .ndo_get_vf_stats       = i40e_get_vf_stats,
13333         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
13334         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
13335         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
13336         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
13337         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
13338         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
13339         .ndo_fdb_add            = i40e_ndo_fdb_add,
13340         .ndo_features_check     = i40e_features_check,
13341         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
13342         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
13343         .ndo_bpf                = i40e_xdp,
13344         .ndo_xdp_xmit           = i40e_xdp_xmit,
13345         .ndo_xsk_wakeup         = i40e_xsk_wakeup,
13346         .ndo_dfwd_add_station   = i40e_fwd_add,
13347         .ndo_dfwd_del_station   = i40e_fwd_del,
13348 };
13349
13350 /**
13351  * i40e_config_netdev - Setup the netdev flags
13352  * @vsi: the VSI being configured
13353  *
13354  * Returns 0 on success, negative value on failure
13355  **/
13356 static int i40e_config_netdev(struct i40e_vsi *vsi)
13357 {
13358         struct i40e_pf *pf = vsi->back;
13359         struct i40e_hw *hw = &pf->hw;
13360         struct i40e_netdev_priv *np;
13361         struct net_device *netdev;
13362         u8 broadcast[ETH_ALEN];
13363         u8 mac_addr[ETH_ALEN];
13364         int etherdev_size;
13365         netdev_features_t hw_enc_features;
13366         netdev_features_t hw_features;
13367
13368         etherdev_size = sizeof(struct i40e_netdev_priv);
13369         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13370         if (!netdev)
13371                 return -ENOMEM;
13372
13373         vsi->netdev = netdev;
13374         np = netdev_priv(netdev);
13375         np->vsi = vsi;
13376
13377         hw_enc_features = NETIF_F_SG                    |
13378                           NETIF_F_IP_CSUM               |
13379                           NETIF_F_IPV6_CSUM             |
13380                           NETIF_F_HIGHDMA               |
13381                           NETIF_F_SOFT_FEATURES         |
13382                           NETIF_F_TSO                   |
13383                           NETIF_F_TSO_ECN               |
13384                           NETIF_F_TSO6                  |
13385                           NETIF_F_GSO_GRE               |
13386                           NETIF_F_GSO_GRE_CSUM          |
13387                           NETIF_F_GSO_PARTIAL           |
13388                           NETIF_F_GSO_IPXIP4            |
13389                           NETIF_F_GSO_IPXIP6            |
13390                           NETIF_F_GSO_UDP_TUNNEL        |
13391                           NETIF_F_GSO_UDP_TUNNEL_CSUM   |
13392                           NETIF_F_GSO_UDP_L4            |
13393                           NETIF_F_SCTP_CRC              |
13394                           NETIF_F_RXHASH                |
13395                           NETIF_F_RXCSUM                |
13396                           0;
13397
13398         if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
13399                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13400
13401         netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13402
13403         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13404
13405         netdev->hw_enc_features |= hw_enc_features;
13406
13407         /* record features VLANs can make use of */
13408         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13409
13410         /* enable macvlan offloads */
13411         netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13412
13413         hw_features = hw_enc_features           |
13414                       NETIF_F_HW_VLAN_CTAG_TX   |
13415                       NETIF_F_HW_VLAN_CTAG_RX;
13416
13417         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
13418                 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13419
13420         netdev->hw_features |= hw_features;
13421
13422         netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13423         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13424
13425         if (vsi->type == I40E_VSI_MAIN) {
13426                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13427                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
13428                 /* The following steps are necessary for two reasons. First,
13429                  * some older NVM configurations load a default MAC-VLAN
13430                  * filter that will accept any tagged packet, and we want to
13431                  * replace this with a normal filter. Additionally, it is
13432                  * possible our MAC address was provided by the platform using
13433                  * Open Firmware or similar.
13434                  *
13435                  * Thus, we need to remove the default filter and install one
13436                  * specific to the MAC address.
13437                  */
13438                 i40e_rm_default_mac_filter(vsi, mac_addr);
13439                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13440                 i40e_add_mac_filter(vsi, mac_addr);
13441                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13442         } else {
13443                 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13444                  * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13445                  * the end, which is 4 bytes long, so force truncation of the
13446                  * original name by IFNAMSIZ - 4
13447                  */
13448                 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
13449                          IFNAMSIZ - 4,
13450                          pf->vsi[pf->lan_vsi]->netdev->name);
13451                 eth_random_addr(mac_addr);
13452
13453                 spin_lock_bh(&vsi->mac_filter_hash_lock);
13454                 i40e_add_mac_filter(vsi, mac_addr);
13455                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13456         }
13457
13458         /* Add the broadcast filter so that we initially will receive
13459          * broadcast packets. Note that when a new VLAN is first added the
13460          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13461          * specific filters as part of transitioning into "vlan" operation.
13462          * When more VLANs are added, the driver will copy each existing MAC
13463          * filter and add it for the new VLAN.
13464          *
13465          * Broadcast filters are handled specially by
13466          * i40e_sync_filters_subtask, as the driver must to set the broadcast
13467          * promiscuous bit instead of adding this directly as a MAC/VLAN
13468          * filter. The subtask will update the correct broadcast promiscuous
13469          * bits as VLANs become active or inactive.
13470          */
13471         eth_broadcast_addr(broadcast);
13472         spin_lock_bh(&vsi->mac_filter_hash_lock);
13473         i40e_add_mac_filter(vsi, broadcast);
13474         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13475
13476         eth_hw_addr_set(netdev, mac_addr);
13477         ether_addr_copy(netdev->perm_addr, mac_addr);
13478
13479         /* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13480         netdev->neigh_priv_len = sizeof(u32) * 4;
13481
13482         netdev->priv_flags |= IFF_UNICAST_FLT;
13483         netdev->priv_flags |= IFF_SUPP_NOFCS;
13484         /* Setup netdev TC information */
13485         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13486
13487         netdev->netdev_ops = &i40e_netdev_ops;
13488         netdev->watchdog_timeo = 5 * HZ;
13489         i40e_set_ethtool_ops(netdev);
13490
13491         /* MTU range: 68 - 9706 */
13492         netdev->min_mtu = ETH_MIN_MTU;
13493         netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13494
13495         return 0;
13496 }
13497
13498 /**
13499  * i40e_vsi_delete - Delete a VSI from the switch
13500  * @vsi: the VSI being removed
13501  *
13502  * Returns 0 on success, negative value on failure
13503  **/
13504 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13505 {
13506         /* remove default VSI is not allowed */
13507         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13508                 return;
13509
13510         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13511 }
13512
13513 /**
13514  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13515  * @vsi: the VSI being queried
13516  *
13517  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13518  **/
13519 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13520 {
13521         struct i40e_veb *veb;
13522         struct i40e_pf *pf = vsi->back;
13523
13524         /* Uplink is not a bridge so default to VEB */
13525         if (vsi->veb_idx >= I40E_MAX_VEB)
13526                 return 1;
13527
13528         veb = pf->veb[vsi->veb_idx];
13529         if (!veb) {
13530                 dev_info(&pf->pdev->dev,
13531                          "There is no veb associated with the bridge\n");
13532                 return -ENOENT;
13533         }
13534
13535         /* Uplink is a bridge in VEPA mode */
13536         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13537                 return 0;
13538         } else {
13539                 /* Uplink is a bridge in VEB mode */
13540                 return 1;
13541         }
13542
13543         /* VEPA is now default bridge, so return 0 */
13544         return 0;
13545 }
13546
13547 /**
13548  * i40e_add_vsi - Add a VSI to the switch
13549  * @vsi: the VSI being configured
13550  *
13551  * This initializes a VSI context depending on the VSI type to be added and
13552  * passes it down to the add_vsi aq command.
13553  **/
13554 static int i40e_add_vsi(struct i40e_vsi *vsi)
13555 {
13556         int ret = -ENODEV;
13557         struct i40e_pf *pf = vsi->back;
13558         struct i40e_hw *hw = &pf->hw;
13559         struct i40e_vsi_context ctxt;
13560         struct i40e_mac_filter *f;
13561         struct hlist_node *h;
13562         int bkt;
13563
13564         u8 enabled_tc = 0x1; /* TC0 enabled */
13565         int f_count = 0;
13566
13567         memset(&ctxt, 0, sizeof(ctxt));
13568         switch (vsi->type) {
13569         case I40E_VSI_MAIN:
13570                 /* The PF's main VSI is already setup as part of the
13571                  * device initialization, so we'll not bother with
13572                  * the add_vsi call, but we will retrieve the current
13573                  * VSI context.
13574                  */
13575                 ctxt.seid = pf->main_vsi_seid;
13576                 ctxt.pf_num = pf->hw.pf_id;
13577                 ctxt.vf_num = 0;
13578                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13579                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13580                 if (ret) {
13581                         dev_info(&pf->pdev->dev,
13582                                  "couldn't get PF vsi config, err %s aq_err %s\n",
13583                                  i40e_stat_str(&pf->hw, ret),
13584                                  i40e_aq_str(&pf->hw,
13585                                              pf->hw.aq.asq_last_status));
13586                         return -ENOENT;
13587                 }
13588                 vsi->info = ctxt.info;
13589                 vsi->info.valid_sections = 0;
13590
13591                 vsi->seid = ctxt.seid;
13592                 vsi->id = ctxt.vsi_number;
13593
13594                 enabled_tc = i40e_pf_get_tc_map(pf);
13595
13596                 /* Source pruning is enabled by default, so the flag is
13597                  * negative logic - if it's set, we need to fiddle with
13598                  * the VSI to disable source pruning.
13599                  */
13600                 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
13601                         memset(&ctxt, 0, sizeof(ctxt));
13602                         ctxt.seid = pf->main_vsi_seid;
13603                         ctxt.pf_num = pf->hw.pf_id;
13604                         ctxt.vf_num = 0;
13605                         ctxt.info.valid_sections |=
13606                                      cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13607                         ctxt.info.switch_id =
13608                                    cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13609                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13610                         if (ret) {
13611                                 dev_info(&pf->pdev->dev,
13612                                          "update vsi failed, err %s aq_err %s\n",
13613                                          i40e_stat_str(&pf->hw, ret),
13614                                          i40e_aq_str(&pf->hw,
13615                                                      pf->hw.aq.asq_last_status));
13616                                 ret = -ENOENT;
13617                                 goto err;
13618                         }
13619                 }
13620
13621                 /* MFP mode setup queue map and update VSI */
13622                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
13623                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
13624                         memset(&ctxt, 0, sizeof(ctxt));
13625                         ctxt.seid = pf->main_vsi_seid;
13626                         ctxt.pf_num = pf->hw.pf_id;
13627                         ctxt.vf_num = 0;
13628                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13629                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13630                         if (ret) {
13631                                 dev_info(&pf->pdev->dev,
13632                                          "update vsi failed, err %s aq_err %s\n",
13633                                          i40e_stat_str(&pf->hw, ret),
13634                                          i40e_aq_str(&pf->hw,
13635                                                     pf->hw.aq.asq_last_status));
13636                                 ret = -ENOENT;
13637                                 goto err;
13638                         }
13639                         /* update the local VSI info queue map */
13640                         i40e_vsi_update_queue_map(vsi, &ctxt);
13641                         vsi->info.valid_sections = 0;
13642                 } else {
13643                         /* Default/Main VSI is only enabled for TC0
13644                          * reconfigure it to enable all TCs that are
13645                          * available on the port in SFP mode.
13646                          * For MFP case the iSCSI PF would use this
13647                          * flow to enable LAN+iSCSI TC.
13648                          */
13649                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
13650                         if (ret) {
13651                                 /* Single TC condition is not fatal,
13652                                  * message and continue
13653                                  */
13654                                 dev_info(&pf->pdev->dev,
13655                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
13656                                          enabled_tc,
13657                                          i40e_stat_str(&pf->hw, ret),
13658                                          i40e_aq_str(&pf->hw,
13659                                                     pf->hw.aq.asq_last_status));
13660                         }
13661                 }
13662                 break;
13663
13664         case I40E_VSI_FDIR:
13665                 ctxt.pf_num = hw->pf_id;
13666                 ctxt.vf_num = 0;
13667                 ctxt.uplink_seid = vsi->uplink_seid;
13668                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13669                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13670                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
13671                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
13672                         ctxt.info.valid_sections |=
13673                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13674                         ctxt.info.switch_id =
13675                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13676                 }
13677                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13678                 break;
13679
13680         case I40E_VSI_VMDQ2:
13681                 ctxt.pf_num = hw->pf_id;
13682                 ctxt.vf_num = 0;
13683                 ctxt.uplink_seid = vsi->uplink_seid;
13684                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13685                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
13686
13687                 /* This VSI is connected to VEB so the switch_id
13688                  * should be set to zero by default.
13689                  */
13690                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13691                         ctxt.info.valid_sections |=
13692                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13693                         ctxt.info.switch_id =
13694                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13695                 }
13696
13697                 /* Setup the VSI tx/rx queue map for TC0 only for now */
13698                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13699                 break;
13700
13701         case I40E_VSI_SRIOV:
13702                 ctxt.pf_num = hw->pf_id;
13703                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
13704                 ctxt.uplink_seid = vsi->uplink_seid;
13705                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13706                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
13707
13708                 /* This VSI is connected to VEB so the switch_id
13709                  * should be set to zero by default.
13710                  */
13711                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13712                         ctxt.info.valid_sections |=
13713                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13714                         ctxt.info.switch_id =
13715                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13716                 }
13717
13718                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
13719                         ctxt.info.valid_sections |=
13720                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
13721                         ctxt.info.queueing_opt_flags |=
13722                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
13723                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
13724                 }
13725
13726                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
13727                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
13728                 if (pf->vf[vsi->vf_id].spoofchk) {
13729                         ctxt.info.valid_sections |=
13730                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
13731                         ctxt.info.sec_flags |=
13732                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
13733                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
13734                 }
13735                 /* Setup the VSI tx/rx queue map for TC0 only for now */
13736                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13737                 break;
13738
13739         case I40E_VSI_IWARP:
13740                 /* send down message to iWARP */
13741                 break;
13742
13743         default:
13744                 return -ENODEV;
13745         }
13746
13747         if (vsi->type != I40E_VSI_MAIN) {
13748                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
13749                 if (ret) {
13750                         dev_info(&vsi->back->pdev->dev,
13751                                  "add vsi failed, err %s aq_err %s\n",
13752                                  i40e_stat_str(&pf->hw, ret),
13753                                  i40e_aq_str(&pf->hw,
13754                                              pf->hw.aq.asq_last_status));
13755                         ret = -ENOENT;
13756                         goto err;
13757                 }
13758                 vsi->info = ctxt.info;
13759                 vsi->info.valid_sections = 0;
13760                 vsi->seid = ctxt.seid;
13761                 vsi->id = ctxt.vsi_number;
13762         }
13763
13764         vsi->active_filters = 0;
13765         clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
13766         spin_lock_bh(&vsi->mac_filter_hash_lock);
13767         /* If macvlan filters already exist, force them to get loaded */
13768         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
13769                 f->state = I40E_FILTER_NEW;
13770                 f_count++;
13771         }
13772         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13773
13774         if (f_count) {
13775                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
13776                 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
13777         }
13778
13779         /* Update VSI BW information */
13780         ret = i40e_vsi_get_bw_info(vsi);
13781         if (ret) {
13782                 dev_info(&pf->pdev->dev,
13783                          "couldn't get vsi bw info, err %s aq_err %s\n",
13784                          i40e_stat_str(&pf->hw, ret),
13785                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13786                 /* VSI is already added so not tearing that up */
13787                 ret = 0;
13788         }
13789
13790 err:
13791         return ret;
13792 }
13793
13794 /**
13795  * i40e_vsi_release - Delete a VSI and free its resources
13796  * @vsi: the VSI being removed
13797  *
13798  * Returns 0 on success or < 0 on error
13799  **/
13800 int i40e_vsi_release(struct i40e_vsi *vsi)
13801 {
13802         struct i40e_mac_filter *f;
13803         struct hlist_node *h;
13804         struct i40e_veb *veb = NULL;
13805         struct i40e_pf *pf;
13806         u16 uplink_seid;
13807         int i, n, bkt;
13808
13809         pf = vsi->back;
13810
13811         /* release of a VEB-owner or last VSI is not allowed */
13812         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
13813                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
13814                          vsi->seid, vsi->uplink_seid);
13815                 return -ENODEV;
13816         }
13817         if (vsi == pf->vsi[pf->lan_vsi] &&
13818             !test_bit(__I40E_DOWN, pf->state)) {
13819                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
13820                 return -ENODEV;
13821         }
13822         set_bit(__I40E_VSI_RELEASING, vsi->state);
13823         uplink_seid = vsi->uplink_seid;
13824         if (vsi->type != I40E_VSI_SRIOV) {
13825                 if (vsi->netdev_registered) {
13826                         vsi->netdev_registered = false;
13827                         if (vsi->netdev) {
13828                                 /* results in a call to i40e_close() */
13829                                 unregister_netdev(vsi->netdev);
13830                         }
13831                 } else {
13832                         i40e_vsi_close(vsi);
13833                 }
13834                 i40e_vsi_disable_irq(vsi);
13835         }
13836
13837         spin_lock_bh(&vsi->mac_filter_hash_lock);
13838
13839         /* clear the sync flag on all filters */
13840         if (vsi->netdev) {
13841                 __dev_uc_unsync(vsi->netdev, NULL);
13842                 __dev_mc_unsync(vsi->netdev, NULL);
13843         }
13844
13845         /* make sure any remaining filters are marked for deletion */
13846         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
13847                 __i40e_del_filter(vsi, f);
13848
13849         spin_unlock_bh(&vsi->mac_filter_hash_lock);
13850
13851         i40e_sync_vsi_filters(vsi);
13852
13853         i40e_vsi_delete(vsi);
13854         i40e_vsi_free_q_vectors(vsi);
13855         if (vsi->netdev) {
13856                 free_netdev(vsi->netdev);
13857                 vsi->netdev = NULL;
13858         }
13859         i40e_vsi_clear_rings(vsi);
13860         i40e_vsi_clear(vsi);
13861
13862         /* If this was the last thing on the VEB, except for the
13863          * controlling VSI, remove the VEB, which puts the controlling
13864          * VSI onto the next level down in the switch.
13865          *
13866          * Well, okay, there's one more exception here: don't remove
13867          * the orphan VEBs yet.  We'll wait for an explicit remove request
13868          * from up the network stack.
13869          */
13870         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
13871                 if (pf->vsi[i] &&
13872                     pf->vsi[i]->uplink_seid == uplink_seid &&
13873                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
13874                         n++;      /* count the VSIs */
13875                 }
13876         }
13877         for (i = 0; i < I40E_MAX_VEB; i++) {
13878                 if (!pf->veb[i])
13879                         continue;
13880                 if (pf->veb[i]->uplink_seid == uplink_seid)
13881                         n++;     /* count the VEBs */
13882                 if (pf->veb[i]->seid == uplink_seid)
13883                         veb = pf->veb[i];
13884         }
13885         if (n == 0 && veb && veb->uplink_seid != 0)
13886                 i40e_veb_release(veb);
13887
13888         return 0;
13889 }
13890
13891 /**
13892  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
13893  * @vsi: ptr to the VSI
13894  *
13895  * This should only be called after i40e_vsi_mem_alloc() which allocates the
13896  * corresponding SW VSI structure and initializes num_queue_pairs for the
13897  * newly allocated VSI.
13898  *
13899  * Returns 0 on success or negative on failure
13900  **/
13901 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
13902 {
13903         int ret = -ENOENT;
13904         struct i40e_pf *pf = vsi->back;
13905
13906         if (vsi->q_vectors[0]) {
13907                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
13908                          vsi->seid);
13909                 return -EEXIST;
13910         }
13911
13912         if (vsi->base_vector) {
13913                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
13914                          vsi->seid, vsi->base_vector);
13915                 return -EEXIST;
13916         }
13917
13918         ret = i40e_vsi_alloc_q_vectors(vsi);
13919         if (ret) {
13920                 dev_info(&pf->pdev->dev,
13921                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
13922                          vsi->num_q_vectors, vsi->seid, ret);
13923                 vsi->num_q_vectors = 0;
13924                 goto vector_setup_out;
13925         }
13926
13927         /* In Legacy mode, we do not have to get any other vector since we
13928          * piggyback on the misc/ICR0 for queue interrupts.
13929         */
13930         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
13931                 return ret;
13932         if (vsi->num_q_vectors)
13933                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
13934                                                  vsi->num_q_vectors, vsi->idx);
13935         if (vsi->base_vector < 0) {
13936                 dev_info(&pf->pdev->dev,
13937                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
13938                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
13939                 i40e_vsi_free_q_vectors(vsi);
13940                 ret = -ENOENT;
13941                 goto vector_setup_out;
13942         }
13943
13944 vector_setup_out:
13945         return ret;
13946 }
13947
13948 /**
13949  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
13950  * @vsi: pointer to the vsi.
13951  *
13952  * This re-allocates a vsi's queue resources.
13953  *
13954  * Returns pointer to the successfully allocated and configured VSI sw struct
13955  * on success, otherwise returns NULL on failure.
13956  **/
13957 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
13958 {
13959         u16 alloc_queue_pairs;
13960         struct i40e_pf *pf;
13961         u8 enabled_tc;
13962         int ret;
13963
13964         if (!vsi)
13965                 return NULL;
13966
13967         pf = vsi->back;
13968
13969         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
13970         i40e_vsi_clear_rings(vsi);
13971
13972         i40e_vsi_free_arrays(vsi, false);
13973         i40e_set_num_rings_in_vsi(vsi);
13974         ret = i40e_vsi_alloc_arrays(vsi, false);
13975         if (ret)
13976                 goto err_vsi;
13977
13978         alloc_queue_pairs = vsi->alloc_queue_pairs *
13979                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
13980
13981         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
13982         if (ret < 0) {
13983                 dev_info(&pf->pdev->dev,
13984                          "failed to get tracking for %d queues for VSI %d err %d\n",
13985                          alloc_queue_pairs, vsi->seid, ret);
13986                 goto err_vsi;
13987         }
13988         vsi->base_queue = ret;
13989
13990         /* Update the FW view of the VSI. Force a reset of TC and queue
13991          * layout configurations.
13992          */
13993         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
13994         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
13995         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
13996         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
13997         if (vsi->type == I40E_VSI_MAIN)
13998                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
13999
14000         /* assign it some queues */
14001         ret = i40e_alloc_rings(vsi);
14002         if (ret)
14003                 goto err_rings;
14004
14005         /* map all of the rings to the q_vectors */
14006         i40e_vsi_map_rings_to_vectors(vsi);
14007         return vsi;
14008
14009 err_rings:
14010         i40e_vsi_free_q_vectors(vsi);
14011         if (vsi->netdev_registered) {
14012                 vsi->netdev_registered = false;
14013                 unregister_netdev(vsi->netdev);
14014                 free_netdev(vsi->netdev);
14015                 vsi->netdev = NULL;
14016         }
14017         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14018 err_vsi:
14019         i40e_vsi_clear(vsi);
14020         return NULL;
14021 }
14022
14023 /**
14024  * i40e_vsi_setup - Set up a VSI by a given type
14025  * @pf: board private structure
14026  * @type: VSI type
14027  * @uplink_seid: the switch element to link to
14028  * @param1: usage depends upon VSI type. For VF types, indicates VF id
14029  *
14030  * This allocates the sw VSI structure and its queue resources, then add a VSI
14031  * to the identified VEB.
14032  *
14033  * Returns pointer to the successfully allocated and configure VSI sw struct on
14034  * success, otherwise returns NULL on failure.
14035  **/
14036 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
14037                                 u16 uplink_seid, u32 param1)
14038 {
14039         struct i40e_vsi *vsi = NULL;
14040         struct i40e_veb *veb = NULL;
14041         u16 alloc_queue_pairs;
14042         int ret, i;
14043         int v_idx;
14044
14045         /* The requested uplink_seid must be either
14046          *     - the PF's port seid
14047          *              no VEB is needed because this is the PF
14048          *              or this is a Flow Director special case VSI
14049          *     - seid of an existing VEB
14050          *     - seid of a VSI that owns an existing VEB
14051          *     - seid of a VSI that doesn't own a VEB
14052          *              a new VEB is created and the VSI becomes the owner
14053          *     - seid of the PF VSI, which is what creates the first VEB
14054          *              this is a special case of the previous
14055          *
14056          * Find which uplink_seid we were given and create a new VEB if needed
14057          */
14058         for (i = 0; i < I40E_MAX_VEB; i++) {
14059                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
14060                         veb = pf->veb[i];
14061                         break;
14062                 }
14063         }
14064
14065         if (!veb && uplink_seid != pf->mac_seid) {
14066
14067                 for (i = 0; i < pf->num_alloc_vsi; i++) {
14068                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
14069                                 vsi = pf->vsi[i];
14070                                 break;
14071                         }
14072                 }
14073                 if (!vsi) {
14074                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14075                                  uplink_seid);
14076                         return NULL;
14077                 }
14078
14079                 if (vsi->uplink_seid == pf->mac_seid)
14080                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
14081                                              vsi->tc_config.enabled_tc);
14082                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14083                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
14084                                              vsi->tc_config.enabled_tc);
14085                 if (veb) {
14086                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
14087                                 dev_info(&vsi->back->pdev->dev,
14088                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
14089                                 return NULL;
14090                         }
14091                         /* We come up by default in VEPA mode if SRIOV is not
14092                          * already enabled, in which case we can't force VEPA
14093                          * mode.
14094                          */
14095                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
14096                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
14097                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
14098                         }
14099                         i40e_config_bridge_mode(veb);
14100                 }
14101                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
14102                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
14103                                 veb = pf->veb[i];
14104                 }
14105                 if (!veb) {
14106                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14107                         return NULL;
14108                 }
14109
14110                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14111                 uplink_seid = veb->seid;
14112         }
14113
14114         /* get vsi sw struct */
14115         v_idx = i40e_vsi_mem_alloc(pf, type);
14116         if (v_idx < 0)
14117                 goto err_alloc;
14118         vsi = pf->vsi[v_idx];
14119         if (!vsi)
14120                 goto err_alloc;
14121         vsi->type = type;
14122         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14123
14124         if (type == I40E_VSI_MAIN)
14125                 pf->lan_vsi = v_idx;
14126         else if (type == I40E_VSI_SRIOV)
14127                 vsi->vf_id = param1;
14128         /* assign it some queues */
14129         alloc_queue_pairs = vsi->alloc_queue_pairs *
14130                             (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14131
14132         ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14133         if (ret < 0) {
14134                 dev_info(&pf->pdev->dev,
14135                          "failed to get tracking for %d queues for VSI %d err=%d\n",
14136                          alloc_queue_pairs, vsi->seid, ret);
14137                 goto err_vsi;
14138         }
14139         vsi->base_queue = ret;
14140
14141         /* get a VSI from the hardware */
14142         vsi->uplink_seid = uplink_seid;
14143         ret = i40e_add_vsi(vsi);
14144         if (ret)
14145                 goto err_vsi;
14146
14147         switch (vsi->type) {
14148         /* setup the netdev if needed */
14149         case I40E_VSI_MAIN:
14150         case I40E_VSI_VMDQ2:
14151                 ret = i40e_config_netdev(vsi);
14152                 if (ret)
14153                         goto err_netdev;
14154                 ret = register_netdev(vsi->netdev);
14155                 if (ret)
14156                         goto err_netdev;
14157                 vsi->netdev_registered = true;
14158                 netif_carrier_off(vsi->netdev);
14159 #ifdef CONFIG_I40E_DCB
14160                 /* Setup DCB netlink interface */
14161                 i40e_dcbnl_setup(vsi);
14162 #endif /* CONFIG_I40E_DCB */
14163                 fallthrough;
14164         case I40E_VSI_FDIR:
14165                 /* set up vectors and rings if needed */
14166                 ret = i40e_vsi_setup_vectors(vsi);
14167                 if (ret)
14168                         goto err_msix;
14169
14170                 ret = i40e_alloc_rings(vsi);
14171                 if (ret)
14172                         goto err_rings;
14173
14174                 /* map all of the rings to the q_vectors */
14175                 i40e_vsi_map_rings_to_vectors(vsi);
14176
14177                 i40e_vsi_reset_stats(vsi);
14178                 break;
14179         default:
14180                 /* no netdev or rings for the other VSI types */
14181                 break;
14182         }
14183
14184         if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
14185             (vsi->type == I40E_VSI_VMDQ2)) {
14186                 ret = i40e_vsi_config_rss(vsi);
14187         }
14188         return vsi;
14189
14190 err_rings:
14191         i40e_vsi_free_q_vectors(vsi);
14192 err_msix:
14193         if (vsi->netdev_registered) {
14194                 vsi->netdev_registered = false;
14195                 unregister_netdev(vsi->netdev);
14196                 free_netdev(vsi->netdev);
14197                 vsi->netdev = NULL;
14198         }
14199 err_netdev:
14200         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14201 err_vsi:
14202         i40e_vsi_clear(vsi);
14203 err_alloc:
14204         return NULL;
14205 }
14206
14207 /**
14208  * i40e_veb_get_bw_info - Query VEB BW information
14209  * @veb: the veb to query
14210  *
14211  * Query the Tx scheduler BW configuration data for given VEB
14212  **/
14213 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14214 {
14215         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14216         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14217         struct i40e_pf *pf = veb->pf;
14218         struct i40e_hw *hw = &pf->hw;
14219         u32 tc_bw_max;
14220         int ret = 0;
14221         int i;
14222
14223         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
14224                                                   &bw_data, NULL);
14225         if (ret) {
14226                 dev_info(&pf->pdev->dev,
14227                          "query veb bw config failed, err %s aq_err %s\n",
14228                          i40e_stat_str(&pf->hw, ret),
14229                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14230                 goto out;
14231         }
14232
14233         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
14234                                                    &ets_data, NULL);
14235         if (ret) {
14236                 dev_info(&pf->pdev->dev,
14237                          "query veb bw ets config failed, err %s aq_err %s\n",
14238                          i40e_stat_str(&pf->hw, ret),
14239                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14240                 goto out;
14241         }
14242
14243         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14244         veb->bw_max_quanta = ets_data.tc_bw_max;
14245         veb->is_abs_credits = bw_data.absolute_credits_enable;
14246         veb->enabled_tc = ets_data.tc_valid_bits;
14247         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14248                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14249         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14250                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14251                 veb->bw_tc_limit_credits[i] =
14252                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
14253                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14254         }
14255
14256 out:
14257         return ret;
14258 }
14259
14260 /**
14261  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14262  * @pf: board private structure
14263  *
14264  * On error: returns error code (negative)
14265  * On success: returns vsi index in PF (positive)
14266  **/
14267 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14268 {
14269         int ret = -ENOENT;
14270         struct i40e_veb *veb;
14271         int i;
14272
14273         /* Need to protect the allocation of switch elements at the PF level */
14274         mutex_lock(&pf->switch_mutex);
14275
14276         /* VEB list may be fragmented if VEB creation/destruction has
14277          * been happening.  We can afford to do a quick scan to look
14278          * for any free slots in the list.
14279          *
14280          * find next empty veb slot, looping back around if necessary
14281          */
14282         i = 0;
14283         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14284                 i++;
14285         if (i >= I40E_MAX_VEB) {
14286                 ret = -ENOMEM;
14287                 goto err_alloc_veb;  /* out of VEB slots! */
14288         }
14289
14290         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14291         if (!veb) {
14292                 ret = -ENOMEM;
14293                 goto err_alloc_veb;
14294         }
14295         veb->pf = pf;
14296         veb->idx = i;
14297         veb->enabled_tc = 1;
14298
14299         pf->veb[i] = veb;
14300         ret = i;
14301 err_alloc_veb:
14302         mutex_unlock(&pf->switch_mutex);
14303         return ret;
14304 }
14305
14306 /**
14307  * i40e_switch_branch_release - Delete a branch of the switch tree
14308  * @branch: where to start deleting
14309  *
14310  * This uses recursion to find the tips of the branch to be
14311  * removed, deleting until we get back to and can delete this VEB.
14312  **/
14313 static void i40e_switch_branch_release(struct i40e_veb *branch)
14314 {
14315         struct i40e_pf *pf = branch->pf;
14316         u16 branch_seid = branch->seid;
14317         u16 veb_idx = branch->idx;
14318         int i;
14319
14320         /* release any VEBs on this VEB - RECURSION */
14321         for (i = 0; i < I40E_MAX_VEB; i++) {
14322                 if (!pf->veb[i])
14323                         continue;
14324                 if (pf->veb[i]->uplink_seid == branch->seid)
14325                         i40e_switch_branch_release(pf->veb[i]);
14326         }
14327
14328         /* Release the VSIs on this VEB, but not the owner VSI.
14329          *
14330          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14331          *       the VEB itself, so don't use (*branch) after this loop.
14332          */
14333         for (i = 0; i < pf->num_alloc_vsi; i++) {
14334                 if (!pf->vsi[i])
14335                         continue;
14336                 if (pf->vsi[i]->uplink_seid == branch_seid &&
14337                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
14338                         i40e_vsi_release(pf->vsi[i]);
14339                 }
14340         }
14341
14342         /* There's one corner case where the VEB might not have been
14343          * removed, so double check it here and remove it if needed.
14344          * This case happens if the veb was created from the debugfs
14345          * commands and no VSIs were added to it.
14346          */
14347         if (pf->veb[veb_idx])
14348                 i40e_veb_release(pf->veb[veb_idx]);
14349 }
14350
14351 /**
14352  * i40e_veb_clear - remove veb struct
14353  * @veb: the veb to remove
14354  **/
14355 static void i40e_veb_clear(struct i40e_veb *veb)
14356 {
14357         if (!veb)
14358                 return;
14359
14360         if (veb->pf) {
14361                 struct i40e_pf *pf = veb->pf;
14362
14363                 mutex_lock(&pf->switch_mutex);
14364                 if (pf->veb[veb->idx] == veb)
14365                         pf->veb[veb->idx] = NULL;
14366                 mutex_unlock(&pf->switch_mutex);
14367         }
14368
14369         kfree(veb);
14370 }
14371
14372 /**
14373  * i40e_veb_release - Delete a VEB and free its resources
14374  * @veb: the VEB being removed
14375  **/
14376 void i40e_veb_release(struct i40e_veb *veb)
14377 {
14378         struct i40e_vsi *vsi = NULL;
14379         struct i40e_pf *pf;
14380         int i, n = 0;
14381
14382         pf = veb->pf;
14383
14384         /* find the remaining VSI and check for extras */
14385         for (i = 0; i < pf->num_alloc_vsi; i++) {
14386                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
14387                         n++;
14388                         vsi = pf->vsi[i];
14389                 }
14390         }
14391         if (n != 1) {
14392                 dev_info(&pf->pdev->dev,
14393                          "can't remove VEB %d with %d VSIs left\n",
14394                          veb->seid, n);
14395                 return;
14396         }
14397
14398         /* move the remaining VSI to uplink veb */
14399         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14400         if (veb->uplink_seid) {
14401                 vsi->uplink_seid = veb->uplink_seid;
14402                 if (veb->uplink_seid == pf->mac_seid)
14403                         vsi->veb_idx = I40E_NO_VEB;
14404                 else
14405                         vsi->veb_idx = veb->veb_idx;
14406         } else {
14407                 /* floating VEB */
14408                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
14409                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
14410         }
14411
14412         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14413         i40e_veb_clear(veb);
14414 }
14415
14416 /**
14417  * i40e_add_veb - create the VEB in the switch
14418  * @veb: the VEB to be instantiated
14419  * @vsi: the controlling VSI
14420  **/
14421 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14422 {
14423         struct i40e_pf *pf = veb->pf;
14424         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
14425         int ret;
14426
14427         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
14428                               veb->enabled_tc, false,
14429                               &veb->seid, enable_stats, NULL);
14430
14431         /* get a VEB from the hardware */
14432         if (ret) {
14433                 dev_info(&pf->pdev->dev,
14434                          "couldn't add VEB, err %s aq_err %s\n",
14435                          i40e_stat_str(&pf->hw, ret),
14436                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14437                 return -EPERM;
14438         }
14439
14440         /* get statistics counter */
14441         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14442                                          &veb->stats_idx, NULL, NULL, NULL);
14443         if (ret) {
14444                 dev_info(&pf->pdev->dev,
14445                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
14446                          i40e_stat_str(&pf->hw, ret),
14447                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14448                 return -EPERM;
14449         }
14450         ret = i40e_veb_get_bw_info(veb);
14451         if (ret) {
14452                 dev_info(&pf->pdev->dev,
14453                          "couldn't get VEB bw info, err %s aq_err %s\n",
14454                          i40e_stat_str(&pf->hw, ret),
14455                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14456                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14457                 return -ENOENT;
14458         }
14459
14460         vsi->uplink_seid = veb->seid;
14461         vsi->veb_idx = veb->idx;
14462         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14463
14464         return 0;
14465 }
14466
14467 /**
14468  * i40e_veb_setup - Set up a VEB
14469  * @pf: board private structure
14470  * @flags: VEB setup flags
14471  * @uplink_seid: the switch element to link to
14472  * @vsi_seid: the initial VSI seid
14473  * @enabled_tc: Enabled TC bit-map
14474  *
14475  * This allocates the sw VEB structure and links it into the switch
14476  * It is possible and legal for this to be a duplicate of an already
14477  * existing VEB.  It is also possible for both uplink and vsi seids
14478  * to be zero, in order to create a floating VEB.
14479  *
14480  * Returns pointer to the successfully allocated VEB sw struct on
14481  * success, otherwise returns NULL on failure.
14482  **/
14483 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14484                                 u16 uplink_seid, u16 vsi_seid,
14485                                 u8 enabled_tc)
14486 {
14487         struct i40e_veb *veb, *uplink_veb = NULL;
14488         int vsi_idx, veb_idx;
14489         int ret;
14490
14491         /* if one seid is 0, the other must be 0 to create a floating relay */
14492         if ((uplink_seid == 0 || vsi_seid == 0) &&
14493             (uplink_seid + vsi_seid != 0)) {
14494                 dev_info(&pf->pdev->dev,
14495                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
14496                          uplink_seid, vsi_seid);
14497                 return NULL;
14498         }
14499
14500         /* make sure there is such a vsi and uplink */
14501         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
14502                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
14503                         break;
14504         if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) {
14505                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
14506                          vsi_seid);
14507                 return NULL;
14508         }
14509
14510         if (uplink_seid && uplink_seid != pf->mac_seid) {
14511                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
14512                         if (pf->veb[veb_idx] &&
14513                             pf->veb[veb_idx]->seid == uplink_seid) {
14514                                 uplink_veb = pf->veb[veb_idx];
14515                                 break;
14516                         }
14517                 }
14518                 if (!uplink_veb) {
14519                         dev_info(&pf->pdev->dev,
14520                                  "uplink seid %d not found\n", uplink_seid);
14521                         return NULL;
14522                 }
14523         }
14524
14525         /* get veb sw struct */
14526         veb_idx = i40e_veb_mem_alloc(pf);
14527         if (veb_idx < 0)
14528                 goto err_alloc;
14529         veb = pf->veb[veb_idx];
14530         veb->flags = flags;
14531         veb->uplink_seid = uplink_seid;
14532         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
14533         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14534
14535         /* create the VEB in the switch */
14536         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
14537         if (ret)
14538                 goto err_veb;
14539         if (vsi_idx == pf->lan_vsi)
14540                 pf->lan_veb = veb->idx;
14541
14542         return veb;
14543
14544 err_veb:
14545         i40e_veb_clear(veb);
14546 err_alloc:
14547         return NULL;
14548 }
14549
14550 /**
14551  * i40e_setup_pf_switch_element - set PF vars based on switch type
14552  * @pf: board private structure
14553  * @ele: element we are building info from
14554  * @num_reported: total number of elements
14555  * @printconfig: should we print the contents
14556  *
14557  * helper function to assist in extracting a few useful SEID values.
14558  **/
14559 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14560                                 struct i40e_aqc_switch_config_element_resp *ele,
14561                                 u16 num_reported, bool printconfig)
14562 {
14563         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14564         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14565         u8 element_type = ele->element_type;
14566         u16 seid = le16_to_cpu(ele->seid);
14567
14568         if (printconfig)
14569                 dev_info(&pf->pdev->dev,
14570                          "type=%d seid=%d uplink=%d downlink=%d\n",
14571                          element_type, seid, uplink_seid, downlink_seid);
14572
14573         switch (element_type) {
14574         case I40E_SWITCH_ELEMENT_TYPE_MAC:
14575                 pf->mac_seid = seid;
14576                 break;
14577         case I40E_SWITCH_ELEMENT_TYPE_VEB:
14578                 /* Main VEB? */
14579                 if (uplink_seid != pf->mac_seid)
14580                         break;
14581                 if (pf->lan_veb >= I40E_MAX_VEB) {
14582                         int v;
14583
14584                         /* find existing or else empty VEB */
14585                         for (v = 0; v < I40E_MAX_VEB; v++) {
14586                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
14587                                         pf->lan_veb = v;
14588                                         break;
14589                                 }
14590                         }
14591                         if (pf->lan_veb >= I40E_MAX_VEB) {
14592                                 v = i40e_veb_mem_alloc(pf);
14593                                 if (v < 0)
14594                                         break;
14595                                 pf->lan_veb = v;
14596                         }
14597                 }
14598                 if (pf->lan_veb >= I40E_MAX_VEB)
14599                         break;
14600
14601                 pf->veb[pf->lan_veb]->seid = seid;
14602                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14603                 pf->veb[pf->lan_veb]->pf = pf;
14604                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
14605                 break;
14606         case I40E_SWITCH_ELEMENT_TYPE_VSI:
14607                 if (num_reported != 1)
14608                         break;
14609                 /* This is immediately after a reset so we can assume this is
14610                  * the PF's VSI
14611                  */
14612                 pf->mac_seid = uplink_seid;
14613                 pf->pf_seid = downlink_seid;
14614                 pf->main_vsi_seid = seid;
14615                 if (printconfig)
14616                         dev_info(&pf->pdev->dev,
14617                                  "pf_seid=%d main_vsi_seid=%d\n",
14618                                  pf->pf_seid, pf->main_vsi_seid);
14619                 break;
14620         case I40E_SWITCH_ELEMENT_TYPE_PF:
14621         case I40E_SWITCH_ELEMENT_TYPE_VF:
14622         case I40E_SWITCH_ELEMENT_TYPE_EMP:
14623         case I40E_SWITCH_ELEMENT_TYPE_BMC:
14624         case I40E_SWITCH_ELEMENT_TYPE_PE:
14625         case I40E_SWITCH_ELEMENT_TYPE_PA:
14626                 /* ignore these for now */
14627                 break;
14628         default:
14629                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14630                          element_type, seid);
14631                 break;
14632         }
14633 }
14634
14635 /**
14636  * i40e_fetch_switch_configuration - Get switch config from firmware
14637  * @pf: board private structure
14638  * @printconfig: should we print the contents
14639  *
14640  * Get the current switch configuration from the device and
14641  * extract a few useful SEID values.
14642  **/
14643 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14644 {
14645         struct i40e_aqc_get_switch_config_resp *sw_config;
14646         u16 next_seid = 0;
14647         int ret = 0;
14648         u8 *aq_buf;
14649         int i;
14650
14651         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14652         if (!aq_buf)
14653                 return -ENOMEM;
14654
14655         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14656         do {
14657                 u16 num_reported, num_total;
14658
14659                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14660                                                 I40E_AQ_LARGE_BUF,
14661                                                 &next_seid, NULL);
14662                 if (ret) {
14663                         dev_info(&pf->pdev->dev,
14664                                  "get switch config failed err %s aq_err %s\n",
14665                                  i40e_stat_str(&pf->hw, ret),
14666                                  i40e_aq_str(&pf->hw,
14667                                              pf->hw.aq.asq_last_status));
14668                         kfree(aq_buf);
14669                         return -ENOENT;
14670                 }
14671
14672                 num_reported = le16_to_cpu(sw_config->header.num_reported);
14673                 num_total = le16_to_cpu(sw_config->header.num_total);
14674
14675                 if (printconfig)
14676                         dev_info(&pf->pdev->dev,
14677                                  "header: %d reported %d total\n",
14678                                  num_reported, num_total);
14679
14680                 for (i = 0; i < num_reported; i++) {
14681                         struct i40e_aqc_switch_config_element_resp *ele =
14682                                 &sw_config->element[i];
14683
14684                         i40e_setup_pf_switch_element(pf, ele, num_reported,
14685                                                      printconfig);
14686                 }
14687         } while (next_seid != 0);
14688
14689         kfree(aq_buf);
14690         return ret;
14691 }
14692
14693 /**
14694  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
14695  * @pf: board private structure
14696  * @reinit: if the Main VSI needs to re-initialized.
14697  * @lock_acquired: indicates whether or not the lock has been acquired
14698  *
14699  * Returns 0 on success, negative value on failure
14700  **/
14701 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
14702 {
14703         u16 flags = 0;
14704         int ret;
14705
14706         /* find out what's out there already */
14707         ret = i40e_fetch_switch_configuration(pf, false);
14708         if (ret) {
14709                 dev_info(&pf->pdev->dev,
14710                          "couldn't fetch switch config, err %s aq_err %s\n",
14711                          i40e_stat_str(&pf->hw, ret),
14712                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14713                 return ret;
14714         }
14715         i40e_pf_reset_stats(pf);
14716
14717         /* set the switch config bit for the whole device to
14718          * support limited promisc or true promisc
14719          * when user requests promisc. The default is limited
14720          * promisc.
14721         */
14722
14723         if ((pf->hw.pf_id == 0) &&
14724             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
14725                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14726                 pf->last_sw_conf_flags = flags;
14727         }
14728
14729         if (pf->hw.pf_id == 0) {
14730                 u16 valid_flags;
14731
14732                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14733                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
14734                                                 NULL);
14735                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
14736                         dev_info(&pf->pdev->dev,
14737                                  "couldn't set switch config bits, err %s aq_err %s\n",
14738                                  i40e_stat_str(&pf->hw, ret),
14739                                  i40e_aq_str(&pf->hw,
14740                                              pf->hw.aq.asq_last_status));
14741                         /* not a fatal problem, just keep going */
14742                 }
14743                 pf->last_sw_conf_valid_flags = valid_flags;
14744         }
14745
14746         /* first time setup */
14747         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
14748                 struct i40e_vsi *vsi = NULL;
14749                 u16 uplink_seid;
14750
14751                 /* Set up the PF VSI associated with the PF's main VSI
14752                  * that is already in the HW switch
14753                  */
14754                 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
14755                         uplink_seid = pf->veb[pf->lan_veb]->seid;
14756                 else
14757                         uplink_seid = pf->mac_seid;
14758                 if (pf->lan_vsi == I40E_NO_VSI)
14759                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
14760                 else if (reinit)
14761                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
14762                 if (!vsi) {
14763                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
14764                         i40e_cloud_filter_exit(pf);
14765                         i40e_fdir_teardown(pf);
14766                         return -EAGAIN;
14767                 }
14768         } else {
14769                 /* force a reset of TC and queue layout configurations */
14770                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14771
14772                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14773                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14774                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14775         }
14776         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
14777
14778         i40e_fdir_sb_setup(pf);
14779
14780         /* Setup static PF queue filter control settings */
14781         ret = i40e_setup_pf_filter_control(pf);
14782         if (ret) {
14783                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
14784                          ret);
14785                 /* Failure here should not stop continuing other steps */
14786         }
14787
14788         /* enable RSS in the HW, even for only one queue, as the stack can use
14789          * the hash
14790          */
14791         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
14792                 i40e_pf_config_rss(pf);
14793
14794         /* fill in link information and enable LSE reporting */
14795         i40e_link_event(pf);
14796
14797         /* Initialize user-specific link properties */
14798         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
14799                                   I40E_AQ_AN_COMPLETED) ? true : false);
14800
14801         i40e_ptp_init(pf);
14802
14803         if (!lock_acquired)
14804                 rtnl_lock();
14805
14806         /* repopulate tunnel port filters */
14807         udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev);
14808
14809         if (!lock_acquired)
14810                 rtnl_unlock();
14811
14812         return ret;
14813 }
14814
14815 /**
14816  * i40e_determine_queue_usage - Work out queue distribution
14817  * @pf: board private structure
14818  **/
14819 static void i40e_determine_queue_usage(struct i40e_pf *pf)
14820 {
14821         int queues_left;
14822         int q_max;
14823
14824         pf->num_lan_qps = 0;
14825
14826         /* Find the max queues to be put into basic use.  We'll always be
14827          * using TC0, whether or not DCB is running, and TC0 will get the
14828          * big RSS set.
14829          */
14830         queues_left = pf->hw.func_caps.num_tx_qp;
14831
14832         if ((queues_left == 1) ||
14833             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
14834                 /* one qp for PF, no queues for anything else */
14835                 queues_left = 0;
14836                 pf->alloc_rss_size = pf->num_lan_qps = 1;
14837
14838                 /* make sure all the fancies are disabled */
14839                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
14840                                I40E_FLAG_IWARP_ENABLED  |
14841                                I40E_FLAG_FD_SB_ENABLED  |
14842                                I40E_FLAG_FD_ATR_ENABLED |
14843                                I40E_FLAG_DCB_CAPABLE    |
14844                                I40E_FLAG_DCB_ENABLED    |
14845                                I40E_FLAG_SRIOV_ENABLED  |
14846                                I40E_FLAG_VMDQ_ENABLED);
14847                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14848         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
14849                                   I40E_FLAG_FD_SB_ENABLED |
14850                                   I40E_FLAG_FD_ATR_ENABLED |
14851                                   I40E_FLAG_DCB_CAPABLE))) {
14852                 /* one qp for PF */
14853                 pf->alloc_rss_size = pf->num_lan_qps = 1;
14854                 queues_left -= pf->num_lan_qps;
14855
14856                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
14857                                I40E_FLAG_IWARP_ENABLED  |
14858                                I40E_FLAG_FD_SB_ENABLED  |
14859                                I40E_FLAG_FD_ATR_ENABLED |
14860                                I40E_FLAG_DCB_ENABLED    |
14861                                I40E_FLAG_VMDQ_ENABLED);
14862                 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14863         } else {
14864                 /* Not enough queues for all TCs */
14865                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
14866                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
14867                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
14868                                         I40E_FLAG_DCB_ENABLED);
14869                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
14870                 }
14871
14872                 /* limit lan qps to the smaller of qps, cpus or msix */
14873                 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
14874                 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
14875                 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
14876                 pf->num_lan_qps = q_max;
14877
14878                 queues_left -= pf->num_lan_qps;
14879         }
14880
14881         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14882                 if (queues_left > 1) {
14883                         queues_left -= 1; /* save 1 queue for FD */
14884                 } else {
14885                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
14886                         pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14887                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
14888                 }
14889         }
14890
14891         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
14892             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
14893                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
14894                                         (queues_left / pf->num_vf_qps));
14895                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
14896         }
14897
14898         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
14899             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
14900                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
14901                                           (queues_left / pf->num_vmdq_qps));
14902                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
14903         }
14904
14905         pf->queues_left = queues_left;
14906         dev_dbg(&pf->pdev->dev,
14907                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
14908                 pf->hw.func_caps.num_tx_qp,
14909                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
14910                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
14911                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
14912                 queues_left);
14913 }
14914
14915 /**
14916  * i40e_setup_pf_filter_control - Setup PF static filter control
14917  * @pf: PF to be setup
14918  *
14919  * i40e_setup_pf_filter_control sets up a PF's initial filter control
14920  * settings. If PE/FCoE are enabled then it will also set the per PF
14921  * based filter sizes required for them. It also enables Flow director,
14922  * ethertype and macvlan type filter settings for the pf.
14923  *
14924  * Returns 0 on success, negative on failure
14925  **/
14926 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
14927 {
14928         struct i40e_filter_control_settings *settings = &pf->filter_settings;
14929
14930         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
14931
14932         /* Flow Director is enabled */
14933         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
14934                 settings->enable_fdir = true;
14935
14936         /* Ethtype and MACVLAN filters enabled for PF */
14937         settings->enable_ethtype = true;
14938         settings->enable_macvlan = true;
14939
14940         if (i40e_set_filter_control(&pf->hw, settings))
14941                 return -ENOENT;
14942
14943         return 0;
14944 }
14945
14946 #define INFO_STRING_LEN 255
14947 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
14948 static void i40e_print_features(struct i40e_pf *pf)
14949 {
14950         struct i40e_hw *hw = &pf->hw;
14951         char *buf;
14952         int i;
14953
14954         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
14955         if (!buf)
14956                 return;
14957
14958         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
14959 #ifdef CONFIG_PCI_IOV
14960         i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
14961 #endif
14962         i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
14963                       pf->hw.func_caps.num_vsis,
14964                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
14965         if (pf->flags & I40E_FLAG_RSS_ENABLED)
14966                 i += scnprintf(&buf[i], REMAIN(i), " RSS");
14967         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
14968                 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
14969         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14970                 i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
14971                 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
14972         }
14973         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
14974                 i += scnprintf(&buf[i], REMAIN(i), " DCB");
14975         i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
14976         i += scnprintf(&buf[i], REMAIN(i), " Geneve");
14977         if (pf->flags & I40E_FLAG_PTP)
14978                 i += scnprintf(&buf[i], REMAIN(i), " PTP");
14979         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
14980                 i += scnprintf(&buf[i], REMAIN(i), " VEB");
14981         else
14982                 i += scnprintf(&buf[i], REMAIN(i), " VEPA");
14983
14984         dev_info(&pf->pdev->dev, "%s\n", buf);
14985         kfree(buf);
14986         WARN_ON(i > INFO_STRING_LEN);
14987 }
14988
14989 /**
14990  * i40e_get_platform_mac_addr - get platform-specific MAC address
14991  * @pdev: PCI device information struct
14992  * @pf: board private structure
14993  *
14994  * Look up the MAC address for the device. First we'll try
14995  * eth_platform_get_mac_address, which will check Open Firmware, or arch
14996  * specific fallback. Otherwise, we'll default to the stored value in
14997  * firmware.
14998  **/
14999 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
15000 {
15001         if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
15002                 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
15003 }
15004
15005 /**
15006  * i40e_set_fec_in_flags - helper function for setting FEC options in flags
15007  * @fec_cfg: FEC option to set in flags
15008  * @flags: ptr to flags in which we set FEC option
15009  **/
15010 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags)
15011 {
15012         if (fec_cfg & I40E_AQ_SET_FEC_AUTO)
15013                 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC;
15014         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
15015             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
15016                 *flags |= I40E_FLAG_RS_FEC;
15017                 *flags &= ~I40E_FLAG_BASE_R_FEC;
15018         }
15019         if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
15020             (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
15021                 *flags |= I40E_FLAG_BASE_R_FEC;
15022                 *flags &= ~I40E_FLAG_RS_FEC;
15023         }
15024         if (fec_cfg == 0)
15025                 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC);
15026 }
15027
15028 /**
15029  * i40e_check_recovery_mode - check if we are running transition firmware
15030  * @pf: board private structure
15031  *
15032  * Check registers indicating the firmware runs in recovery mode. Sets the
15033  * appropriate driver state.
15034  *
15035  * Returns true if the recovery mode was detected, false otherwise
15036  **/
15037 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
15038 {
15039         u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
15040
15041         if (val & I40E_GL_FWSTS_FWS1B_MASK) {
15042                 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
15043                 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
15044                 set_bit(__I40E_RECOVERY_MODE, pf->state);
15045
15046                 return true;
15047         }
15048         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15049                 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
15050
15051         return false;
15052 }
15053
15054 /**
15055  * i40e_pf_loop_reset - perform reset in a loop.
15056  * @pf: board private structure
15057  *
15058  * This function is useful when a NIC is about to enter recovery mode.
15059  * When a NIC's internal data structures are corrupted the NIC's
15060  * firmware is going to enter recovery mode.
15061  * Right after a POR it takes about 7 minutes for firmware to enter
15062  * recovery mode. Until that time a NIC is in some kind of intermediate
15063  * state. After that time period the NIC almost surely enters
15064  * recovery mode. The only way for a driver to detect intermediate
15065  * state is to issue a series of pf-resets and check a return value.
15066  * If a PF reset returns success then the firmware could be in recovery
15067  * mode so the caller of this code needs to check for recovery mode
15068  * if this function returns success. There is a little chance that
15069  * firmware will hang in intermediate state forever.
15070  * Since waiting 7 minutes is quite a lot of time this function waits
15071  * 10 seconds and then gives up by returning an error.
15072  *
15073  * Return 0 on success, negative on failure.
15074  **/
15075 static i40e_status i40e_pf_loop_reset(struct i40e_pf *pf)
15076 {
15077         /* wait max 10 seconds for PF reset to succeed */
15078         const unsigned long time_end = jiffies + 10 * HZ;
15079
15080         struct i40e_hw *hw = &pf->hw;
15081         i40e_status ret;
15082
15083         ret = i40e_pf_reset(hw);
15084         while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) {
15085                 usleep_range(10000, 20000);
15086                 ret = i40e_pf_reset(hw);
15087         }
15088
15089         if (ret == I40E_SUCCESS)
15090                 pf->pfr_count++;
15091         else
15092                 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15093
15094         return ret;
15095 }
15096
15097 /**
15098  * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15099  * @pf: board private structure
15100  *
15101  * Check FW registers to determine if FW issued unexpected EMP Reset.
15102  * Every time when unexpected EMP Reset occurs the FW increments
15103  * a counter of unexpected EMP Resets. When the counter reaches 10
15104  * the FW should enter the Recovery mode
15105  *
15106  * Returns true if FW issued unexpected EMP Reset
15107  **/
15108 static bool i40e_check_fw_empr(struct i40e_pf *pf)
15109 {
15110         const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15111                            I40E_GL_FWSTS_FWS1B_MASK;
15112         return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15113                (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15114 }
15115
15116 /**
15117  * i40e_handle_resets - handle EMP resets and PF resets
15118  * @pf: board private structure
15119  *
15120  * Handle both EMP resets and PF resets and conclude whether there are
15121  * any issues regarding these resets. If there are any issues then
15122  * generate log entry.
15123  *
15124  * Return 0 if NIC is healthy or negative value when there are issues
15125  * with resets
15126  **/
15127 static i40e_status i40e_handle_resets(struct i40e_pf *pf)
15128 {
15129         const i40e_status pfr = i40e_pf_loop_reset(pf);
15130         const bool is_empr = i40e_check_fw_empr(pf);
15131
15132         if (is_empr || pfr != I40E_SUCCESS)
15133                 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15134
15135         return is_empr ? I40E_ERR_RESET_FAILED : pfr;
15136 }
15137
15138 /**
15139  * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15140  * @pf: board private structure
15141  * @hw: ptr to the hardware info
15142  *
15143  * This function does a minimal setup of all subsystems needed for running
15144  * recovery mode.
15145  *
15146  * Returns 0 on success, negative on failure
15147  **/
15148 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15149 {
15150         struct i40e_vsi *vsi;
15151         int err;
15152         int v_idx;
15153
15154         pci_save_state(pf->pdev);
15155
15156         /* set up periodic task facility */
15157         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15158         pf->service_timer_period = HZ;
15159
15160         INIT_WORK(&pf->service_task, i40e_service_task);
15161         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15162
15163         err = i40e_init_interrupt_scheme(pf);
15164         if (err)
15165                 goto err_switch_setup;
15166
15167         /* The number of VSIs reported by the FW is the minimum guaranteed
15168          * to us; HW supports far more and we share the remaining pool with
15169          * the other PFs. We allocate space for more than the guarantee with
15170          * the understanding that we might not get them all later.
15171          */
15172         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15173                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15174         else
15175                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15176
15177         /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15178         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15179                           GFP_KERNEL);
15180         if (!pf->vsi) {
15181                 err = -ENOMEM;
15182                 goto err_switch_setup;
15183         }
15184
15185         /* We allocate one VSI which is needed as absolute minimum
15186          * in order to register the netdev
15187          */
15188         v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
15189         if (v_idx < 0) {
15190                 err = v_idx;
15191                 goto err_switch_setup;
15192         }
15193         pf->lan_vsi = v_idx;
15194         vsi = pf->vsi[v_idx];
15195         if (!vsi) {
15196                 err = -EFAULT;
15197                 goto err_switch_setup;
15198         }
15199         vsi->alloc_queue_pairs = 1;
15200         err = i40e_config_netdev(vsi);
15201         if (err)
15202                 goto err_switch_setup;
15203         err = register_netdev(vsi->netdev);
15204         if (err)
15205                 goto err_switch_setup;
15206         vsi->netdev_registered = true;
15207         i40e_dbg_pf_init(pf);
15208
15209         err = i40e_setup_misc_vector_for_recovery_mode(pf);
15210         if (err)
15211                 goto err_switch_setup;
15212
15213         /* tell the firmware that we're starting */
15214         i40e_send_version(pf);
15215
15216         /* since everything's happy, start the service_task timer */
15217         mod_timer(&pf->service_timer,
15218                   round_jiffies(jiffies + pf->service_timer_period));
15219
15220         return 0;
15221
15222 err_switch_setup:
15223         i40e_reset_interrupt_capability(pf);
15224         del_timer_sync(&pf->service_timer);
15225         i40e_shutdown_adminq(hw);
15226         iounmap(hw->hw_addr);
15227         pci_disable_pcie_error_reporting(pf->pdev);
15228         pci_release_mem_regions(pf->pdev);
15229         pci_disable_device(pf->pdev);
15230         kfree(pf);
15231
15232         return err;
15233 }
15234
15235 /**
15236  * i40e_set_subsystem_device_id - set subsystem device id
15237  * @hw: pointer to the hardware info
15238  *
15239  * Set PCI subsystem device id either from a pci_dev structure or
15240  * a specific FW register.
15241  **/
15242 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)
15243 {
15244         struct pci_dev *pdev = ((struct i40e_pf *)hw->back)->pdev;
15245
15246         hw->subsystem_device_id = pdev->subsystem_device ?
15247                 pdev->subsystem_device :
15248                 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);
15249 }
15250
15251 /**
15252  * i40e_probe - Device initialization routine
15253  * @pdev: PCI device information struct
15254  * @ent: entry in i40e_pci_tbl
15255  *
15256  * i40e_probe initializes a PF identified by a pci_dev structure.
15257  * The OS initialization, configuring of the PF private structure,
15258  * and a hardware reset occur.
15259  *
15260  * Returns 0 on success, negative on failure
15261  **/
15262 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15263 {
15264         struct i40e_aq_get_phy_abilities_resp abilities;
15265 #ifdef CONFIG_I40E_DCB
15266         enum i40e_get_fw_lldp_status_resp lldp_status;
15267         i40e_status status;
15268 #endif /* CONFIG_I40E_DCB */
15269         struct i40e_pf *pf;
15270         struct i40e_hw *hw;
15271         static u16 pfs_found;
15272         u16 wol_nvm_bits;
15273         u16 link_status;
15274         int err;
15275         u32 val;
15276         u32 i;
15277
15278         err = pci_enable_device_mem(pdev);
15279         if (err)
15280                 return err;
15281
15282         /* set up for high or low dma */
15283         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
15284         if (err) {
15285                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
15286                 if (err) {
15287                         dev_err(&pdev->dev,
15288                                 "DMA configuration failed: 0x%x\n", err);
15289                         goto err_dma;
15290                 }
15291         }
15292
15293         /* set up pci connections */
15294         err = pci_request_mem_regions(pdev, i40e_driver_name);
15295         if (err) {
15296                 dev_info(&pdev->dev,
15297                          "pci_request_selected_regions failed %d\n", err);
15298                 goto err_pci_reg;
15299         }
15300
15301         pci_enable_pcie_error_reporting(pdev);
15302         pci_set_master(pdev);
15303
15304         /* Now that we have a PCI connection, we need to do the
15305          * low level device setup.  This is primarily setting up
15306          * the Admin Queue structures and then querying for the
15307          * device's current profile information.
15308          */
15309         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
15310         if (!pf) {
15311                 err = -ENOMEM;
15312                 goto err_pf_alloc;
15313         }
15314         pf->next_vsi = 0;
15315         pf->pdev = pdev;
15316         set_bit(__I40E_DOWN, pf->state);
15317
15318         hw = &pf->hw;
15319         hw->back = pf;
15320
15321         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15322                                 I40E_MAX_CSR_SPACE);
15323         /* We believe that the highest register to read is
15324          * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15325          * is not less than that before mapping to prevent a
15326          * kernel panic.
15327          */
15328         if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15329                 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15330                         pf->ioremap_len);
15331                 err = -ENOMEM;
15332                 goto err_ioremap;
15333         }
15334         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15335         if (!hw->hw_addr) {
15336                 err = -EIO;
15337                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15338                          (unsigned int)pci_resource_start(pdev, 0),
15339                          pf->ioremap_len, err);
15340                 goto err_ioremap;
15341         }
15342         hw->vendor_id = pdev->vendor;
15343         hw->device_id = pdev->device;
15344         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15345         hw->subsystem_vendor_id = pdev->subsystem_vendor;
15346         i40e_set_subsystem_device_id(hw);
15347         hw->bus.device = PCI_SLOT(pdev->devfn);
15348         hw->bus.func = PCI_FUNC(pdev->devfn);
15349         hw->bus.bus_id = pdev->bus->number;
15350         pf->instance = pfs_found;
15351
15352         /* Select something other than the 802.1ad ethertype for the
15353          * switch to use internally and drop on ingress.
15354          */
15355         hw->switch_tag = 0xffff;
15356         hw->first_tag = ETH_P_8021AD;
15357         hw->second_tag = ETH_P_8021Q;
15358
15359         INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15360         INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15361         INIT_LIST_HEAD(&pf->ddp_old_prof);
15362
15363         /* set up the locks for the AQ, do this only once in probe
15364          * and destroy them only once in remove
15365          */
15366         mutex_init(&hw->aq.asq_mutex);
15367         mutex_init(&hw->aq.arq_mutex);
15368
15369         pf->msg_enable = netif_msg_init(debug,
15370                                         NETIF_MSG_DRV |
15371                                         NETIF_MSG_PROBE |
15372                                         NETIF_MSG_LINK);
15373         if (debug < -1)
15374                 pf->hw.debug_mask = debug;
15375
15376         /* do a special CORER for clearing PXE mode once at init */
15377         if (hw->revision_id == 0 &&
15378             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15379                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15380                 i40e_flush(hw);
15381                 msleep(200);
15382                 pf->corer_count++;
15383
15384                 i40e_clear_pxe_mode(hw);
15385         }
15386
15387         /* Reset here to make sure all is clean and to define PF 'n' */
15388         i40e_clear_hw(hw);
15389
15390         err = i40e_set_mac_type(hw);
15391         if (err) {
15392                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15393                          err);
15394                 goto err_pf_reset;
15395         }
15396
15397         err = i40e_handle_resets(pf);
15398         if (err)
15399                 goto err_pf_reset;
15400
15401         i40e_check_recovery_mode(pf);
15402
15403         if (is_kdump_kernel()) {
15404                 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;
15405                 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;
15406         } else {
15407                 hw->aq.num_arq_entries = I40E_AQ_LEN;
15408                 hw->aq.num_asq_entries = I40E_AQ_LEN;
15409         }
15410         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15411         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15412         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
15413
15414         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15415                  "%s-%s:misc",
15416                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15417
15418         err = i40e_init_shared_code(hw);
15419         if (err) {
15420                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15421                          err);
15422                 goto err_pf_reset;
15423         }
15424
15425         /* set up a default setting for link flow control */
15426         pf->hw.fc.requested_mode = I40E_FC_NONE;
15427
15428         err = i40e_init_adminq(hw);
15429         if (err) {
15430                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
15431                         dev_info(&pdev->dev,
15432                                  "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15433                                  hw->aq.api_maj_ver,
15434                                  hw->aq.api_min_ver,
15435                                  I40E_FW_API_VERSION_MAJOR,
15436                                  I40E_FW_MINOR_VERSION(hw));
15437                 else
15438                         dev_info(&pdev->dev,
15439                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15440
15441                 goto err_pf_reset;
15442         }
15443         i40e_get_oem_version(hw);
15444
15445         /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15446         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15447                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15448                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
15449                  i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id,
15450                  hw->subsystem_vendor_id, hw->subsystem_device_id);
15451
15452         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
15453             hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
15454                 dev_info(&pdev->dev,
15455                          "The driver for the device detected a newer version of the NVM image v%u.%u than expected v%u.%u. Please install the most recent version of the network driver.\n",
15456                          hw->aq.api_maj_ver,
15457                          hw->aq.api_min_ver,
15458                          I40E_FW_API_VERSION_MAJOR,
15459                          I40E_FW_MINOR_VERSION(hw));
15460         else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
15461                 dev_info(&pdev->dev,
15462                          "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15463                          hw->aq.api_maj_ver,
15464                          hw->aq.api_min_ver,
15465                          I40E_FW_API_VERSION_MAJOR,
15466                          I40E_FW_MINOR_VERSION(hw));
15467
15468         i40e_verify_eeprom(pf);
15469
15470         /* Rev 0 hardware was never productized */
15471         if (hw->revision_id < 1)
15472                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15473
15474         i40e_clear_pxe_mode(hw);
15475
15476         err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15477         if (err)
15478                 goto err_adminq_setup;
15479
15480         err = i40e_sw_init(pf);
15481         if (err) {
15482                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15483                 goto err_sw_init;
15484         }
15485
15486         if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15487                 return i40e_init_recovery_mode(pf, hw);
15488
15489         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15490                                 hw->func_caps.num_rx_qp, 0, 0);
15491         if (err) {
15492                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15493                 goto err_init_lan_hmc;
15494         }
15495
15496         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15497         if (err) {
15498                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15499                 err = -ENOENT;
15500                 goto err_configure_lan_hmc;
15501         }
15502
15503         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
15504          * Ignore error return codes because if it was already disabled via
15505          * hardware settings this will fail
15506          */
15507         if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
15508                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15509                 i40e_aq_stop_lldp(hw, true, false, NULL);
15510         }
15511
15512         /* allow a platform config to override the HW addr */
15513         i40e_get_platform_mac_addr(pdev, pf);
15514
15515         if (!is_valid_ether_addr(hw->mac.addr)) {
15516                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15517                 err = -EIO;
15518                 goto err_mac_addr;
15519         }
15520         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15521         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15522         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15523         if (is_valid_ether_addr(hw->mac.port_addr))
15524                 pf->hw_features |= I40E_HW_PORT_ID_VALID;
15525
15526         i40e_ptp_alloc_pins(pf);
15527         pci_set_drvdata(pdev, pf);
15528         pci_save_state(pdev);
15529
15530 #ifdef CONFIG_I40E_DCB
15531         status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
15532         (!status &&
15533          lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15534                 (pf->flags &= ~I40E_FLAG_DISABLE_FW_LLDP) :
15535                 (pf->flags |= I40E_FLAG_DISABLE_FW_LLDP);
15536         dev_info(&pdev->dev,
15537                  (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ?
15538                         "FW LLDP is disabled\n" :
15539                         "FW LLDP is enabled\n");
15540
15541         /* Enable FW to write default DCB config on link-up */
15542         i40e_aq_set_dcb_parameters(hw, true, NULL);
15543
15544         err = i40e_init_pf_dcb(pf);
15545         if (err) {
15546                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15547                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
15548                 /* Continue without DCB enabled */
15549         }
15550 #endif /* CONFIG_I40E_DCB */
15551
15552         /* set up periodic task facility */
15553         timer_setup(&pf->service_timer, i40e_service_timer, 0);
15554         pf->service_timer_period = HZ;
15555
15556         INIT_WORK(&pf->service_task, i40e_service_task);
15557         clear_bit(__I40E_SERVICE_SCHED, pf->state);
15558
15559         /* NVM bit on means WoL disabled for the port */
15560         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15561         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15562                 pf->wol_en = false;
15563         else
15564                 pf->wol_en = true;
15565         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15566
15567         /* set up the main switch operations */
15568         i40e_determine_queue_usage(pf);
15569         err = i40e_init_interrupt_scheme(pf);
15570         if (err)
15571                 goto err_switch_setup;
15572
15573         /* Reduce Tx and Rx pairs for kdump
15574          * When MSI-X is enabled, it's not allowed to use more TC queue
15575          * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus
15576          * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.
15577          */
15578         if (is_kdump_kernel())
15579                 pf->num_lan_msix = 1;
15580
15581         pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15582         pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15583         pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15584         pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15585         pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15586         pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15587                                                     UDP_TUNNEL_TYPE_GENEVE;
15588
15589         /* The number of VSIs reported by the FW is the minimum guaranteed
15590          * to us; HW supports far more and we share the remaining pool with
15591          * the other PFs. We allocate space for more than the guarantee with
15592          * the understanding that we might not get them all later.
15593          */
15594         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15595                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15596         else
15597                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15598         if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15599                 dev_warn(&pf->pdev->dev,
15600                          "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15601                          pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15602                 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15603         }
15604
15605         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15606         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15607                           GFP_KERNEL);
15608         if (!pf->vsi) {
15609                 err = -ENOMEM;
15610                 goto err_switch_setup;
15611         }
15612
15613 #ifdef CONFIG_PCI_IOV
15614         /* prep for VF support */
15615         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15616             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15617             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15618                 if (pci_num_vf(pdev))
15619                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
15620         }
15621 #endif
15622         err = i40e_setup_pf_switch(pf, false, false);
15623         if (err) {
15624                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15625                 goto err_vsis;
15626         }
15627         INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
15628
15629         /* if FDIR VSI was set up, start it now */
15630         for (i = 0; i < pf->num_alloc_vsi; i++) {
15631                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
15632                         i40e_vsi_open(pf->vsi[i]);
15633                         break;
15634                 }
15635         }
15636
15637         /* The driver only wants link up/down and module qualification
15638          * reports from firmware.  Note the negative logic.
15639          */
15640         err = i40e_aq_set_phy_int_mask(&pf->hw,
15641                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
15642                                          I40E_AQ_EVENT_MEDIA_NA |
15643                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15644         if (err)
15645                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
15646                          i40e_stat_str(&pf->hw, err),
15647                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15648
15649         /* Reconfigure hardware for allowing smaller MSS in the case
15650          * of TSO, so that we avoid the MDD being fired and causing
15651          * a reset in the case of small MSS+TSO.
15652          */
15653         val = rd32(hw, I40E_REG_MSS);
15654         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15655                 val &= ~I40E_REG_MSS_MIN_MASK;
15656                 val |= I40E_64BYTE_MSS;
15657                 wr32(hw, I40E_REG_MSS, val);
15658         }
15659
15660         if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
15661                 msleep(75);
15662                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15663                 if (err)
15664                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
15665                                  i40e_stat_str(&pf->hw, err),
15666                                  i40e_aq_str(&pf->hw,
15667                                              pf->hw.aq.asq_last_status));
15668         }
15669         /* The main driver is (mostly) up and happy. We need to set this state
15670          * before setting up the misc vector or we get a race and the vector
15671          * ends up disabled forever.
15672          */
15673         clear_bit(__I40E_DOWN, pf->state);
15674
15675         /* In case of MSIX we are going to setup the misc vector right here
15676          * to handle admin queue events etc. In case of legacy and MSI
15677          * the misc functionality and queue processing is combined in
15678          * the same vector and that gets setup at open.
15679          */
15680         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
15681                 err = i40e_setup_misc_vector(pf);
15682                 if (err) {
15683                         dev_info(&pdev->dev,
15684                                  "setup of misc vector failed: %d\n", err);
15685                         i40e_cloud_filter_exit(pf);
15686                         i40e_fdir_teardown(pf);
15687                         goto err_vsis;
15688                 }
15689         }
15690
15691 #ifdef CONFIG_PCI_IOV
15692         /* prep for VF support */
15693         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15694             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15695             !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15696                 /* disable link interrupts for VFs */
15697                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
15698                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
15699                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
15700                 i40e_flush(hw);
15701
15702                 if (pci_num_vf(pdev)) {
15703                         dev_info(&pdev->dev,
15704                                  "Active VFs found, allocating resources.\n");
15705                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
15706                         if (err)
15707                                 dev_info(&pdev->dev,
15708                                          "Error %d allocating resources for existing VFs\n",
15709                                          err);
15710                 }
15711         }
15712 #endif /* CONFIG_PCI_IOV */
15713
15714         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15715                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
15716                                                       pf->num_iwarp_msix,
15717                                                       I40E_IWARP_IRQ_PILE_ID);
15718                 if (pf->iwarp_base_vector < 0) {
15719                         dev_info(&pdev->dev,
15720                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
15721                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
15722                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
15723                 }
15724         }
15725
15726         i40e_dbg_pf_init(pf);
15727
15728         /* tell the firmware that we're starting */
15729         i40e_send_version(pf);
15730
15731         /* since everything's happy, start the service_task timer */
15732         mod_timer(&pf->service_timer,
15733                   round_jiffies(jiffies + pf->service_timer_period));
15734
15735         /* add this PF to client device list and launch a client service task */
15736         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15737                 err = i40e_lan_add_device(pf);
15738                 if (err)
15739                         dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
15740                                  err);
15741         }
15742
15743 #define PCI_SPEED_SIZE 8
15744 #define PCI_WIDTH_SIZE 8
15745         /* Devices on the IOSF bus do not have this information
15746          * and will report PCI Gen 1 x 1 by default so don't bother
15747          * checking them.
15748          */
15749         if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
15750                 char speed[PCI_SPEED_SIZE] = "Unknown";
15751                 char width[PCI_WIDTH_SIZE] = "Unknown";
15752
15753                 /* Get the negotiated link width and speed from PCI config
15754                  * space
15755                  */
15756                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
15757                                           &link_status);
15758
15759                 i40e_set_pci_config_data(hw, link_status);
15760
15761                 switch (hw->bus.speed) {
15762                 case i40e_bus_speed_8000:
15763                         strlcpy(speed, "8.0", PCI_SPEED_SIZE); break;
15764                 case i40e_bus_speed_5000:
15765                         strlcpy(speed, "5.0", PCI_SPEED_SIZE); break;
15766                 case i40e_bus_speed_2500:
15767                         strlcpy(speed, "2.5", PCI_SPEED_SIZE); break;
15768                 default:
15769                         break;
15770                 }
15771                 switch (hw->bus.width) {
15772                 case i40e_bus_width_pcie_x8:
15773                         strlcpy(width, "8", PCI_WIDTH_SIZE); break;
15774                 case i40e_bus_width_pcie_x4:
15775                         strlcpy(width, "4", PCI_WIDTH_SIZE); break;
15776                 case i40e_bus_width_pcie_x2:
15777                         strlcpy(width, "2", PCI_WIDTH_SIZE); break;
15778                 case i40e_bus_width_pcie_x1:
15779                         strlcpy(width, "1", PCI_WIDTH_SIZE); break;
15780                 default:
15781                         break;
15782                 }
15783
15784                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
15785                          speed, width);
15786
15787                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
15788                     hw->bus.speed < i40e_bus_speed_8000) {
15789                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
15790                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
15791                 }
15792         }
15793
15794         /* get the requested speeds from the fw */
15795         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
15796         if (err)
15797                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
15798                         i40e_stat_str(&pf->hw, err),
15799                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15800         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
15801
15802         /* set the FEC config due to the board capabilities */
15803         i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags);
15804
15805         /* get the supported phy types from the fw */
15806         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
15807         if (err)
15808                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
15809                         i40e_stat_str(&pf->hw, err),
15810                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15811
15812         /* make sure the MFS hasn't been set lower than the default */
15813 #define MAX_FRAME_SIZE_DEFAULT 0x2600
15814         val = (rd32(&pf->hw, I40E_PRTGL_SAH) &
15815                I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT;
15816         if (val < MAX_FRAME_SIZE_DEFAULT)
15817                 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n",
15818                          i, val);
15819
15820         /* Add a filter to drop all Flow control frames from any VSI from being
15821          * transmitted. By doing so we stop a malicious VF from sending out
15822          * PAUSE or PFC frames and potentially controlling traffic for other
15823          * PF/VF VSIs.
15824          * The FW can still send Flow control frames if enabled.
15825          */
15826         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
15827                                                        pf->main_vsi_seid);
15828
15829         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
15830                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
15831                 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
15832         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
15833                 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
15834         /* print a string summarizing features */
15835         i40e_print_features(pf);
15836
15837         return 0;
15838
15839         /* Unwind what we've done if something failed in the setup */
15840 err_vsis:
15841         set_bit(__I40E_DOWN, pf->state);
15842         i40e_clear_interrupt_scheme(pf);
15843         kfree(pf->vsi);
15844 err_switch_setup:
15845         i40e_reset_interrupt_capability(pf);
15846         del_timer_sync(&pf->service_timer);
15847 err_mac_addr:
15848 err_configure_lan_hmc:
15849         (void)i40e_shutdown_lan_hmc(hw);
15850 err_init_lan_hmc:
15851         kfree(pf->qp_pile);
15852 err_sw_init:
15853 err_adminq_setup:
15854 err_pf_reset:
15855         iounmap(hw->hw_addr);
15856 err_ioremap:
15857         kfree(pf);
15858 err_pf_alloc:
15859         pci_disable_pcie_error_reporting(pdev);
15860         pci_release_mem_regions(pdev);
15861 err_pci_reg:
15862 err_dma:
15863         pci_disable_device(pdev);
15864         return err;
15865 }
15866
15867 /**
15868  * i40e_remove - Device removal routine
15869  * @pdev: PCI device information struct
15870  *
15871  * i40e_remove is called by the PCI subsystem to alert the driver
15872  * that is should release a PCI device.  This could be caused by a
15873  * Hot-Plug event, or because the driver is going to be removed from
15874  * memory.
15875  **/
15876 static void i40e_remove(struct pci_dev *pdev)
15877 {
15878         struct i40e_pf *pf = pci_get_drvdata(pdev);
15879         struct i40e_hw *hw = &pf->hw;
15880         i40e_status ret_code;
15881         int i;
15882
15883         i40e_dbg_pf_exit(pf);
15884
15885         i40e_ptp_stop(pf);
15886
15887         /* Disable RSS in hw */
15888         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
15889         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
15890
15891         while (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
15892                 usleep_range(1000, 2000);
15893
15894         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
15895                 set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
15896                 i40e_free_vfs(pf);
15897                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
15898         }
15899         /* no more scheduling of any task */
15900         set_bit(__I40E_SUSPENDED, pf->state);
15901         set_bit(__I40E_DOWN, pf->state);
15902         if (pf->service_timer.function)
15903                 del_timer_sync(&pf->service_timer);
15904         if (pf->service_task.func)
15905                 cancel_work_sync(&pf->service_task);
15906
15907         if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
15908                 struct i40e_vsi *vsi = pf->vsi[0];
15909
15910                 /* We know that we have allocated only one vsi for this PF,
15911                  * it was just for registering netdevice, so the interface
15912                  * could be visible in the 'ifconfig' output
15913                  */
15914                 unregister_netdev(vsi->netdev);
15915                 free_netdev(vsi->netdev);
15916
15917                 goto unmap;
15918         }
15919
15920         /* Client close must be called explicitly here because the timer
15921          * has been stopped.
15922          */
15923         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15924
15925         i40e_fdir_teardown(pf);
15926
15927         /* If there is a switch structure or any orphans, remove them.
15928          * This will leave only the PF's VSI remaining.
15929          */
15930         for (i = 0; i < I40E_MAX_VEB; i++) {
15931                 if (!pf->veb[i])
15932                         continue;
15933
15934                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
15935                     pf->veb[i]->uplink_seid == 0)
15936                         i40e_switch_branch_release(pf->veb[i]);
15937         }
15938
15939         /* Now we can shutdown the PF's VSI, just before we kill
15940          * adminq and hmc.
15941          */
15942         if (pf->vsi[pf->lan_vsi])
15943                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
15944
15945         i40e_cloud_filter_exit(pf);
15946
15947         /* remove attached clients */
15948         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15949                 ret_code = i40e_lan_del_device(pf);
15950                 if (ret_code)
15951                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
15952                                  ret_code);
15953         }
15954
15955         /* shutdown and destroy the HMC */
15956         if (hw->hmc.hmc_obj) {
15957                 ret_code = i40e_shutdown_lan_hmc(hw);
15958                 if (ret_code)
15959                         dev_warn(&pdev->dev,
15960                                  "Failed to destroy the HMC resources: %d\n",
15961                                  ret_code);
15962         }
15963
15964 unmap:
15965         /* Free MSI/legacy interrupt 0 when in recovery mode. */
15966         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
15967             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
15968                 free_irq(pf->pdev->irq, pf);
15969
15970         /* shutdown the adminq */
15971         i40e_shutdown_adminq(hw);
15972
15973         /* destroy the locks only once, here */
15974         mutex_destroy(&hw->aq.arq_mutex);
15975         mutex_destroy(&hw->aq.asq_mutex);
15976
15977         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
15978         rtnl_lock();
15979         i40e_clear_interrupt_scheme(pf);
15980         for (i = 0; i < pf->num_alloc_vsi; i++) {
15981                 if (pf->vsi[i]) {
15982                         if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
15983                                 i40e_vsi_clear_rings(pf->vsi[i]);
15984                         i40e_vsi_clear(pf->vsi[i]);
15985                         pf->vsi[i] = NULL;
15986                 }
15987         }
15988         rtnl_unlock();
15989
15990         for (i = 0; i < I40E_MAX_VEB; i++) {
15991                 kfree(pf->veb[i]);
15992                 pf->veb[i] = NULL;
15993         }
15994
15995         kfree(pf->qp_pile);
15996         kfree(pf->vsi);
15997
15998         iounmap(hw->hw_addr);
15999         kfree(pf);
16000         pci_release_mem_regions(pdev);
16001
16002         pci_disable_pcie_error_reporting(pdev);
16003         pci_disable_device(pdev);
16004 }
16005
16006 /**
16007  * i40e_pci_error_detected - warning that something funky happened in PCI land
16008  * @pdev: PCI device information struct
16009  * @error: the type of PCI error
16010  *
16011  * Called to warn that something happened and the error handling steps
16012  * are in progress.  Allows the driver to quiesce things, be ready for
16013  * remediation.
16014  **/
16015 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
16016                                                 pci_channel_state_t error)
16017 {
16018         struct i40e_pf *pf = pci_get_drvdata(pdev);
16019
16020         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
16021
16022         if (!pf) {
16023                 dev_info(&pdev->dev,
16024                          "Cannot recover - error happened during device probe\n");
16025                 return PCI_ERS_RESULT_DISCONNECT;
16026         }
16027
16028         /* shutdown all operations */
16029         if (!test_bit(__I40E_SUSPENDED, pf->state))
16030                 i40e_prep_for_reset(pf);
16031
16032         /* Request a slot reset */
16033         return PCI_ERS_RESULT_NEED_RESET;
16034 }
16035
16036 /**
16037  * i40e_pci_error_slot_reset - a PCI slot reset just happened
16038  * @pdev: PCI device information struct
16039  *
16040  * Called to find if the driver can work with the device now that
16041  * the pci slot has been reset.  If a basic connection seems good
16042  * (registers are readable and have sane content) then return a
16043  * happy little PCI_ERS_RESULT_xxx.
16044  **/
16045 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
16046 {
16047         struct i40e_pf *pf = pci_get_drvdata(pdev);
16048         pci_ers_result_t result;
16049         u32 reg;
16050
16051         dev_dbg(&pdev->dev, "%s\n", __func__);
16052         if (pci_enable_device_mem(pdev)) {
16053                 dev_info(&pdev->dev,
16054                          "Cannot re-enable PCI device after reset.\n");
16055                 result = PCI_ERS_RESULT_DISCONNECT;
16056         } else {
16057                 pci_set_master(pdev);
16058                 pci_restore_state(pdev);
16059                 pci_save_state(pdev);
16060                 pci_wake_from_d3(pdev, false);
16061
16062                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
16063                 if (reg == 0)
16064                         result = PCI_ERS_RESULT_RECOVERED;
16065                 else
16066                         result = PCI_ERS_RESULT_DISCONNECT;
16067         }
16068
16069         return result;
16070 }
16071
16072 /**
16073  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
16074  * @pdev: PCI device information struct
16075  */
16076 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
16077 {
16078         struct i40e_pf *pf = pci_get_drvdata(pdev);
16079
16080         i40e_prep_for_reset(pf);
16081 }
16082
16083 /**
16084  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16085  * @pdev: PCI device information struct
16086  */
16087 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
16088 {
16089         struct i40e_pf *pf = pci_get_drvdata(pdev);
16090
16091         i40e_reset_and_rebuild(pf, false, false);
16092 }
16093
16094 /**
16095  * i40e_pci_error_resume - restart operations after PCI error recovery
16096  * @pdev: PCI device information struct
16097  *
16098  * Called to allow the driver to bring things back up after PCI error
16099  * and/or reset recovery has finished.
16100  **/
16101 static void i40e_pci_error_resume(struct pci_dev *pdev)
16102 {
16103         struct i40e_pf *pf = pci_get_drvdata(pdev);
16104
16105         dev_dbg(&pdev->dev, "%s\n", __func__);
16106         if (test_bit(__I40E_SUSPENDED, pf->state))
16107                 return;
16108
16109         i40e_handle_reset_warning(pf, false);
16110 }
16111
16112 /**
16113  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16114  * using the mac_address_write admin q function
16115  * @pf: pointer to i40e_pf struct
16116  **/
16117 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16118 {
16119         struct i40e_hw *hw = &pf->hw;
16120         i40e_status ret;
16121         u8 mac_addr[6];
16122         u16 flags = 0;
16123
16124         /* Get current MAC address in case it's an LAA */
16125         if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
16126                 ether_addr_copy(mac_addr,
16127                                 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
16128         } else {
16129                 dev_err(&pf->pdev->dev,
16130                         "Failed to retrieve MAC address; using default\n");
16131                 ether_addr_copy(mac_addr, hw->mac.addr);
16132         }
16133
16134         /* The FW expects the mac address write cmd to first be called with
16135          * one of these flags before calling it again with the multicast
16136          * enable flags.
16137          */
16138         flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16139
16140         if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16141                 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16142
16143         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16144         if (ret) {
16145                 dev_err(&pf->pdev->dev,
16146                         "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16147                 return;
16148         }
16149
16150         flags = I40E_AQC_MC_MAG_EN
16151                         | I40E_AQC_WOL_PRESERVE_ON_PFR
16152                         | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16153         ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16154         if (ret)
16155                 dev_err(&pf->pdev->dev,
16156                         "Failed to enable Multicast Magic Packet wake up\n");
16157 }
16158
16159 /**
16160  * i40e_shutdown - PCI callback for shutting down
16161  * @pdev: PCI device information struct
16162  **/
16163 static void i40e_shutdown(struct pci_dev *pdev)
16164 {
16165         struct i40e_pf *pf = pci_get_drvdata(pdev);
16166         struct i40e_hw *hw = &pf->hw;
16167
16168         set_bit(__I40E_SUSPENDED, pf->state);
16169         set_bit(__I40E_DOWN, pf->state);
16170
16171         del_timer_sync(&pf->service_timer);
16172         cancel_work_sync(&pf->service_task);
16173         i40e_cloud_filter_exit(pf);
16174         i40e_fdir_teardown(pf);
16175
16176         /* Client close must be called explicitly here because the timer
16177          * has been stopped.
16178          */
16179         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16180
16181         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16182                 i40e_enable_mc_magic_wake(pf);
16183
16184         i40e_prep_for_reset(pf);
16185
16186         wr32(hw, I40E_PFPM_APM,
16187              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16188         wr32(hw, I40E_PFPM_WUFC,
16189              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16190
16191         /* Free MSI/legacy interrupt 0 when in recovery mode. */
16192         if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16193             !(pf->flags & I40E_FLAG_MSIX_ENABLED))
16194                 free_irq(pf->pdev->irq, pf);
16195
16196         /* Since we're going to destroy queues during the
16197          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16198          * whole section
16199          */
16200         rtnl_lock();
16201         i40e_clear_interrupt_scheme(pf);
16202         rtnl_unlock();
16203
16204         if (system_state == SYSTEM_POWER_OFF) {
16205                 pci_wake_from_d3(pdev, pf->wol_en);
16206                 pci_set_power_state(pdev, PCI_D3hot);
16207         }
16208 }
16209
16210 /**
16211  * i40e_suspend - PM callback for moving to D3
16212  * @dev: generic device information structure
16213  **/
16214 static int __maybe_unused i40e_suspend(struct device *dev)
16215 {
16216         struct i40e_pf *pf = dev_get_drvdata(dev);
16217         struct i40e_hw *hw = &pf->hw;
16218
16219         /* If we're already suspended, then there is nothing to do */
16220         if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
16221                 return 0;
16222
16223         set_bit(__I40E_DOWN, pf->state);
16224
16225         /* Ensure service task will not be running */
16226         del_timer_sync(&pf->service_timer);
16227         cancel_work_sync(&pf->service_task);
16228
16229         /* Client close must be called explicitly here because the timer
16230          * has been stopped.
16231          */
16232         i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16233
16234         if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16235                 i40e_enable_mc_magic_wake(pf);
16236
16237         /* Since we're going to destroy queues during the
16238          * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16239          * whole section
16240          */
16241         rtnl_lock();
16242
16243         i40e_prep_for_reset(pf);
16244
16245         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16246         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16247
16248         /* Clear the interrupt scheme and release our IRQs so that the system
16249          * can safely hibernate even when there are a large number of CPUs.
16250          * Otherwise hibernation might fail when mapping all the vectors back
16251          * to CPU0.
16252          */
16253         i40e_clear_interrupt_scheme(pf);
16254
16255         rtnl_unlock();
16256
16257         return 0;
16258 }
16259
16260 /**
16261  * i40e_resume - PM callback for waking up from D3
16262  * @dev: generic device information structure
16263  **/
16264 static int __maybe_unused i40e_resume(struct device *dev)
16265 {
16266         struct i40e_pf *pf = dev_get_drvdata(dev);
16267         int err;
16268
16269         /* If we're not suspended, then there is nothing to do */
16270         if (!test_bit(__I40E_SUSPENDED, pf->state))
16271                 return 0;
16272
16273         /* We need to hold the RTNL lock prior to restoring interrupt schemes,
16274          * since we're going to be restoring queues
16275          */
16276         rtnl_lock();
16277
16278         /* We cleared the interrupt scheme when we suspended, so we need to
16279          * restore it now to resume device functionality.
16280          */
16281         err = i40e_restore_interrupt_scheme(pf);
16282         if (err) {
16283                 dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16284                         err);
16285         }
16286
16287         clear_bit(__I40E_DOWN, pf->state);
16288         i40e_reset_and_rebuild(pf, false, true);
16289
16290         rtnl_unlock();
16291
16292         /* Clear suspended state last after everything is recovered */
16293         clear_bit(__I40E_SUSPENDED, pf->state);
16294
16295         /* Restart the service task */
16296         mod_timer(&pf->service_timer,
16297                   round_jiffies(jiffies + pf->service_timer_period));
16298
16299         return 0;
16300 }
16301
16302 static const struct pci_error_handlers i40e_err_handler = {
16303         .error_detected = i40e_pci_error_detected,
16304         .slot_reset = i40e_pci_error_slot_reset,
16305         .reset_prepare = i40e_pci_error_reset_prepare,
16306         .reset_done = i40e_pci_error_reset_done,
16307         .resume = i40e_pci_error_resume,
16308 };
16309
16310 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16311
16312 static struct pci_driver i40e_driver = {
16313         .name     = i40e_driver_name,
16314         .id_table = i40e_pci_tbl,
16315         .probe    = i40e_probe,
16316         .remove   = i40e_remove,
16317         .driver   = {
16318                 .pm = &i40e_pm_ops,
16319         },
16320         .shutdown = i40e_shutdown,
16321         .err_handler = &i40e_err_handler,
16322         .sriov_configure = i40e_pci_sriov_configure,
16323 };
16324
16325 /**
16326  * i40e_init_module - Driver registration routine
16327  *
16328  * i40e_init_module is the first routine called when the driver is
16329  * loaded. All it does is register with the PCI subsystem.
16330  **/
16331 static int __init i40e_init_module(void)
16332 {
16333         pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16334         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16335
16336         /* There is no need to throttle the number of active tasks because
16337          * each device limits its own task using a state bit for scheduling
16338          * the service task, and the device tasks do not interfere with each
16339          * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16340          * since we need to be able to guarantee forward progress even under
16341          * memory pressure.
16342          */
16343         i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
16344         if (!i40e_wq) {
16345                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16346                 return -ENOMEM;
16347         }
16348
16349         i40e_dbg_init();
16350         return pci_register_driver(&i40e_driver);
16351 }
16352 module_init(i40e_init_module);
16353
16354 /**
16355  * i40e_exit_module - Driver exit cleanup routine
16356  *
16357  * i40e_exit_module is called just before the driver is removed
16358  * from memory.
16359  **/
16360 static void __exit i40e_exit_module(void)
16361 {
16362         pci_unregister_driver(&i40e_driver);
16363         destroy_workqueue(i40e_wq);
16364         ida_destroy(&i40e_client_ida);
16365         i40e_dbg_exit();
16366 }
16367 module_exit(i40e_exit_module);