drivers/net/ethernet/intel/i40e/i40e_main.c

   1 /*******************************************************************************
   2  *
   3  * Intel Ethernet Controller XL710 Family Linux Driver
   4  * Copyright(c) 2013 - 2016 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms and conditions of the GNU General Public License,
   8  * version 2, as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13  * more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program.  If not, see <http://www.gnu.org/licenses/>.
  17  *
  18  * The full GNU General Public License is included in this distribution in
  19  * the file called "COPYING".
  20  *
  21  * Contact Information:
  22  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  23  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  24  *
  25  ******************************************************************************/
  26
  27 #include <linux/etherdevice.h>
  28 #include <linux/of_net.h>
  29 #include <linux/pci.h>
  30
  31 /* Local includes */
  32 #include "i40e.h"
  33 #include "i40e_diag.h"
  34 #include <net/udp_tunnel.h>
  35
  36 const char i40e_driver_name[] = "i40e";
  37 static const char i40e_driver_string[] =
  38                         "Intel(R) Ethernet Connection XL710 Network Driver";
  39
  40 #define DRV_KERN "-k"
  41
  42 #define DRV_VERSION_MAJOR 1
  43 #define DRV_VERSION_MINOR 6
  44 #define DRV_VERSION_BUILD 25
  45 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
  46              __stringify(DRV_VERSION_MINOR) "." \
  47              __stringify(DRV_VERSION_BUILD)    DRV_KERN
  48 const char i40e_driver_version_str[] = DRV_VERSION;
  49 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation.";
  50
  51 /* a bit of forward declarations */
  52 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
  53 static void i40e_handle_reset_warning(struct i40e_pf *pf);
  54 static int i40e_add_vsi(struct i40e_vsi *vsi);
  55 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
  56 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
  57 static int i40e_setup_misc_vector(struct i40e_pf *pf);
  58 static void i40e_determine_queue_usage(struct i40e_pf *pf);
  59 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
  60 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
  61 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
  62
  63 /* i40e_pci_tbl - PCI Device ID Table
  64  *
  65  * Last entry must be all 0s
  66  *
  67  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  68  *   Class, Class Mask, private data (not used) }
  69  */
  70 static const struct pci_device_id i40e_pci_tbl[] = {
  71         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
  72         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
  73         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
  74         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
  75         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
  76         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
  77         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
  78         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
  79         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
  80         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  81         {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
  82         {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
  83         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
  84         {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
  85         {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
  86         {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
  87         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
  88         {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
  89         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
  90         {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
  91         /* required last entry */
  92         {0, }
  93 };
  94 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
  95
  96 #define I40E_MAX_VF_COUNT 128
  97 static int debug = -1;
  98 module_param(debug, uint, 0);
  99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
 100
 101 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
 102 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
 103 MODULE_LICENSE("GPL");
 104 MODULE_VERSION(DRV_VERSION);
 105
 106 static struct workqueue_struct *i40e_wq;
 107
 108 /**
 109  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
 110  * @hw:   pointer to the HW structure
 111  * @mem:  ptr to mem struct to fill out
 112  * @size: size of memory requested
 113  * @alignment: what to align the allocation to
 114  **/
 115 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
 116                             u64 size, u32 alignment)
 117 {
 118         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 119
 120         mem->size = ALIGN(size, alignment);
 121         mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size,
 122                                       &mem->pa, GFP_KERNEL);
 123         if (!mem->va)
 124                 return -ENOMEM;
 125
 126         return 0;
 127 }
 128
 129 /**
 130  * i40e_free_dma_mem_d - OS specific memory free for shared code
 131  * @hw:   pointer to the HW structure
 132  * @mem:  ptr to mem struct to free
 133  **/
 134 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
 135 {
 136         struct i40e_pf *pf = (struct i40e_pf *)hw->back;
 137
 138         dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
 139         mem->va = NULL;
 140         mem->pa = 0;
 141         mem->size = 0;
 142
 143         return 0;
 144 }
 145
 146 /**
 147  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
 148  * @hw:   pointer to the HW structure
 149  * @mem:  ptr to mem struct to fill out
 150  * @size: size of memory requested
 151  **/
 152 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
 153                              u32 size)
 154 {
 155         mem->size = size;
 156         mem->va = kzalloc(size, GFP_KERNEL);
 157
 158         if (!mem->va)
 159                 return -ENOMEM;
 160
 161         return 0;
 162 }
 163
 164 /**
 165  * i40e_free_virt_mem_d - OS specific memory free for shared code
 166  * @hw:   pointer to the HW structure
 167  * @mem:  ptr to mem struct to free
 168  **/
 169 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
 170 {
 171         /* it's ok to kfree a NULL pointer */
 172         kfree(mem->va);
 173         mem->va = NULL;
 174         mem->size = 0;
 175
 176         return 0;
 177 }
 178
 179 /**
 180  * i40e_get_lump - find a lump of free generic resource
 181  * @pf: board private structure
 182  * @pile: the pile of resource to search
 183  * @needed: the number of items needed
 184  * @id: an owner id to stick on the items assigned
 185  *
 186  * Returns the base item index of the lump, or negative for error
 187  *
 188  * The search_hint trick and lack of advanced fit-finding only work
 189  * because we're highly likely to have all the same size lump requests.
 190  * Linear search time and any fragmentation should be minimal.
 191  **/
 192 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
 193                          u16 needed, u16 id)
 194 {
 195         int ret = -ENOMEM;
 196         int i, j;
 197
 198         if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
 199                 dev_info(&pf->pdev->dev,
 200                          "param err: pile=%p needed=%d id=0x%04x\n",
 201                          pile, needed, id);
 202                 return -EINVAL;
 203         }
 204
 205         /* start the linear search with an imperfect hint */
 206         i = pile->search_hint;
 207         while (i < pile->num_entries) {
 208                 /* skip already allocated entries */
 209                 if (pile->list[i] & I40E_PILE_VALID_BIT) {
 210                         i++;
 211                         continue;
 212                 }
 213
 214                 /* do we have enough in this lump? */
 215                 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
 216                         if (pile->list[i+j] & I40E_PILE_VALID_BIT)
 217                                 break;
 218                 }
 219
 220                 if (j == needed) {
 221                         /* there was enough, so assign it to the requestor */
 222                         for (j = 0; j < needed; j++)
 223                                 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
 224                         ret = i;
 225                         pile->search_hint = i + j;
 226                         break;
 227                 }
 228
 229                 /* not enough, so skip over it and continue looking */
 230                 i += j;
 231         }
 232
 233         return ret;
 234 }
 235
 236 /**
 237  * i40e_put_lump - return a lump of generic resource
 238  * @pile: the pile of resource to search
 239  * @index: the base item index
 240  * @id: the owner id of the items assigned
 241  *
 242  * Returns the count of items in the lump
 243  **/
 244 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
 245 {
 246         int valid_id = (id | I40E_PILE_VALID_BIT);
 247         int count = 0;
 248         int i;
 249
 250         if (!pile || index >= pile->num_entries)
 251                 return -EINVAL;
 252
 253         for (i = index;
 254              i < pile->num_entries && pile->list[i] == valid_id;
 255              i++) {
 256                 pile->list[i] = 0;
 257                 count++;
 258         }
 259
 260         if (count && index < pile->search_hint)
 261                 pile->search_hint = index;
 262
 263         return count;
 264 }
 265
 266 /**
 267  * i40e_find_vsi_from_id - searches for the vsi with the given id
 268  * @pf - the pf structure to search for the vsi
 269  * @id - id of the vsi it is searching for
 270  **/
 271 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
 272 {
 273         int i;
 274
 275         for (i = 0; i < pf->num_alloc_vsi; i++)
 276                 if (pf->vsi[i] && (pf->vsi[i]->id == id))
 277                         return pf->vsi[i];
 278
 279         return NULL;
 280 }
 281
 282 /**
 283  * i40e_service_event_schedule - Schedule the service task to wake up
 284  * @pf: board private structure
 285  *
 286  * If not already scheduled, this puts the task into the work queue
 287  **/
 288 void i40e_service_event_schedule(struct i40e_pf *pf)
 289 {
 290         if (!test_bit(__I40E_DOWN, &pf->state) &&
 291             !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
 292                 queue_work(i40e_wq, &pf->service_task);
 293 }
 294
 295 /**
 296  * i40e_tx_timeout - Respond to a Tx Hang
 297  * @netdev: network interface device structure
 298  *
 299  * If any port has noticed a Tx timeout, it is likely that the whole
 300  * device is munged, not just the one netdev port, so go for the full
 301  * reset.
 302  **/
 303 #ifdef I40E_FCOE
 304 void i40e_tx_timeout(struct net_device *netdev)
 305 #else
 306 static void i40e_tx_timeout(struct net_device *netdev)
 307 #endif
 308 {
 309         struct i40e_netdev_priv *np = netdev_priv(netdev);
 310         struct i40e_vsi *vsi = np->vsi;
 311         struct i40e_pf *pf = vsi->back;
 312         struct i40e_ring *tx_ring = NULL;
 313         unsigned int i, hung_queue = 0;
 314         u32 head, val;
 315
 316         pf->tx_timeout_count++;
 317
 318         /* find the stopped queue the same way the stack does */
 319         for (i = 0; i < netdev->num_tx_queues; i++) {
 320                 struct netdev_queue *q;
 321                 unsigned long trans_start;
 322
 323                 q = netdev_get_tx_queue(netdev, i);
 324                 trans_start = q->trans_start;
 325                 if (netif_xmit_stopped(q) &&
 326                     time_after(jiffies,
 327                                (trans_start + netdev->watchdog_timeo))) {
 328                         hung_queue = i;
 329                         break;
 330                 }
 331         }
 332
 333         if (i == netdev->num_tx_queues) {
 334                 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
 335         } else {
 336                 /* now that we have an index, find the tx_ring struct */
 337                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 338                         if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
 339                                 if (hung_queue ==
 340                                     vsi->tx_rings[i]->queue_index) {
 341                                         tx_ring = vsi->tx_rings[i];
 342                                         break;
 343                                 }
 344                         }
 345                 }
 346         }
 347
 348         if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
 349                 pf->tx_timeout_recovery_level = 1;  /* reset after some time */
 350         else if (time_before(jiffies,
 351                       (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
 352                 return;   /* don't do any new action before the next timeout */
 353
 354         if (tx_ring) {
 355                 head = i40e_get_head(tx_ring);
 356                 /* Read interrupt register */
 357                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
 358                         val = rd32(&pf->hw,
 359                              I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
 360                                                 tx_ring->vsi->base_vector - 1));
 361                 else
 362                         val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
 363
 364                 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
 365                             vsi->seid, hung_queue, tx_ring->next_to_clean,
 366                             head, tx_ring->next_to_use,
 367                             readl(tx_ring->tail), val);
 368         }
 369
 370         pf->tx_timeout_last_recovery = jiffies;
 371         netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
 372                     pf->tx_timeout_recovery_level, hung_queue);
 373
 374         switch (pf->tx_timeout_recovery_level) {
 375         case 1:
 376                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
 377                 break;
 378         case 2:
 379                 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
 380                 break;
 381         case 3:
 382                 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
 383                 break;
 384         default:
 385                 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
 386                 break;
 387         }
 388
 389         i40e_service_event_schedule(pf);
 390         pf->tx_timeout_recovery_level++;
 391 }
 392
 393 /**
 394  * i40e_get_vsi_stats_struct - Get System Network Statistics
 395  * @vsi: the VSI we care about
 396  *
 397  * Returns the address of the device statistics structure.
 398  * The statistics are actually updated from the service task.
 399  **/
 400 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
 401 {
 402         return &vsi->net_stats;
 403 }
 404
 405 /**
 406  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
 407  * @netdev: network interface device structure
 408  *
 409  * Returns the address of the device statistics structure.
 410  * The statistics are actually updated from the service task.
 411  **/
 412 #ifndef I40E_FCOE
 413 static
 414 #endif
 415 void i40e_get_netdev_stats_struct(struct net_device *netdev,
 416                                   struct rtnl_link_stats64 *stats)
 417 {
 418         struct i40e_netdev_priv *np = netdev_priv(netdev);
 419         struct i40e_ring *tx_ring, *rx_ring;
 420         struct i40e_vsi *vsi = np->vsi;
 421         struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
 422         int i;
 423
 424         if (test_bit(__I40E_DOWN, &vsi->state))
 425                 return;
 426
 427         if (!vsi->tx_rings)
 428                 return;
 429
 430         rcu_read_lock();
 431         for (i = 0; i < vsi->num_queue_pairs; i++) {
 432                 u64 bytes, packets;
 433                 unsigned int start;
 434
 435                 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]);
 436                 if (!tx_ring)
 437                         continue;
 438
 439                 do {
 440                         start = u64_stats_fetch_begin_irq(&tx_ring->syncp);
 441                         packets = tx_ring->stats.packets;
 442                         bytes   = tx_ring->stats.bytes;
 443                 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start));
 444
 445                 stats->tx_packets += packets;
 446                 stats->tx_bytes   += bytes;
 447                 rx_ring = &tx_ring[1];
 448
 449                 do {
 450                         start = u64_stats_fetch_begin_irq(&rx_ring->syncp);
 451                         packets = rx_ring->stats.packets;
 452                         bytes   = rx_ring->stats.bytes;
 453                 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start));
 454
 455                 stats->rx_packets += packets;
 456                 stats->rx_bytes   += bytes;
 457         }
 458         rcu_read_unlock();
 459
 460         /* following stats updated by i40e_watchdog_subtask() */
 461         stats->multicast        = vsi_stats->multicast;
 462         stats->tx_errors        = vsi_stats->tx_errors;
 463         stats->tx_dropped       = vsi_stats->tx_dropped;
 464         stats->rx_errors        = vsi_stats->rx_errors;
 465         stats->rx_dropped       = vsi_stats->rx_dropped;
 466         stats->rx_crc_errors    = vsi_stats->rx_crc_errors;
 467         stats->rx_length_errors = vsi_stats->rx_length_errors;
 468 }
 469
 470 /**
 471  * i40e_vsi_reset_stats - Resets all stats of the given vsi
 472  * @vsi: the VSI to have its stats reset
 473  **/
 474 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
 475 {
 476         struct rtnl_link_stats64 *ns;
 477         int i;
 478
 479         if (!vsi)
 480                 return;
 481
 482         ns = i40e_get_vsi_stats_struct(vsi);
 483         memset(ns, 0, sizeof(*ns));
 484         memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
 485         memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
 486         memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
 487         if (vsi->rx_rings && vsi->rx_rings[0]) {
 488                 for (i = 0; i < vsi->num_queue_pairs; i++) {
 489                         memset(&vsi->rx_rings[i]->stats, 0,
 490                                sizeof(vsi->rx_rings[i]->stats));
 491                         memset(&vsi->rx_rings[i]->rx_stats, 0,
 492                                sizeof(vsi->rx_rings[i]->rx_stats));
 493                         memset(&vsi->tx_rings[i]->stats, 0,
 494                                sizeof(vsi->tx_rings[i]->stats));
 495                         memset(&vsi->tx_rings[i]->tx_stats, 0,
 496                                sizeof(vsi->tx_rings[i]->tx_stats));
 497                 }
 498         }
 499         vsi->stat_offsets_loaded = false;
 500 }
 501
 502 /**
 503  * i40e_pf_reset_stats - Reset all of the stats for the given PF
 504  * @pf: the PF to be reset
 505  **/
 506 void i40e_pf_reset_stats(struct i40e_pf *pf)
 507 {
 508         int i;
 509
 510         memset(&pf->stats, 0, sizeof(pf->stats));
 511         memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
 512         pf->stat_offsets_loaded = false;
 513
 514         for (i = 0; i < I40E_MAX_VEB; i++) {
 515                 if (pf->veb[i]) {
 516                         memset(&pf->veb[i]->stats, 0,
 517                                sizeof(pf->veb[i]->stats));
 518                         memset(&pf->veb[i]->stats_offsets, 0,
 519                                sizeof(pf->veb[i]->stats_offsets));
 520                         pf->veb[i]->stat_offsets_loaded = false;
 521                 }
 522         }
 523         pf->hw_csum_rx_error = 0;
 524 }
 525
 526 /**
 527  * i40e_stat_update48 - read and update a 48 bit stat from the chip
 528  * @hw: ptr to the hardware info
 529  * @hireg: the high 32 bit reg to read
 530  * @loreg: the low 32 bit reg to read
 531  * @offset_loaded: has the initial offset been loaded yet
 532  * @offset: ptr to current offset value
 533  * @stat: ptr to the stat
 534  *
 535  * Since the device stats are not reset at PFReset, they likely will not
 536  * be zeroed when the driver starts.  We'll save the first values read
 537  * and use them as offsets to be subtracted from the raw values in order
 538  * to report stats that count from zero.  In the process, we also manage
 539  * the potential roll-over.
 540  **/
 541 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
 542                                bool offset_loaded, u64 *offset, u64 *stat)
 543 {
 544         u64 new_data;
 545
 546         if (hw->device_id == I40E_DEV_ID_QEMU) {
 547                 new_data = rd32(hw, loreg);
 548                 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
 549         } else {
 550                 new_data = rd64(hw, loreg);
 551         }
 552         if (!offset_loaded)
 553                 *offset = new_data;
 554         if (likely(new_data >= *offset))
 555                 *stat = new_data - *offset;
 556         else
 557                 *stat = (new_data + BIT_ULL(48)) - *offset;
 558         *stat &= 0xFFFFFFFFFFFFULL;
 559 }
 560
 561 /**
 562  * i40e_stat_update32 - read and update a 32 bit stat from the chip
 563  * @hw: ptr to the hardware info
 564  * @reg: the hw reg to read
 565  * @offset_loaded: has the initial offset been loaded yet
 566  * @offset: ptr to current offset value
 567  * @stat: ptr to the stat
 568  **/
 569 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
 570                                bool offset_loaded, u64 *offset, u64 *stat)
 571 {
 572         u32 new_data;
 573
 574         new_data = rd32(hw, reg);
 575         if (!offset_loaded)
 576                 *offset = new_data;
 577         if (likely(new_data >= *offset))
 578                 *stat = (u32)(new_data - *offset);
 579         else
 580                 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
 581 }
 582
 583 /**
 584  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
 585  * @vsi: the VSI to be updated
 586  **/
 587 void i40e_update_eth_stats(struct i40e_vsi *vsi)
 588 {
 589         int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
 590         struct i40e_pf *pf = vsi->back;
 591         struct i40e_hw *hw = &pf->hw;
 592         struct i40e_eth_stats *oes;
 593         struct i40e_eth_stats *es;     /* device's eth stats */
 594
 595         es = &vsi->eth_stats;
 596         oes = &vsi->eth_stats_offsets;
 597
 598         /* Gather up the stats that the hw collects */
 599         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 600                            vsi->stat_offsets_loaded,
 601                            &oes->tx_errors, &es->tx_errors);
 602         i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
 603                            vsi->stat_offsets_loaded,
 604                            &oes->rx_discards, &es->rx_discards);
 605         i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
 606                            vsi->stat_offsets_loaded,
 607                            &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
 608         i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
 609                            vsi->stat_offsets_loaded,
 610                            &oes->tx_errors, &es->tx_errors);
 611
 612         i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
 613                            I40E_GLV_GORCL(stat_idx),
 614                            vsi->stat_offsets_loaded,
 615                            &oes->rx_bytes, &es->rx_bytes);
 616         i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
 617                            I40E_GLV_UPRCL(stat_idx),
 618                            vsi->stat_offsets_loaded,
 619                            &oes->rx_unicast, &es->rx_unicast);
 620         i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
 621                            I40E_GLV_MPRCL(stat_idx),
 622                            vsi->stat_offsets_loaded,
 623                            &oes->rx_multicast, &es->rx_multicast);
 624         i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
 625                            I40E_GLV_BPRCL(stat_idx),
 626                            vsi->stat_offsets_loaded,
 627                            &oes->rx_broadcast, &es->rx_broadcast);
 628
 629         i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
 630                            I40E_GLV_GOTCL(stat_idx),
 631                            vsi->stat_offsets_loaded,
 632                            &oes->tx_bytes, &es->tx_bytes);
 633         i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
 634                            I40E_GLV_UPTCL(stat_idx),
 635                            vsi->stat_offsets_loaded,
 636                            &oes->tx_unicast, &es->tx_unicast);
 637         i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
 638                            I40E_GLV_MPTCL(stat_idx),
 639                            vsi->stat_offsets_loaded,
 640                            &oes->tx_multicast, &es->tx_multicast);
 641         i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
 642                            I40E_GLV_BPTCL(stat_idx),
 643                            vsi->stat_offsets_loaded,
 644                            &oes->tx_broadcast, &es->tx_broadcast);
 645         vsi->stat_offsets_loaded = true;
 646 }
 647
 648 /**
 649  * i40e_update_veb_stats - Update Switch component statistics
 650  * @veb: the VEB being updated
 651  **/
 652 static void i40e_update_veb_stats(struct i40e_veb *veb)
 653 {
 654         struct i40e_pf *pf = veb->pf;
 655         struct i40e_hw *hw = &pf->hw;
 656         struct i40e_eth_stats *oes;
 657         struct i40e_eth_stats *es;     /* device's eth stats */
 658         struct i40e_veb_tc_stats *veb_oes;
 659         struct i40e_veb_tc_stats *veb_es;
 660         int i, idx = 0;
 661
 662         idx = veb->stats_idx;
 663         es = &veb->stats;
 664         oes = &veb->stats_offsets;
 665         veb_es = &veb->tc_stats;
 666         veb_oes = &veb->tc_stats_offsets;
 667
 668         /* Gather up the stats that the hw collects */
 669         i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
 670                            veb->stat_offsets_loaded,
 671                            &oes->tx_discards, &es->tx_discards);
 672         if (hw->revision_id > 0)
 673                 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
 674                                    veb->stat_offsets_loaded,
 675                                    &oes->rx_unknown_protocol,
 676                                    &es->rx_unknown_protocol);
 677         i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
 678                            veb->stat_offsets_loaded,
 679                            &oes->rx_bytes, &es->rx_bytes);
 680         i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
 681                            veb->stat_offsets_loaded,
 682                            &oes->rx_unicast, &es->rx_unicast);
 683         i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
 684                            veb->stat_offsets_loaded,
 685                            &oes->rx_multicast, &es->rx_multicast);
 686         i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
 687                            veb->stat_offsets_loaded,
 688                            &oes->rx_broadcast, &es->rx_broadcast);
 689
 690         i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
 691                            veb->stat_offsets_loaded,
 692                            &oes->tx_bytes, &es->tx_bytes);
 693         i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
 694                            veb->stat_offsets_loaded,
 695                            &oes->tx_unicast, &es->tx_unicast);
 696         i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
 697                            veb->stat_offsets_loaded,
 698                            &oes->tx_multicast, &es->tx_multicast);
 699         i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
 700                            veb->stat_offsets_loaded,
 701                            &oes->tx_broadcast, &es->tx_broadcast);
 702         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
 703                 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
 704                                    I40E_GLVEBTC_RPCL(i, idx),
 705                                    veb->stat_offsets_loaded,
 706                                    &veb_oes->tc_rx_packets[i],
 707                                    &veb_es->tc_rx_packets[i]);
 708                 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
 709                                    I40E_GLVEBTC_RBCL(i, idx),
 710                                    veb->stat_offsets_loaded,
 711                                    &veb_oes->tc_rx_bytes[i],
 712                                    &veb_es->tc_rx_bytes[i]);
 713                 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
 714                                    I40E_GLVEBTC_TPCL(i, idx),
 715                                    veb->stat_offsets_loaded,
 716                                    &veb_oes->tc_tx_packets[i],
 717                                    &veb_es->tc_tx_packets[i]);
 718                 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
 719                                    I40E_GLVEBTC_TBCL(i, idx),
 720                                    veb->stat_offsets_loaded,
 721                                    &veb_oes->tc_tx_bytes[i],
 722                                    &veb_es->tc_tx_bytes[i]);
 723         }
 724         veb->stat_offsets_loaded = true;
 725 }
 726
 727 #ifdef I40E_FCOE
 728 /**
 729  * i40e_update_fcoe_stats - Update FCoE-specific ethernet statistics counters.
 730  * @vsi: the VSI that is capable of doing FCoE
 731  **/
 732 static void i40e_update_fcoe_stats(struct i40e_vsi *vsi)
 733 {
 734         struct i40e_pf *pf = vsi->back;
 735         struct i40e_hw *hw = &pf->hw;
 736         struct i40e_fcoe_stats *ofs;
 737         struct i40e_fcoe_stats *fs;     /* device's eth stats */
 738         int idx;
 739
 740         if (vsi->type != I40E_VSI_FCOE)
 741                 return;
 742
 743         idx = hw->pf_id + I40E_FCOE_PF_STAT_OFFSET;
 744         fs = &vsi->fcoe_stats;
 745         ofs = &vsi->fcoe_stats_offsets;
 746
 747         i40e_stat_update32(hw, I40E_GL_FCOEPRC(idx),
 748                            vsi->fcoe_stat_offsets_loaded,
 749                            &ofs->rx_fcoe_packets, &fs->rx_fcoe_packets);
 750         i40e_stat_update48(hw, I40E_GL_FCOEDWRCH(idx), I40E_GL_FCOEDWRCL(idx),
 751                            vsi->fcoe_stat_offsets_loaded,
 752                            &ofs->rx_fcoe_dwords, &fs->rx_fcoe_dwords);
 753         i40e_stat_update32(hw, I40E_GL_FCOERPDC(idx),
 754                            vsi->fcoe_stat_offsets_loaded,
 755                            &ofs->rx_fcoe_dropped, &fs->rx_fcoe_dropped);
 756         i40e_stat_update32(hw, I40E_GL_FCOEPTC(idx),
 757                            vsi->fcoe_stat_offsets_loaded,
 758                            &ofs->tx_fcoe_packets, &fs->tx_fcoe_packets);
 759         i40e_stat_update48(hw, I40E_GL_FCOEDWTCH(idx), I40E_GL_FCOEDWTCL(idx),
 760                            vsi->fcoe_stat_offsets_loaded,
 761                            &ofs->tx_fcoe_dwords, &fs->tx_fcoe_dwords);
 762         i40e_stat_update32(hw, I40E_GL_FCOECRC(idx),
 763                            vsi->fcoe_stat_offsets_loaded,
 764                            &ofs->fcoe_bad_fccrc, &fs->fcoe_bad_fccrc);
 765         i40e_stat_update32(hw, I40E_GL_FCOELAST(idx),
 766                            vsi->fcoe_stat_offsets_loaded,
 767                            &ofs->fcoe_last_error, &fs->fcoe_last_error);
 768         i40e_stat_update32(hw, I40E_GL_FCOEDDPC(idx),
 769                            vsi->fcoe_stat_offsets_loaded,
 770                            &ofs->fcoe_ddp_count, &fs->fcoe_ddp_count);
 771
 772         vsi->fcoe_stat_offsets_loaded = true;
 773 }
 774
 775 #endif
 776 /**
 777  * i40e_update_vsi_stats - Update the vsi statistics counters.
 778  * @vsi: the VSI to be updated
 779  *
 780  * There are a few instances where we store the same stat in a
 781  * couple of different structs.  This is partly because we have
 782  * the netdev stats that need to be filled out, which is slightly
 783  * different from the "eth_stats" defined by the chip and used in
 784  * VF communications.  We sort it out here.
 785  **/
 786 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
 787 {
 788         struct i40e_pf *pf = vsi->back;
 789         struct rtnl_link_stats64 *ons;
 790         struct rtnl_link_stats64 *ns;   /* netdev stats */
 791         struct i40e_eth_stats *oes;
 792         struct i40e_eth_stats *es;     /* device's eth stats */
 793         u32 tx_restart, tx_busy;
 794         u64 tx_lost_interrupt;
 795         struct i40e_ring *p;
 796         u32 rx_page, rx_buf;
 797         u64 bytes, packets;
 798         unsigned int start;
 799         u64 tx_linearize;
 800         u64 tx_force_wb;
 801         u64 rx_p, rx_b;
 802         u64 tx_p, tx_b;
 803         u16 q;
 804
 805         if (test_bit(__I40E_DOWN, &vsi->state) ||
 806             test_bit(__I40E_CONFIG_BUSY, &pf->state))
 807                 return;
 808
 809         ns = i40e_get_vsi_stats_struct(vsi);
 810         ons = &vsi->net_stats_offsets;
 811         es = &vsi->eth_stats;
 812         oes = &vsi->eth_stats_offsets;
 813
 814         /* Gather up the netdev and vsi stats that the driver collects
 815          * on the fly during packet processing
 816          */
 817         rx_b = rx_p = 0;
 818         tx_b = tx_p = 0;
 819         tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
 820         tx_lost_interrupt = 0;
 821         rx_page = 0;
 822         rx_buf = 0;
 823         rcu_read_lock();
 824         for (q = 0; q < vsi->num_queue_pairs; q++) {
 825                 /* locate Tx ring */
 826                 p = ACCESS_ONCE(vsi->tx_rings[q]);
 827
 828                 do {
 829                         start = u64_stats_fetch_begin_irq(&p->syncp);
 830                         packets = p->stats.packets;
 831                         bytes = p->stats.bytes;
 832                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 833                 tx_b += bytes;
 834                 tx_p += packets;
 835                 tx_restart += p->tx_stats.restart_queue;
 836                 tx_busy += p->tx_stats.tx_busy;
 837                 tx_linearize += p->tx_stats.tx_linearize;
 838                 tx_force_wb += p->tx_stats.tx_force_wb;
 839                 tx_lost_interrupt += p->tx_stats.tx_lost_interrupt;
 840
 841                 /* Rx queue is part of the same block as Tx queue */
 842                 p = &p[1];
 843                 do {
 844                         start = u64_stats_fetch_begin_irq(&p->syncp);
 845                         packets = p->stats.packets;
 846                         bytes = p->stats.bytes;
 847                 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
 848                 rx_b += bytes;
 849                 rx_p += packets;
 850                 rx_buf += p->rx_stats.alloc_buff_failed;
 851                 rx_page += p->rx_stats.alloc_page_failed;
 852         }
 853         rcu_read_unlock();
 854         vsi->tx_restart = tx_restart;
 855         vsi->tx_busy = tx_busy;
 856         vsi->tx_linearize = tx_linearize;
 857         vsi->tx_force_wb = tx_force_wb;
 858         vsi->tx_lost_interrupt = tx_lost_interrupt;
 859         vsi->rx_page_failed = rx_page;
 860         vsi->rx_buf_failed = rx_buf;
 861
 862         ns->rx_packets = rx_p;
 863         ns->rx_bytes = rx_b;
 864         ns->tx_packets = tx_p;
 865         ns->tx_bytes = tx_b;
 866
 867         /* update netdev stats from eth stats */
 868         i40e_update_eth_stats(vsi);
 869         ons->tx_errors = oes->tx_errors;
 870         ns->tx_errors = es->tx_errors;
 871         ons->multicast = oes->rx_multicast;
 872         ns->multicast = es->rx_multicast;
 873         ons->rx_dropped = oes->rx_discards;
 874         ns->rx_dropped = es->rx_discards;
 875         ons->tx_dropped = oes->tx_discards;
 876         ns->tx_dropped = es->tx_discards;
 877
 878         /* pull in a couple PF stats if this is the main vsi */
 879         if (vsi == pf->vsi[pf->lan_vsi]) {
 880                 ns->rx_crc_errors = pf->stats.crc_errors;
 881                 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
 882                 ns->rx_length_errors = pf->stats.rx_length_errors;
 883         }
 884 }
 885
 886 /**
 887  * i40e_update_pf_stats - Update the PF statistics counters.
 888  * @pf: the PF to be updated
 889  **/
 890 static void i40e_update_pf_stats(struct i40e_pf *pf)
 891 {
 892         struct i40e_hw_port_stats *osd = &pf->stats_offsets;
 893         struct i40e_hw_port_stats *nsd = &pf->stats;
 894         struct i40e_hw *hw = &pf->hw;
 895         u32 val;
 896         int i;
 897
 898         i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
 899                            I40E_GLPRT_GORCL(hw->port),
 900                            pf->stat_offsets_loaded,
 901                            &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
 902         i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
 903                            I40E_GLPRT_GOTCL(hw->port),
 904                            pf->stat_offsets_loaded,
 905                            &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
 906         i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
 907                            pf->stat_offsets_loaded,
 908                            &osd->eth.rx_discards,
 909                            &nsd->eth.rx_discards);
 910         i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
 911                            I40E_GLPRT_UPRCL(hw->port),
 912                            pf->stat_offsets_loaded,
 913                            &osd->eth.rx_unicast,
 914                            &nsd->eth.rx_unicast);
 915         i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
 916                            I40E_GLPRT_MPRCL(hw->port),
 917                            pf->stat_offsets_loaded,
 918                            &osd->eth.rx_multicast,
 919                            &nsd->eth.rx_multicast);
 920         i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
 921                            I40E_GLPRT_BPRCL(hw->port),
 922                            pf->stat_offsets_loaded,
 923                            &osd->eth.rx_broadcast,
 924                            &nsd->eth.rx_broadcast);
 925         i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
 926                            I40E_GLPRT_UPTCL(hw->port),
 927                            pf->stat_offsets_loaded,
 928                            &osd->eth.tx_unicast,
 929                            &nsd->eth.tx_unicast);
 930         i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
 931                            I40E_GLPRT_MPTCL(hw->port),
 932                            pf->stat_offsets_loaded,
 933                            &osd->eth.tx_multicast,
 934                            &nsd->eth.tx_multicast);
 935         i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
 936                            I40E_GLPRT_BPTCL(hw->port),
 937                            pf->stat_offsets_loaded,
 938                            &osd->eth.tx_broadcast,
 939                            &nsd->eth.tx_broadcast);
 940
 941         i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
 942                            pf->stat_offsets_loaded,
 943                            &osd->tx_dropped_link_down,
 944                            &nsd->tx_dropped_link_down);
 945
 946         i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
 947                            pf->stat_offsets_loaded,
 948                            &osd->crc_errors, &nsd->crc_errors);
 949
 950         i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
 951                            pf->stat_offsets_loaded,
 952                            &osd->illegal_bytes, &nsd->illegal_bytes);
 953
 954         i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
 955                            pf->stat_offsets_loaded,
 956                            &osd->mac_local_faults,
 957                            &nsd->mac_local_faults);
 958         i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
 959                            pf->stat_offsets_loaded,
 960                            &osd->mac_remote_faults,
 961                            &nsd->mac_remote_faults);
 962
 963         i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
 964                            pf->stat_offsets_loaded,
 965                            &osd->rx_length_errors,
 966                            &nsd->rx_length_errors);
 967
 968         i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
 969                            pf->stat_offsets_loaded,
 970                            &osd->link_xon_rx, &nsd->link_xon_rx);
 971         i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
 972                            pf->stat_offsets_loaded,
 973                            &osd->link_xon_tx, &nsd->link_xon_tx);
 974         i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
 975                            pf->stat_offsets_loaded,
 976                            &osd->link_xoff_rx, &nsd->link_xoff_rx);
 977         i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
 978                            pf->stat_offsets_loaded,
 979                            &osd->link_xoff_tx, &nsd->link_xoff_tx);
 980
 981         for (i = 0; i < 8; i++) {
 982                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
 983                                    pf->stat_offsets_loaded,
 984                                    &osd->priority_xoff_rx[i],
 985                                    &nsd->priority_xoff_rx[i]);
 986                 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
 987                                    pf->stat_offsets_loaded,
 988                                    &osd->priority_xon_rx[i],
 989                                    &nsd->priority_xon_rx[i]);
 990                 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
 991                                    pf->stat_offsets_loaded,
 992                                    &osd->priority_xon_tx[i],
 993                                    &nsd->priority_xon_tx[i]);
 994                 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
 995                                    pf->stat_offsets_loaded,
 996                                    &osd->priority_xoff_tx[i],
 997                                    &nsd->priority_xoff_tx[i]);
 998                 i40e_stat_update32(hw,
 999                                    I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1000                                    pf->stat_offsets_loaded,
1001                                    &osd->priority_xon_2_xoff[i],
1002                                    &nsd->priority_xon_2_xoff[i]);
1003         }
1004
1005         i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1006                            I40E_GLPRT_PRC64L(hw->port),
1007                            pf->stat_offsets_loaded,
1008                            &osd->rx_size_64, &nsd->rx_size_64);
1009         i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1010                            I40E_GLPRT_PRC127L(hw->port),
1011                            pf->stat_offsets_loaded,
1012                            &osd->rx_size_127, &nsd->rx_size_127);
1013         i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1014                            I40E_GLPRT_PRC255L(hw->port),
1015                            pf->stat_offsets_loaded,
1016                            &osd->rx_size_255, &nsd->rx_size_255);
1017         i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1018                            I40E_GLPRT_PRC511L(hw->port),
1019                            pf->stat_offsets_loaded,
1020                            &osd->rx_size_511, &nsd->rx_size_511);
1021         i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1022                            I40E_GLPRT_PRC1023L(hw->port),
1023                            pf->stat_offsets_loaded,
1024                            &osd->rx_size_1023, &nsd->rx_size_1023);
1025         i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1026                            I40E_GLPRT_PRC1522L(hw->port),
1027                            pf->stat_offsets_loaded,
1028                            &osd->rx_size_1522, &nsd->rx_size_1522);
1029         i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1030                            I40E_GLPRT_PRC9522L(hw->port),
1031                            pf->stat_offsets_loaded,
1032                            &osd->rx_size_big, &nsd->rx_size_big);
1033
1034         i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1035                            I40E_GLPRT_PTC64L(hw->port),
1036                            pf->stat_offsets_loaded,
1037                            &osd->tx_size_64, &nsd->tx_size_64);
1038         i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1039                            I40E_GLPRT_PTC127L(hw->port),
1040                            pf->stat_offsets_loaded,
1041                            &osd->tx_size_127, &nsd->tx_size_127);
1042         i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1043                            I40E_GLPRT_PTC255L(hw->port),
1044                            pf->stat_offsets_loaded,
1045                            &osd->tx_size_255, &nsd->tx_size_255);
1046         i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1047                            I40E_GLPRT_PTC511L(hw->port),
1048                            pf->stat_offsets_loaded,
1049                            &osd->tx_size_511, &nsd->tx_size_511);
1050         i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1051                            I40E_GLPRT_PTC1023L(hw->port),
1052                            pf->stat_offsets_loaded,
1053                            &osd->tx_size_1023, &nsd->tx_size_1023);
1054         i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1055                            I40E_GLPRT_PTC1522L(hw->port),
1056                            pf->stat_offsets_loaded,
1057                            &osd->tx_size_1522, &nsd->tx_size_1522);
1058         i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1059                            I40E_GLPRT_PTC9522L(hw->port),
1060                            pf->stat_offsets_loaded,
1061                            &osd->tx_size_big, &nsd->tx_size_big);
1062
1063         i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1064                            pf->stat_offsets_loaded,
1065                            &osd->rx_undersize, &nsd->rx_undersize);
1066         i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1067                            pf->stat_offsets_loaded,
1068                            &osd->rx_fragments, &nsd->rx_fragments);
1069         i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1070                            pf->stat_offsets_loaded,
1071                            &osd->rx_oversize, &nsd->rx_oversize);
1072         i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1073                            pf->stat_offsets_loaded,
1074                            &osd->rx_jabber, &nsd->rx_jabber);
1075
1076         /* FDIR stats */
1077         i40e_stat_update32(hw,
1078                            I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(pf->hw.pf_id)),
1079                            pf->stat_offsets_loaded,
1080                            &osd->fd_atr_match, &nsd->fd_atr_match);
1081         i40e_stat_update32(hw,
1082                            I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(pf->hw.pf_id)),
1083                            pf->stat_offsets_loaded,
1084                            &osd->fd_sb_match, &nsd->fd_sb_match);
1085         i40e_stat_update32(hw,
1086                       I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id)),
1087                       pf->stat_offsets_loaded,
1088                       &osd->fd_atr_tunnel_match, &nsd->fd_atr_tunnel_match);
1089
1090         val = rd32(hw, I40E_PRTPM_EEE_STAT);
1091         nsd->tx_lpi_status =
1092                        (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1093                         I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1094         nsd->rx_lpi_status =
1095                        (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1096                         I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1097         i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1098                            pf->stat_offsets_loaded,
1099                            &osd->tx_lpi_count, &nsd->tx_lpi_count);
1100         i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1101                            pf->stat_offsets_loaded,
1102                            &osd->rx_lpi_count, &nsd->rx_lpi_count);
1103
1104         if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1105             !(pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED))
1106                 nsd->fd_sb_status = true;
1107         else
1108                 nsd->fd_sb_status = false;
1109
1110         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1111             !(pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED))
1112                 nsd->fd_atr_status = true;
1113         else
1114                 nsd->fd_atr_status = false;
1115
1116         pf->stat_offsets_loaded = true;
1117 }
1118
1119 /**
1120  * i40e_update_stats - Update the various statistics counters.
1121  * @vsi: the VSI to be updated
1122  *
1123  * Update the various stats for this VSI and its related entities.
1124  **/
1125 void i40e_update_stats(struct i40e_vsi *vsi)
1126 {
1127         struct i40e_pf *pf = vsi->back;
1128
1129         if (vsi == pf->vsi[pf->lan_vsi])
1130                 i40e_update_pf_stats(pf);
1131
1132         i40e_update_vsi_stats(vsi);
1133 #ifdef I40E_FCOE
1134         i40e_update_fcoe_stats(vsi);
1135 #endif
1136 }
1137
1138 /**
1139  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1140  * @vsi: the VSI to be searched
1141  * @macaddr: the MAC address
1142  * @vlan: the vlan
1143  *
1144  * Returns ptr to the filter object or NULL
1145  **/
1146 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1147                                                 const u8 *macaddr, s16 vlan)
1148 {
1149         struct i40e_mac_filter *f;
1150         u64 key;
1151
1152         if (!vsi || !macaddr)
1153                 return NULL;
1154
1155         key = i40e_addr_to_hkey(macaddr);
1156         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1157                 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1158                     (vlan == f->vlan))
1159                         return f;
1160         }
1161         return NULL;
1162 }
1163
1164 /**
1165  * i40e_find_mac - Find a mac addr in the macvlan filters list
1166  * @vsi: the VSI to be searched
1167  * @macaddr: the MAC address we are searching for
1168  *
1169  * Returns the first filter with the provided MAC address or NULL if
1170  * MAC address was not found
1171  **/
1172 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1173 {
1174         struct i40e_mac_filter *f;
1175         u64 key;
1176
1177         if (!vsi || !macaddr)
1178                 return NULL;
1179
1180         key = i40e_addr_to_hkey(macaddr);
1181         hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1182                 if ((ether_addr_equal(macaddr, f->macaddr)))
1183                         return f;
1184         }
1185         return NULL;
1186 }
1187
1188 /**
1189  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1190  * @vsi: the VSI to be searched
1191  *
1192  * Returns true if VSI is in vlan mode or false otherwise
1193  **/
1194 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1195 {
1196         /* If we have a PVID, always operate in VLAN mode */
1197         if (vsi->info.pvid)
1198                 return true;
1199
1200         /* We need to operate in VLAN mode whenever we have any filters with
1201          * a VLAN other than I40E_VLAN_ALL. We could check the table each
1202          * time, incurring search cost repeatedly. However, we can notice two
1203          * things:
1204          *
1205          * 1) the only place where we can gain a VLAN filter is in
1206          *    i40e_add_filter.
1207          *
1208          * 2) the only place where filters are actually removed is in
1209          *    i40e_sync_filters_subtask.
1210          *
1211          * Thus, we can simply use a boolean value, has_vlan_filters which we
1212          * will set to true when we add a VLAN filter in i40e_add_filter. Then
1213          * we have to perform the full search after deleting filters in
1214          * i40e_sync_filters_subtask, but we already have to search
1215          * filters here and can perform the check at the same time. This
1216          * results in avoiding embedding a loop for VLAN mode inside another
1217          * loop over all the filters, and should maintain correctness as noted
1218          * above.
1219          */
1220         return vsi->has_vlan_filter;
1221 }
1222
1223 /**
1224  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1225  * @vsi: the VSI to configure
1226  * @tmp_add_list: list of filters ready to be added
1227  * @tmp_del_list: list of filters ready to be deleted
1228  * @vlan_filters: the number of active VLAN filters
1229  *
1230  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1231  * behave as expected. If we have any active VLAN filters remaining or about
1232  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1233  * so that they only match against untagged traffic. If we no longer have any
1234  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1235  * so that they match against both tagged and untagged traffic. In this way,
1236  * we ensure that we correctly receive the desired traffic. This ensures that
1237  * when we have an active VLAN we will receive only untagged traffic and
1238  * traffic matching active VLANs. If we have no active VLANs then we will
1239  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1240  *
1241  * Finally, in a similar fashion, this function also corrects filters when
1242  * there is an active PVID assigned to this VSI.
1243  *
1244  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1245  *
1246  * This function is only expected to be called from within
1247  * i40e_sync_vsi_filters.
1248  *
1249  * NOTE: This function expects to be called while under the
1250  * mac_filter_hash_lock
1251  */
1252 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1253                                          struct hlist_head *tmp_add_list,
1254                                          struct hlist_head *tmp_del_list,
1255                                          int vlan_filters)
1256 {
1257         struct i40e_mac_filter *f, *add_head;
1258         struct hlist_node *h;
1259         int bkt, new_vlan;
1260
1261         /* To determine if a particular filter needs to be replaced we
1262          * have the three following conditions:
1263          *
1264          * a) if we have a PVID assigned, then all filters which are
1265          *    not marked as VLAN=PVID must be replaced with filters that
1266          *    are.
1267          * b) otherwise, if we have any active VLANS, all filters
1268          *    which are marked as VLAN=-1 must be replaced with
1269          *    filters marked as VLAN=0
1270          * c) finally, if we do not have any active VLANS, all filters
1271          *    which are marked as VLAN=0 must be replaced with filters
1272          *    marked as VLAN=-1
1273          */
1274
1275         /* Update the filters about to be added in place */
1276         hlist_for_each_entry(f, tmp_add_list, hlist) {
1277                 if (vsi->info.pvid && f->vlan != vsi->info.pvid)
1278                         f->vlan = vsi->info.pvid;
1279                 else if (vlan_filters && f->vlan == I40E_VLAN_ANY)
1280                         f->vlan = 0;
1281                 else if (!vlan_filters && f->vlan == 0)
1282                         f->vlan = I40E_VLAN_ANY;
1283         }
1284
1285         /* Update the remaining active filters */
1286         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1287                 /* Combine the checks for whether a filter needs to be changed
1288                  * and then determine the new VLAN inside the if block, in
1289                  * order to avoid duplicating code for adding the new filter
1290                  * then deleting the old filter.
1291                  */
1292                 if ((vsi->info.pvid && f->vlan != vsi->info.pvid) ||
1293                     (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1294                     (!vlan_filters && f->vlan == 0)) {
1295                         /* Determine the new vlan we will be adding */
1296                         if (vsi->info.pvid)
1297                                 new_vlan = vsi->info.pvid;
1298                         else if (vlan_filters)
1299                                 new_vlan = 0;
1300                         else
1301                                 new_vlan = I40E_VLAN_ANY;
1302
1303                         /* Create the new filter */
1304                         add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1305                         if (!add_head)
1306                                 return -ENOMEM;
1307
1308                         /* Put the replacement filter into the add list */
1309                         hash_del(&add_head->hlist);
1310                         hlist_add_head(&add_head->hlist, tmp_add_list);
1311
1312                         /* Put the original filter into the delete list */
1313                         f->state = I40E_FILTER_REMOVE;
1314                         hash_del(&f->hlist);
1315                         hlist_add_head(&f->hlist, tmp_del_list);
1316                 }
1317         }
1318
1319         vsi->has_vlan_filter = !!vlan_filters;
1320
1321         return 0;
1322 }
1323
1324 /**
1325  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1326  * @vsi: the PF Main VSI - inappropriate for any other VSI
1327  * @macaddr: the MAC address
1328  *
1329  * Remove whatever filter the firmware set up so the driver can manage
1330  * its own filtering intelligently.
1331  **/
1332 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1333 {
1334         struct i40e_aqc_remove_macvlan_element_data element;
1335         struct i40e_pf *pf = vsi->back;
1336
1337         /* Only appropriate for the PF main VSI */
1338         if (vsi->type != I40E_VSI_MAIN)
1339                 return;
1340
1341         memset(&element, 0, sizeof(element));
1342         ether_addr_copy(element.mac_addr, macaddr);
1343         element.vlan_tag = 0;
1344         /* Ignore error returns, some firmware does it this way... */
1345         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1346         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1347
1348         memset(&element, 0, sizeof(element));
1349         ether_addr_copy(element.mac_addr, macaddr);
1350         element.vlan_tag = 0;
1351         /* ...and some firmware does it this way. */
1352         element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1353                         I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1354         i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1355 }
1356
1357 /**
1358  * i40e_add_filter - Add a mac/vlan filter to the VSI
1359  * @vsi: the VSI to be searched
1360  * @macaddr: the MAC address
1361  * @vlan: the vlan
1362  *
1363  * Returns ptr to the filter object or NULL when no memory available.
1364  *
1365  * NOTE: This function is expected to be called with mac_filter_hash_lock
1366  * being held.
1367  **/
1368 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1369                                         const u8 *macaddr, s16 vlan)
1370 {
1371         struct i40e_mac_filter *f;
1372         u64 key;
1373
1374         if (!vsi || !macaddr)
1375                 return NULL;
1376
1377         f = i40e_find_filter(vsi, macaddr, vlan);
1378         if (!f) {
1379                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1380                 if (!f)
1381                         return NULL;
1382
1383                 /* Update the boolean indicating if we need to function in
1384                  * VLAN mode.
1385                  */
1386                 if (vlan >= 0)
1387                         vsi->has_vlan_filter = true;
1388
1389                 ether_addr_copy(f->macaddr, macaddr);
1390                 f->vlan = vlan;
1391                 /* If we're in overflow promisc mode, set the state directly
1392                  * to failed, so we don't bother to try sending the filter
1393                  * to the hardware.
1394                  */
1395                 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state))
1396                         f->state = I40E_FILTER_FAILED;
1397                 else
1398                         f->state = I40E_FILTER_NEW;
1399                 INIT_HLIST_NODE(&f->hlist);
1400
1401                 key = i40e_addr_to_hkey(macaddr);
1402                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1403
1404                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1405                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1406         }
1407
1408         /* If we're asked to add a filter that has been marked for removal, it
1409          * is safe to simply restore it to active state. __i40e_del_filter
1410          * will have simply deleted any filters which were previously marked
1411          * NEW or FAILED, so if it is currently marked REMOVE it must have
1412          * previously been ACTIVE. Since we haven't yet run the sync filters
1413          * task, just restore this filter to the ACTIVE state so that the
1414          * sync task leaves it in place
1415          */
1416         if (f->state == I40E_FILTER_REMOVE)
1417                 f->state = I40E_FILTER_ACTIVE;
1418
1419         return f;
1420 }
1421
1422 /**
1423  * __i40e_del_filter - Remove a specific filter from the VSI
1424  * @vsi: VSI to remove from
1425  * @f: the filter to remove from the list
1426  *
1427  * This function should be called instead of i40e_del_filter only if you know
1428  * the exact filter you will remove already, such as via i40e_find_filter or
1429  * i40e_find_mac.
1430  *
1431  * NOTE: This function is expected to be called with mac_filter_hash_lock
1432  * being held.
1433  * ANOTHER NOTE: This function MUST be called from within the context of
1434  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1435  * instead of list_for_each_entry().
1436  **/
1437 static void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1438 {
1439         if (!f)
1440                 return;
1441
1442         if ((f->state == I40E_FILTER_FAILED) ||
1443             (f->state == I40E_FILTER_NEW)) {
1444                 /* this one never got added by the FW. Just remove it,
1445                  * no need to sync anything.
1446                  */
1447                 hash_del(&f->hlist);
1448                 kfree(f);
1449         } else {
1450                 f->state = I40E_FILTER_REMOVE;
1451                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1452                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1453         }
1454 }
1455
1456 /**
1457  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1458  * @vsi: the VSI to be searched
1459  * @macaddr: the MAC address
1460  * @vlan: the VLAN
1461  *
1462  * NOTE: This function is expected to be called with mac_filter_hash_lock
1463  * being held.
1464  * ANOTHER NOTE: This function MUST be called from within the context of
1465  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1466  * instead of list_for_each_entry().
1467  **/
1468 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1469 {
1470         struct i40e_mac_filter *f;
1471
1472         if (!vsi || !macaddr)
1473                 return;
1474
1475         f = i40e_find_filter(vsi, macaddr, vlan);
1476         __i40e_del_filter(vsi, f);
1477 }
1478
1479 /**
1480  * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans
1481  * @vsi: the VSI to be searched
1482  * @macaddr: the mac address to be filtered
1483  *
1484  * Goes through all the macvlan filters and adds a macvlan filter for each
1485  * unique vlan that already exists. If a PVID has been assigned, instead only
1486  * add the macaddr to that VLAN.
1487  *
1488  * Returns last filter added on success, else NULL
1489  **/
1490 struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi,
1491                                              const u8 *macaddr)
1492 {
1493         struct i40e_mac_filter *f, *add = NULL;
1494         struct hlist_node *h;
1495         int bkt;
1496
1497         if (vsi->info.pvid)
1498                 return i40e_add_filter(vsi, macaddr,
1499                                        le16_to_cpu(vsi->info.pvid));
1500
1501         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1502                 if (f->state == I40E_FILTER_REMOVE)
1503                         continue;
1504                 add = i40e_add_filter(vsi, macaddr, f->vlan);
1505                 if (!add)
1506                         return NULL;
1507         }
1508
1509         return add;
1510 }
1511
1512 /**
1513  * i40e_del_mac_all_vlan - Remove a MAC filter from all VLANS
1514  * @vsi: the VSI to be searched
1515  * @macaddr: the mac address to be removed
1516  *
1517  * Removes a given MAC address from a VSI, regardless of VLAN
1518  *
1519  * Returns 0 for success, or error
1520  **/
1521 int i40e_del_mac_all_vlan(struct i40e_vsi *vsi, const u8 *macaddr)
1522 {
1523         struct i40e_mac_filter *f;
1524         struct hlist_node *h;
1525         bool found = false;
1526         int bkt;
1527
1528         WARN(!spin_is_locked(&vsi->mac_filter_hash_lock),
1529              "Missing mac_filter_hash_lock\n");
1530         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1531                 if (ether_addr_equal(macaddr, f->macaddr)) {
1532                         __i40e_del_filter(vsi, f);
1533                         found = true;
1534                 }
1535         }
1536
1537         if (found)
1538                 return 0;
1539         else
1540                 return -ENOENT;
1541 }
1542
1543 /**
1544  * i40e_set_mac - NDO callback to set mac address
1545  * @netdev: network interface device structure
1546  * @p: pointer to an address structure
1547  *
1548  * Returns 0 on success, negative on failure
1549  **/
1550 #ifdef I40E_FCOE
1551 int i40e_set_mac(struct net_device *netdev, void *p)
1552 #else
1553 static int i40e_set_mac(struct net_device *netdev, void *p)
1554 #endif
1555 {
1556         struct i40e_netdev_priv *np = netdev_priv(netdev);
1557         struct i40e_vsi *vsi = np->vsi;
1558         struct i40e_pf *pf = vsi->back;
1559         struct i40e_hw *hw = &pf->hw;
1560         struct sockaddr *addr = p;
1561
1562         if (!is_valid_ether_addr(addr->sa_data))
1563                 return -EADDRNOTAVAIL;
1564
1565         if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1566                 netdev_info(netdev, "already using mac address %pM\n",
1567                             addr->sa_data);
1568                 return 0;
1569         }
1570
1571         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
1572             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
1573                 return -EADDRNOTAVAIL;
1574
1575         if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1576                 netdev_info(netdev, "returning to hw mac address %pM\n",
1577                             hw->mac.addr);
1578         else
1579                 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1580
1581         spin_lock_bh(&vsi->mac_filter_hash_lock);
1582         i40e_del_mac_all_vlan(vsi, netdev->dev_addr);
1583         i40e_put_mac_in_vlan(vsi, addr->sa_data);
1584         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1585         ether_addr_copy(netdev->dev_addr, addr->sa_data);
1586         if (vsi->type == I40E_VSI_MAIN) {
1587                 i40e_status ret;
1588
1589                 ret = i40e_aq_mac_address_write(&vsi->back->hw,
1590                                                 I40E_AQC_WRITE_TYPE_LAA_WOL,
1591                                                 addr->sa_data, NULL);
1592                 if (ret)
1593                         netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1594                                     i40e_stat_str(hw, ret),
1595                                     i40e_aq_str(hw, hw->aq.asq_last_status));
1596         }
1597
1598         /* schedule our worker thread which will take care of
1599          * applying the new filter changes
1600          */
1601         i40e_service_event_schedule(vsi->back);
1602         return 0;
1603 }
1604
1605 /**
1606  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1607  * @vsi: the VSI being setup
1608  * @ctxt: VSI context structure
1609  * @enabled_tc: Enabled TCs bitmap
1610  * @is_add: True if called before Add VSI
1611  *
1612  * Setup VSI queue mapping for enabled traffic classes.
1613  **/
1614 #ifdef I40E_FCOE
1615 void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1616                               struct i40e_vsi_context *ctxt,
1617                               u8 enabled_tc,
1618                               bool is_add)
1619 #else
1620 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1621                                      struct i40e_vsi_context *ctxt,
1622                                      u8 enabled_tc,
1623                                      bool is_add)
1624 #endif
1625 {
1626         struct i40e_pf *pf = vsi->back;
1627         u16 sections = 0;
1628         u8 netdev_tc = 0;
1629         u16 numtc = 0;
1630         u16 qcount;
1631         u8 offset;
1632         u16 qmap;
1633         int i;
1634         u16 num_tc_qps = 0;
1635
1636         sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1637         offset = 0;
1638
1639         if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1640                 /* Find numtc from enabled TC bitmap */
1641                 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1642                         if (enabled_tc & BIT(i)) /* TC is enabled */
1643                                 numtc++;
1644                 }
1645                 if (!numtc) {
1646                         dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1647                         numtc = 1;
1648                 }
1649         } else {
1650                 /* At least TC0 is enabled in case of non-DCB case */
1651                 numtc = 1;
1652         }
1653
1654         vsi->tc_config.numtc = numtc;
1655         vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1656         /* Number of queues per enabled TC */
1657         qcount = vsi->alloc_queue_pairs;
1658
1659         num_tc_qps = qcount / numtc;
1660         num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf));
1661
1662         /* Setup queue offset/count for all TCs for given VSI */
1663         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1664                 /* See if the given TC is enabled for the given VSI */
1665                 if (vsi->tc_config.enabled_tc & BIT(i)) {
1666                         /* TC is enabled */
1667                         int pow, num_qps;
1668
1669                         switch (vsi->type) {
1670                         case I40E_VSI_MAIN:
1671                                 qcount = min_t(int, pf->alloc_rss_size,
1672                                                num_tc_qps);
1673                                 break;
1674 #ifdef I40E_FCOE
1675                         case I40E_VSI_FCOE:
1676                                 qcount = num_tc_qps;
1677                                 break;
1678 #endif
1679                         case I40E_VSI_FDIR:
1680                         case I40E_VSI_SRIOV:
1681                         case I40E_VSI_VMDQ2:
1682                         default:
1683                                 qcount = num_tc_qps;
1684                                 WARN_ON(i != 0);
1685                                 break;
1686                         }
1687                         vsi->tc_config.tc_info[i].qoffset = offset;
1688                         vsi->tc_config.tc_info[i].qcount = qcount;
1689
1690                         /* find the next higher power-of-2 of num queue pairs */
1691                         num_qps = qcount;
1692                         pow = 0;
1693                         while (num_qps && (BIT_ULL(pow) < qcount)) {
1694                                 pow++;
1695                                 num_qps >>= 1;
1696                         }
1697
1698                         vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1699                         qmap =
1700                             (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1701                             (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1702
1703                         offset += qcount;
1704                 } else {
1705                         /* TC is not enabled so set the offset to
1706                          * default queue and allocate one queue
1707                          * for the given TC.
1708                          */
1709                         vsi->tc_config.tc_info[i].qoffset = 0;
1710                         vsi->tc_config.tc_info[i].qcount = 1;
1711                         vsi->tc_config.tc_info[i].netdev_tc = 0;
1712
1713                         qmap = 0;
1714                 }
1715                 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1716         }
1717
1718         /* Set actual Tx/Rx queue pairs */
1719         vsi->num_queue_pairs = offset;
1720         if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1721                 if (vsi->req_queue_pairs > 0)
1722                         vsi->num_queue_pairs = vsi->req_queue_pairs;
1723                 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1724                         vsi->num_queue_pairs = pf->num_lan_msix;
1725         }
1726
1727         /* Scheduler section valid can only be set for ADD VSI */
1728         if (is_add) {
1729                 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1730
1731                 ctxt->info.up_enable_bits = enabled_tc;
1732         }
1733         if (vsi->type == I40E_VSI_SRIOV) {
1734                 ctxt->info.mapping_flags |=
1735                                      cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1736                 for (i = 0; i < vsi->num_queue_pairs; i++)
1737                         ctxt->info.queue_mapping[i] =
1738                                                cpu_to_le16(vsi->base_queue + i);
1739         } else {
1740                 ctxt->info.mapping_flags |=
1741                                         cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1742                 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1743         }
1744         ctxt->info.valid_sections |= cpu_to_le16(sections);
1745 }
1746
1747 /**
1748  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
1749  * @netdev: the netdevice
1750  * @addr: address to add
1751  *
1752  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1753  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1754  */
1755 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1756 {
1757         struct i40e_netdev_priv *np = netdev_priv(netdev);
1758         struct i40e_vsi *vsi = np->vsi;
1759         struct i40e_mac_filter *f;
1760
1761         if (i40e_is_vsi_in_vlan(vsi))
1762                 f = i40e_put_mac_in_vlan(vsi, addr);
1763         else
1764                 f = i40e_add_filter(vsi, addr, I40E_VLAN_ANY);
1765
1766         if (f)
1767                 return 0;
1768         else
1769                 return -ENOMEM;
1770 }
1771
1772 /**
1773  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1774  * @netdev: the netdevice
1775  * @addr: address to add
1776  *
1777  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1778  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1779  */
1780 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1781 {
1782         struct i40e_netdev_priv *np = netdev_priv(netdev);
1783         struct i40e_vsi *vsi = np->vsi;
1784
1785         if (i40e_is_vsi_in_vlan(vsi))
1786                 i40e_del_mac_all_vlan(vsi, addr);
1787         else
1788                 i40e_del_filter(vsi, addr, I40E_VLAN_ANY);
1789
1790         return 0;
1791 }
1792
1793 /**
1794  * i40e_set_rx_mode - NDO callback to set the netdev filters
1795  * @netdev: network interface device structure
1796  **/
1797 #ifdef I40E_FCOE
1798 void i40e_set_rx_mode(struct net_device *netdev)
1799 #else
1800 static void i40e_set_rx_mode(struct net_device *netdev)
1801 #endif
1802 {
1803         struct i40e_netdev_priv *np = netdev_priv(netdev);
1804         struct i40e_vsi *vsi = np->vsi;
1805
1806         spin_lock_bh(&vsi->mac_filter_hash_lock);
1807
1808         __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1809         __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1810
1811         spin_unlock_bh(&vsi->mac_filter_hash_lock);
1812
1813         /* check for other flag changes */
1814         if (vsi->current_netdev_flags != vsi->netdev->flags) {
1815                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1816                 vsi->back->flags |= I40E_FLAG_FILTER_SYNC;
1817         }
1818
1819         /* schedule our worker thread which will take care of
1820          * applying the new filter changes
1821          */
1822         i40e_service_event_schedule(vsi->back);
1823 }
1824
1825 /**
1826  * i40e_undo_filter_entries - Undo the changes made to MAC filter entries
1827  * @vsi: Pointer to VSI struct
1828  * @from: Pointer to list which contains MAC filter entries - changes to
1829  *        those entries needs to be undone.
1830  *
1831  * MAC filter entries from list were slated to be sent to firmware, either for
1832  * addition or deletion.
1833  **/
1834 static void i40e_undo_filter_entries(struct i40e_vsi *vsi,
1835                                      struct hlist_head *from)
1836 {
1837         struct i40e_mac_filter *f;
1838         struct hlist_node *h;
1839
1840         hlist_for_each_entry_safe(f, h, from, hlist) {
1841                 u64 key = i40e_addr_to_hkey(f->macaddr);
1842
1843                 /* Move the element back into MAC filter list*/
1844                 hlist_del(&f->hlist);
1845                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1846         }
1847 }
1848
1849 /**
1850  * i40e_update_filter_state - Update filter state based on return data
1851  * from firmware
1852  * @count: Number of filters added
1853  * @add_list: return data from fw
1854  * @head: pointer to first filter in current batch
1855  *
1856  * MAC filter entries from list were slated to be added to device. Returns
1857  * number of successful filters. Note that 0 does NOT mean success!
1858  **/
1859 static int
1860 i40e_update_filter_state(int count,
1861                          struct i40e_aqc_add_macvlan_element_data *add_list,
1862                          struct i40e_mac_filter *add_head)
1863 {
1864         int retval = 0;
1865         int i;
1866
1867         for (i = 0; i < count; i++) {
1868                 /* Always check status of each filter. We don't need to check
1869                  * the firmware return status because we pre-set the filter
1870                  * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
1871                  * request to the adminq. Thus, if it no longer matches then
1872                  * we know the filter is active.
1873                  */
1874                 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
1875                         add_head->state = I40E_FILTER_FAILED;
1876                 } else {
1877                         add_head->state = I40E_FILTER_ACTIVE;
1878                         retval++;
1879                 }
1880
1881                 add_head = hlist_entry(add_head->hlist.next,
1882                                        typeof(struct i40e_mac_filter),
1883                                        hlist);
1884         }
1885
1886         return retval;
1887 }
1888
1889 /**
1890  * i40e_aqc_del_filters - Request firmware to delete a set of filters
1891  * @vsi: ptr to the VSI
1892  * @vsi_name: name to display in messages
1893  * @list: the list of filters to send to firmware
1894  * @num_del: the number of filters to delete
1895  * @retval: Set to -EIO on failure to delete
1896  *
1897  * Send a request to firmware via AdminQ to delete a set of filters. Uses
1898  * *retval instead of a return value so that success does not force ret_val to
1899  * be set to 0. This ensures that a sequence of calls to this function
1900  * preserve the previous value of *retval on successful delete.
1901  */
1902 static
1903 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
1904                           struct i40e_aqc_remove_macvlan_element_data *list,
1905                           int num_del, int *retval)
1906 {
1907         struct i40e_hw *hw = &vsi->back->hw;
1908         i40e_status aq_ret;
1909         int aq_err;
1910
1911         aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
1912         aq_err = hw->aq.asq_last_status;
1913
1914         /* Explicitly ignore and do not report when firmware returns ENOENT */
1915         if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
1916                 *retval = -EIO;
1917                 dev_info(&vsi->back->pdev->dev,
1918                          "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
1919                          vsi_name, i40e_stat_str(hw, aq_ret),
1920                          i40e_aq_str(hw, aq_err));
1921         }
1922 }
1923
1924 /**
1925  * i40e_aqc_add_filters - Request firmware to add a set of filters
1926  * @vsi: ptr to the VSI
1927  * @vsi_name: name to display in messages
1928  * @list: the list of filters to send to firmware
1929  * @add_head: Position in the add hlist
1930  * @num_add: the number of filters to add
1931  * @promisc_change: set to true on exit if promiscuous mode was forced on
1932  *
1933  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
1934  * promisc_changed to true if the firmware has run out of space for more
1935  * filters.
1936  */
1937 static
1938 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
1939                           struct i40e_aqc_add_macvlan_element_data *list,
1940                           struct i40e_mac_filter *add_head,
1941                           int num_add, bool *promisc_changed)
1942 {
1943         struct i40e_hw *hw = &vsi->back->hw;
1944         int aq_err, fcnt;
1945
1946         i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
1947         aq_err = hw->aq.asq_last_status;
1948         fcnt = i40e_update_filter_state(num_add, list, add_head);
1949
1950         if (fcnt != num_add) {
1951                 *promisc_changed = true;
1952                 set_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state);
1953                 dev_warn(&vsi->back->pdev->dev,
1954                          "Error %s adding RX filters on %s, promiscuous mode forced on\n",
1955                          i40e_aq_str(hw, aq_err),
1956                          vsi_name);
1957         }
1958 }
1959
1960 /**
1961  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
1962  * @vsi: pointer to the VSI
1963  * @f: filter data
1964  *
1965  * This function sets or clears the promiscuous broadcast flags for VLAN
1966  * filters in order to properly receive broadcast frames. Assumes that only
1967  * broadcast filters are passed.
1968  **/
1969 static
1970 void i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
1971                                struct i40e_mac_filter *f)
1972 {
1973         bool enable = f->state == I40E_FILTER_NEW;
1974         struct i40e_hw *hw = &vsi->back->hw;
1975         i40e_status aq_ret;
1976
1977         if (f->vlan == I40E_VLAN_ANY) {
1978                 aq_ret = i40e_aq_set_vsi_broadcast(hw,
1979                                                    vsi->seid,
1980                                                    enable,
1981                                                    NULL);
1982         } else {
1983                 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
1984                                                             vsi->seid,
1985                                                             enable,
1986                                                             f->vlan,
1987                                                             NULL);
1988         }
1989
1990         if (aq_ret) {
1991                 dev_warn(&vsi->back->pdev->dev,
1992                          "Error %s setting broadcast promiscuous mode on %s\n",
1993                          i40e_aq_str(hw, hw->aq.asq_last_status),
1994                          vsi_name);
1995                 f->state = I40E_FILTER_FAILED;
1996         } else if (enable) {
1997                 f->state = I40E_FILTER_ACTIVE;
1998         }
1999 }
2000
2001 /**
2002  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2003  * @vsi: ptr to the VSI
2004  *
2005  * Push any outstanding VSI filter changes through the AdminQ.
2006  *
2007  * Returns 0 or error value
2008  **/
2009 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2010 {
2011         struct hlist_head tmp_add_list, tmp_del_list;
2012         struct i40e_mac_filter *f, *add_head = NULL;
2013         struct i40e_hw *hw = &vsi->back->hw;
2014         unsigned int failed_filters = 0;
2015         unsigned int vlan_filters = 0;
2016         bool promisc_changed = false;
2017         char vsi_name[16] = "PF";
2018         int filter_list_len = 0;
2019         i40e_status aq_ret = 0;
2020         u32 changed_flags = 0;
2021         struct hlist_node *h;
2022         struct i40e_pf *pf;
2023         int num_add = 0;
2024         int num_del = 0;
2025         int retval = 0;
2026         u16 cmd_flags;
2027         int list_size;
2028         int bkt;
2029
2030         /* empty array typed pointers, kcalloc later */
2031         struct i40e_aqc_add_macvlan_element_data *add_list;
2032         struct i40e_aqc_remove_macvlan_element_data *del_list;
2033
2034         while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state))
2035                 usleep_range(1000, 2000);
2036         pf = vsi->back;
2037
2038         if (vsi->netdev) {
2039                 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2040                 vsi->current_netdev_flags = vsi->netdev->flags;
2041         }
2042
2043         INIT_HLIST_HEAD(&tmp_add_list);
2044         INIT_HLIST_HEAD(&tmp_del_list);
2045
2046         if (vsi->type == I40E_VSI_SRIOV)
2047                 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2048         else if (vsi->type != I40E_VSI_MAIN)
2049                 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2050
2051         if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2052                 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2053
2054                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2055                 /* Create a list of filters to delete. */
2056                 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2057                         if (f->state == I40E_FILTER_REMOVE) {
2058                                 /* Move the element into temporary del_list */
2059                                 hash_del(&f->hlist);
2060                                 hlist_add_head(&f->hlist, &tmp_del_list);
2061
2062                                 /* Avoid counting removed filters */
2063                                 continue;
2064                         }
2065                         if (f->state == I40E_FILTER_NEW) {
2066                                 hash_del(&f->hlist);
2067                                 hlist_add_head(&f->hlist, &tmp_add_list);
2068                         }
2069
2070                         /* Count the number of active (current and new) VLAN
2071                          * filters we have now. Does not count filters which
2072                          * are marked for deletion.
2073                          */
2074                         if (f->vlan > 0)
2075                                 vlan_filters++;
2076                 }
2077
2078                 retval = i40e_correct_mac_vlan_filters(vsi,
2079                                                        &tmp_add_list,
2080                                                        &tmp_del_list,
2081                                                        vlan_filters);
2082                 if (retval)
2083                         goto err_no_memory_locked;
2084
2085                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2086         }
2087
2088         /* Now process 'del_list' outside the lock */
2089         if (!hlist_empty(&tmp_del_list)) {
2090                 filter_list_len = hw->aq.asq_buf_size /
2091                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2092                 list_size = filter_list_len *
2093                             sizeof(struct i40e_aqc_remove_macvlan_element_data);
2094                 del_list = kzalloc(list_size, GFP_ATOMIC);
2095                 if (!del_list)
2096                         goto err_no_memory;
2097
2098                 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2099                         cmd_flags = 0;
2100
2101                         /* handle broadcast filters by updating the broadcast
2102                          * promiscuous flag instead of deleting a MAC filter.
2103                          */
2104                         if (is_broadcast_ether_addr(f->macaddr)) {
2105                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2106
2107                                 hlist_del(&f->hlist);
2108                                 kfree(f);
2109                                 continue;
2110                         }
2111
2112                         /* add to delete list */
2113                         ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2114                         if (f->vlan == I40E_VLAN_ANY) {
2115                                 del_list[num_del].vlan_tag = 0;
2116                                 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2117                         } else {
2118                                 del_list[num_del].vlan_tag =
2119                                         cpu_to_le16((u16)(f->vlan));
2120                         }
2121
2122                         cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2123                         del_list[num_del].flags = cmd_flags;
2124                         num_del++;
2125
2126                         /* flush a full buffer */
2127                         if (num_del == filter_list_len) {
2128                                 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2129                                                      num_del, &retval);
2130                                 memset(del_list, 0, list_size);
2131                                 num_del = 0;
2132                         }
2133                         /* Release memory for MAC filter entries which were
2134                          * synced up with HW.
2135                          */
2136                         hlist_del(&f->hlist);
2137                         kfree(f);
2138                 }
2139
2140                 if (num_del) {
2141                         i40e_aqc_del_filters(vsi, vsi_name, del_list,
2142                                              num_del, &retval);
2143                 }
2144
2145                 kfree(del_list);
2146                 del_list = NULL;
2147         }
2148
2149         if (!hlist_empty(&tmp_add_list)) {
2150                 /* Do all the adds now. */
2151                 filter_list_len = hw->aq.asq_buf_size /
2152                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2153                 list_size = filter_list_len *
2154                                sizeof(struct i40e_aqc_add_macvlan_element_data);
2155                 add_list = kzalloc(list_size, GFP_ATOMIC);
2156                 if (!add_list)
2157                         goto err_no_memory;
2158
2159                 num_add = 0;
2160                 hlist_for_each_entry_safe(f, h, &tmp_add_list, hlist) {
2161                         if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
2162                                      &vsi->state)) {
2163                                 f->state = I40E_FILTER_FAILED;
2164                                 continue;
2165                         }
2166
2167                         /* handle broadcast filters by updating the broadcast
2168                          * promiscuous flag instead of adding a MAC filter.
2169                          */
2170                         if (is_broadcast_ether_addr(f->macaddr)) {
2171                                 u64 key = i40e_addr_to_hkey(f->macaddr);
2172                                 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2173
2174                                 hlist_del(&f->hlist);
2175                                 hash_add(vsi->mac_filter_hash, &f->hlist, key);
2176                                 continue;
2177                         }
2178
2179                         /* add to add array */
2180                         if (num_add == 0)
2181                                 add_head = f;
2182                         cmd_flags = 0;
2183                         ether_addr_copy(add_list[num_add].mac_addr, f->macaddr);
2184                         if (f->vlan == I40E_VLAN_ANY) {
2185                                 add_list[num_add].vlan_tag = 0;
2186                                 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2187                         } else {
2188                                 add_list[num_add].vlan_tag =
2189                                         cpu_to_le16((u16)(f->vlan));
2190                         }
2191                         add_list[num_add].queue_number = 0;
2192                         /* set invalid match method for later detection */
2193                         add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2194                         cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2195                         add_list[num_add].flags = cpu_to_le16(cmd_flags);
2196                         num_add++;
2197
2198                         /* flush a full buffer */
2199                         if (num_add == filter_list_len) {
2200                                 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2201                                                      add_head, num_add,
2202                                                      &promisc_changed);
2203                                 memset(add_list, 0, list_size);
2204                                 num_add = 0;
2205                         }
2206                 }
2207                 if (num_add) {
2208                         i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2209                                              num_add, &promisc_changed);
2210                 }
2211                 /* Now move all of the filters from the temp add list back to
2212                  * the VSI's list.
2213                  */
2214                 spin_lock_bh(&vsi->mac_filter_hash_lock);
2215                 hlist_for_each_entry_safe(f, h, &tmp_add_list, hlist) {
2216                         u64 key = i40e_addr_to_hkey(f->macaddr);
2217
2218                         hlist_del(&f->hlist);
2219                         hash_add(vsi->mac_filter_hash, &f->hlist, key);
2220                 }
2221                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2222                 kfree(add_list);
2223                 add_list = NULL;
2224         }
2225
2226         /* Determine the number of active and failed filters. */
2227         spin_lock_bh(&vsi->mac_filter_hash_lock);
2228         vsi->active_filters = 0;
2229         hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2230                 if (f->state == I40E_FILTER_ACTIVE)
2231                         vsi->active_filters++;
2232                 else if (f->state == I40E_FILTER_FAILED)
2233                         failed_filters++;
2234         }
2235         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2236
2237         /* If promiscuous mode has changed, we need to calculate a new
2238          * threshold for when we are safe to exit
2239          */
2240         if (promisc_changed)
2241                 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2242
2243         /* Check if we are able to exit overflow promiscuous mode. We can
2244          * safely exit if we didn't just enter, we no longer have any failed
2245          * filters, and we have reduced filters below the threshold value.
2246          */
2247         if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state) &&
2248             !promisc_changed && !failed_filters &&
2249             (vsi->active_filters < vsi->promisc_threshold)) {
2250                 dev_info(&pf->pdev->dev,
2251                          "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2252                          vsi_name);
2253                 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state);
2254                 promisc_changed = true;
2255                 vsi->promisc_threshold = 0;
2256         }
2257
2258         /* if the VF is not trusted do not do promisc */
2259         if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2260                 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state);
2261                 goto out;
2262         }
2263
2264         /* check for changes in promiscuous modes */
2265         if (changed_flags & IFF_ALLMULTI) {
2266                 bool cur_multipromisc;
2267
2268                 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2269                 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2270                                                                vsi->seid,
2271                                                                cur_multipromisc,
2272                                                                NULL);
2273                 if (aq_ret) {
2274                         retval = i40e_aq_rc_to_posix(aq_ret,
2275                                                      hw->aq.asq_last_status);
2276                         dev_info(&pf->pdev->dev,
2277                                  "set multi promisc failed on %s, err %s aq_err %s\n",
2278                                  vsi_name,
2279                                  i40e_stat_str(hw, aq_ret),
2280                                  i40e_aq_str(hw, hw->aq.asq_last_status));
2281                 }
2282         }
2283         if ((changed_flags & IFF_PROMISC) ||
2284             (promisc_changed &&
2285              test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state))) {
2286                 bool cur_promisc;
2287
2288                 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2289                                test_bit(__I40E_FILTER_OVERFLOW_PROMISC,
2290                                         &vsi->state));
2291                 if ((vsi->type == I40E_VSI_MAIN) &&
2292                     (pf->lan_veb != I40E_NO_VEB) &&
2293                     !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2294                         /* set defport ON for Main VSI instead of true promisc
2295                          * this way we will get all unicast/multicast and VLAN
2296                          * promisc behavior but will not get VF or VMDq traffic
2297                          * replicated on the Main VSI.
2298                          */
2299                         if (pf->cur_promisc != cur_promisc) {
2300                                 pf->cur_promisc = cur_promisc;
2301                                 if (cur_promisc)
2302                                         aq_ret =
2303                                               i40e_aq_set_default_vsi(hw,
2304                                                                       vsi->seid,
2305                                                                       NULL);
2306                                 else
2307                                         aq_ret =
2308                                             i40e_aq_clear_default_vsi(hw,
2309                                                                       vsi->seid,
2310                                                                       NULL);
2311                                 if (aq_ret) {
2312                                         retval = i40e_aq_rc_to_posix(aq_ret,
2313                                                         hw->aq.asq_last_status);
2314                                         dev_info(&pf->pdev->dev,
2315                                                  "Set default VSI failed on %s, err %s, aq_err %s\n",
2316                                                  vsi_name,
2317                                                  i40e_stat_str(hw, aq_ret),
2318                                                  i40e_aq_str(hw,
2319                                                      hw->aq.asq_last_status));
2320                                 }
2321                         }
2322                 } else {
2323                         aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2324                                                           hw,
2325                                                           vsi->seid,
2326                                                           cur_promisc, NULL,
2327                                                           true);
2328                         if (aq_ret) {
2329                                 retval =
2330                                 i40e_aq_rc_to_posix(aq_ret,
2331                                                     hw->aq.asq_last_status);
2332                                 dev_info(&pf->pdev->dev,
2333                                          "set unicast promisc failed on %s, err %s, aq_err %s\n",
2334                                          vsi_name,
2335                                          i40e_stat_str(hw, aq_ret),
2336                                          i40e_aq_str(hw,
2337                                                      hw->aq.asq_last_status));
2338                         }
2339                         aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2340                                                           hw,
2341                                                           vsi->seid,
2342                                                           cur_promisc, NULL);
2343                         if (aq_ret) {
2344                                 retval =
2345                                 i40e_aq_rc_to_posix(aq_ret,
2346                                                     hw->aq.asq_last_status);
2347                                 dev_info(&pf->pdev->dev,
2348                                          "set multicast promisc failed on %s, err %s, aq_err %s\n",
2349                                          vsi_name,
2350                                          i40e_stat_str(hw, aq_ret),
2351                                          i40e_aq_str(hw,
2352                                                      hw->aq.asq_last_status));
2353                         }
2354                 }
2355                 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw,
2356                                                    vsi->seid,
2357                                                    cur_promisc, NULL);
2358                 if (aq_ret) {
2359                         retval = i40e_aq_rc_to_posix(aq_ret,
2360                                                      pf->hw.aq.asq_last_status);
2361                         dev_info(&pf->pdev->dev,
2362                                  "set brdcast promisc failed, err %s, aq_err %s\n",
2363                                          i40e_stat_str(hw, aq_ret),
2364                                          i40e_aq_str(hw,
2365                                                      hw->aq.asq_last_status));
2366                 }
2367         }
2368 out:
2369         /* if something went wrong then set the changed flag so we try again */
2370         if (retval)
2371                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2372
2373         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
2374         return retval;
2375
2376 err_no_memory:
2377         /* Restore elements on the temporary add and delete lists */
2378         spin_lock_bh(&vsi->mac_filter_hash_lock);
2379 err_no_memory_locked:
2380         i40e_undo_filter_entries(vsi, &tmp_del_list);
2381         i40e_undo_filter_entries(vsi, &tmp_add_list);
2382         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2383
2384         vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2385         clear_bit(__I40E_CONFIG_BUSY, &vsi->state);
2386         return -ENOMEM;
2387 }
2388
2389 /**
2390  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2391  * @pf: board private structure
2392  **/
2393 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2394 {
2395         int v;
2396
2397         if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC))
2398                 return;
2399         pf->flags &= ~I40E_FLAG_FILTER_SYNC;
2400
2401         for (v = 0; v < pf->num_alloc_vsi; v++) {
2402                 if (pf->vsi[v] &&
2403                     (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2404                         int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2405
2406                         if (ret) {
2407                                 /* come back and try again later */
2408                                 pf->flags |= I40E_FLAG_FILTER_SYNC;
2409                                 break;
2410                         }
2411                 }
2412         }
2413 }
2414
2415 /**
2416  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2417  * @netdev: network interface device structure
2418  * @new_mtu: new value for maximum frame size
2419  *
2420  * Returns 0 on success, negative on failure
2421  **/
2422 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2423 {
2424         struct i40e_netdev_priv *np = netdev_priv(netdev);
2425         struct i40e_vsi *vsi = np->vsi;
2426
2427         netdev_info(netdev, "changing MTU from %d to %d\n",
2428                     netdev->mtu, new_mtu);
2429         netdev->mtu = new_mtu;
2430         if (netif_running(netdev))
2431                 i40e_vsi_reinit_locked(vsi);
2432         i40e_notify_client_of_l2_param_changes(vsi);
2433         return 0;
2434 }
2435
2436 /**
2437  * i40e_ioctl - Access the hwtstamp interface
2438  * @netdev: network interface device structure
2439  * @ifr: interface request data
2440  * @cmd: ioctl command
2441  **/
2442 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2443 {
2444         struct i40e_netdev_priv *np = netdev_priv(netdev);
2445         struct i40e_pf *pf = np->vsi->back;
2446
2447         switch (cmd) {
2448         case SIOCGHWTSTAMP:
2449                 return i40e_ptp_get_ts_config(pf, ifr);
2450         case SIOCSHWTSTAMP:
2451                 return i40e_ptp_set_ts_config(pf, ifr);
2452         default:
2453                 return -EOPNOTSUPP;
2454         }
2455 }
2456
2457 /**
2458  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2459  * @vsi: the vsi being adjusted
2460  **/
2461 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2462 {
2463         struct i40e_vsi_context ctxt;
2464         i40e_status ret;
2465
2466         if ((vsi->info.valid_sections &
2467              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2468             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2469                 return;  /* already enabled */
2470
2471         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2472         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2473                                     I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2474
2475         ctxt.seid = vsi->seid;
2476         ctxt.info = vsi->info;
2477         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2478         if (ret) {
2479                 dev_info(&vsi->back->pdev->dev,
2480                          "update vlan stripping failed, err %s aq_err %s\n",
2481                          i40e_stat_str(&vsi->back->hw, ret),
2482                          i40e_aq_str(&vsi->back->hw,
2483                                      vsi->back->hw.aq.asq_last_status));
2484         }
2485 }
2486
2487 /**
2488  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
2489  * @vsi: the vsi being adjusted
2490  **/
2491 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2492 {
2493         struct i40e_vsi_context ctxt;
2494         i40e_status ret;
2495
2496         if ((vsi->info.valid_sections &
2497              cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2498             ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2499              I40E_AQ_VSI_PVLAN_EMOD_MASK))
2500                 return;  /* already disabled */
2501
2502         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2503         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2504                                     I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2505
2506         ctxt.seid = vsi->seid;
2507         ctxt.info = vsi->info;
2508         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2509         if (ret) {
2510                 dev_info(&vsi->back->pdev->dev,
2511                          "update vlan stripping failed, err %s aq_err %s\n",
2512                          i40e_stat_str(&vsi->back->hw, ret),
2513                          i40e_aq_str(&vsi->back->hw,
2514                                      vsi->back->hw.aq.asq_last_status));
2515         }
2516 }
2517
2518 /**
2519  * i40e_vlan_rx_register - Setup or shutdown vlan offload
2520  * @netdev: network interface to be adjusted
2521  * @features: netdev features to test if VLAN offload is enabled or not
2522  **/
2523 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features)
2524 {
2525         struct i40e_netdev_priv *np = netdev_priv(netdev);
2526         struct i40e_vsi *vsi = np->vsi;
2527
2528         if (features & NETIF_F_HW_VLAN_CTAG_RX)
2529                 i40e_vlan_stripping_enable(vsi);
2530         else
2531                 i40e_vlan_stripping_disable(vsi);
2532 }
2533
2534 /**
2535  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
2536  * @vsi: the vsi being configured
2537  * @vid: vlan id to be added (0 = untagged only , -1 = any)
2538  *
2539  * This is a helper function for adding a new MAC/VLAN filter with the
2540  * specified VLAN for each existing MAC address already in the hash table.
2541  * This function does *not* perform any accounting to update filters based on
2542  * VLAN mode.
2543  *
2544  * NOTE: this function expects to be called while under the
2545  * mac_filter_hash_lock
2546  **/
2547 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2548 {
2549         struct i40e_mac_filter *f, *add_f;
2550         struct hlist_node *h;
2551         int bkt;
2552
2553         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2554                 if (f->state == I40E_FILTER_REMOVE)
2555                         continue;
2556                 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2557                 if (!add_f) {
2558                         dev_info(&vsi->back->pdev->dev,
2559                                  "Could not add vlan filter %d for %pM\n",
2560                                  vid, f->macaddr);
2561                         return -ENOMEM;
2562                 }
2563         }
2564
2565         return 0;
2566 }
2567
2568 /**
2569  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
2570  * @vsi: the VSI being configured
2571  * @vid: VLAN id to be added (0 = untagged only , -1 = any)
2572  **/
2573 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid)
2574 {
2575         int err;
2576
2577         /* Locked once because all functions invoked below iterates list*/
2578         spin_lock_bh(&vsi->mac_filter_hash_lock);
2579         err = i40e_add_vlan_all_mac(vsi, vid);
2580         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2581         if (err)
2582                 return err;
2583
2584         /* schedule our worker thread which will take care of
2585          * applying the new filter changes
2586          */
2587         i40e_service_event_schedule(vsi->back);
2588         return 0;
2589 }
2590
2591 /**
2592  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
2593  * @vsi: the vsi being configured
2594  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
2595  *
2596  * This function should be used to remove all VLAN filters which match the
2597  * given VID. It does not schedule the service event and does not take the
2598  * mac_filter_hash_lock so it may be combined with other operations under
2599  * a single invocation of the mac_filter_hash_lock.
2600  *
2601  * NOTE: this function expects to be called while under the
2602  * mac_filter_hash_lock
2603  */
2604 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2605 {
2606         struct i40e_mac_filter *f;
2607         struct hlist_node *h;
2608         int bkt;
2609
2610         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2611                 if (f->vlan == vid)
2612                         __i40e_del_filter(vsi, f);
2613         }
2614 }
2615
2616 /**
2617  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
2618  * @vsi: the VSI being configured
2619  * @vid: VLAN id to be removed (0 = untagged only , -1 = any)
2620  **/
2621 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid)
2622 {
2623         spin_lock_bh(&vsi->mac_filter_hash_lock);
2624         i40e_rm_vlan_all_mac(vsi, vid);
2625         spin_unlock_bh(&vsi->mac_filter_hash_lock);
2626
2627         /* schedule our worker thread which will take care of
2628          * applying the new filter changes
2629          */
2630         i40e_service_event_schedule(vsi->back);
2631 }
2632
2633 /**
2634  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
2635  * @netdev: network interface to be adjusted
2636  * @vid: vlan id to be added
2637  *
2638  * net_device_ops implementation for adding vlan ids
2639  **/
2640 #ifdef I40E_FCOE
2641 int i40e_vlan_rx_add_vid(struct net_device *netdev,
2642                          __always_unused __be16 proto, u16 vid)
2643 #else
2644 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2645                                 __always_unused __be16 proto, u16 vid)
2646 #endif
2647 {
2648         struct i40e_netdev_priv *np = netdev_priv(netdev);
2649         struct i40e_vsi *vsi = np->vsi;
2650         int ret = 0;
2651
2652         if (vid >= VLAN_N_VID)
2653                 return -EINVAL;
2654
2655         /* If the network stack called us with vid = 0 then
2656          * it is asking to receive priority tagged packets with
2657          * vlan id 0.  Our HW receives them by default when configured
2658          * to receive untagged packets so there is no need to add an
2659          * extra filter for vlan 0 tagged packets.
2660          */
2661         if (vid)
2662                 ret = i40e_vsi_add_vlan(vsi, vid);
2663
2664         if (!ret)
2665                 set_bit(vid, vsi->active_vlans);
2666
2667         return ret;
2668 }
2669
2670 /**
2671  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2672  * @netdev: network interface to be adjusted
2673  * @vid: vlan id to be removed
2674  *
2675  * net_device_ops implementation for removing vlan ids
2676  **/
2677 #ifdef I40E_FCOE
2678 int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2679                           __always_unused __be16 proto, u16 vid)
2680 #else
2681 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2682                                  __always_unused __be16 proto, u16 vid)
2683 #endif
2684 {
2685         struct i40e_netdev_priv *np = netdev_priv(netdev);
2686         struct i40e_vsi *vsi = np->vsi;
2687
2688         /* return code is ignored as there is nothing a user
2689          * can do about failure to remove and a log message was
2690          * already printed from the other function
2691          */
2692         i40e_vsi_kill_vlan(vsi, vid);
2693
2694         clear_bit(vid, vsi->active_vlans);
2695
2696         return 0;
2697 }
2698
2699 /**
2700  * i40e_macaddr_init - explicitly write the mac address filters
2701  *
2702  * @vsi: pointer to the vsi
2703  * @macaddr: the MAC address
2704  *
2705  * This is needed when the macaddr has been obtained by other
2706  * means than the default, e.g., from Open Firmware or IDPROM.
2707  * Returns 0 on success, negative on failure
2708  **/
2709 static int i40e_macaddr_init(struct i40e_vsi *vsi, u8 *macaddr)
2710 {
2711         int ret;
2712         struct i40e_aqc_add_macvlan_element_data element;
2713
2714         ret = i40e_aq_mac_address_write(&vsi->back->hw,
2715                                         I40E_AQC_WRITE_TYPE_LAA_WOL,
2716                                         macaddr, NULL);
2717         if (ret) {
2718                 dev_info(&vsi->back->pdev->dev,
2719                          "Addr change for VSI failed: %d\n", ret);
2720                 return -EADDRNOTAVAIL;
2721         }
2722
2723         memset(&element, 0, sizeof(element));
2724         ether_addr_copy(element.mac_addr, macaddr);
2725         element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH);
2726         ret = i40e_aq_add_macvlan(&vsi->back->hw, vsi->seid, &element, 1, NULL);
2727         if (ret) {
2728                 dev_info(&vsi->back->pdev->dev,
2729                          "add filter failed err %s aq_err %s\n",
2730                          i40e_stat_str(&vsi->back->hw, ret),
2731                          i40e_aq_str(&vsi->back->hw,
2732                                      vsi->back->hw.aq.asq_last_status));
2733         }
2734         return ret;
2735 }
2736
2737 /**
2738  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
2739  * @vsi: the vsi being brought back up
2740  **/
2741 static void i40e_restore_vlan(struct i40e_vsi *vsi)
2742 {
2743         u16 vid;
2744
2745         if (!vsi->netdev)
2746                 return;
2747
2748         i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features);
2749
2750         for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2751                 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q),
2752                                      vid);
2753 }
2754
2755 /**
2756  * i40e_vsi_add_pvid - Add pvid for the VSI
2757  * @vsi: the vsi being adjusted
2758  * @vid: the vlan id to set as a PVID
2759  **/
2760 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2761 {
2762         struct i40e_vsi_context ctxt;
2763         i40e_status ret;
2764
2765         vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2766         vsi->info.pvid = cpu_to_le16(vid);
2767         vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2768                                     I40E_AQ_VSI_PVLAN_INSERT_PVID |
2769                                     I40E_AQ_VSI_PVLAN_EMOD_STR;
2770
2771         ctxt.seid = vsi->seid;
2772         ctxt.info = vsi->info;
2773         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2774         if (ret) {
2775                 dev_info(&vsi->back->pdev->dev,
2776                          "add pvid failed, err %s aq_err %s\n",
2777                          i40e_stat_str(&vsi->back->hw, ret),
2778                          i40e_aq_str(&vsi->back->hw,
2779                                      vsi->back->hw.aq.asq_last_status));
2780                 return -ENOENT;
2781         }
2782
2783         return 0;
2784 }
2785
2786 /**
2787  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
2788  * @vsi: the vsi being adjusted
2789  *
2790  * Just use the vlan_rx_register() service to put it back to normal
2791  **/
2792 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2793 {
2794         i40e_vlan_stripping_disable(vsi);
2795
2796         vsi->info.pvid = 0;
2797 }
2798
2799 /**
2800  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
2801  * @vsi: ptr to the VSI
2802  *
2803  * If this function returns with an error, then it's possible one or
2804  * more of the rings is populated (while the rest are not).  It is the
2805  * callers duty to clean those orphaned rings.
2806  *
2807  * Return 0 on success, negative on failure
2808  **/
2809 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2810 {
2811         int i, err = 0;
2812
2813         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2814                 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2815
2816         return err;
2817 }
2818
2819 /**
2820  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
2821  * @vsi: ptr to the VSI
2822  *
2823  * Free VSI's transmit software resources
2824  **/
2825 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
2826 {
2827         int i;
2828
2829         if (!vsi->tx_rings)
2830                 return;
2831
2832         for (i = 0; i < vsi->num_queue_pairs; i++)
2833                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
2834                         i40e_free_tx_resources(vsi->tx_rings[i]);
2835 }
2836
2837 /**
2838  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
2839  * @vsi: ptr to the VSI
2840  *
2841  * If this function returns with an error, then it's possible one or
2842  * more of the rings is populated (while the rest are not).  It is the
2843  * callers duty to clean those orphaned rings.
2844  *
2845  * Return 0 on success, negative on failure
2846  **/
2847 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
2848 {
2849         int i, err = 0;
2850
2851         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2852                 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
2853 #ifdef I40E_FCOE
2854         i40e_fcoe_setup_ddp_resources(vsi);
2855 #endif
2856         return err;
2857 }
2858
2859 /**
2860  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
2861  * @vsi: ptr to the VSI
2862  *
2863  * Free all receive software resources
2864  **/
2865 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
2866 {
2867         int i;
2868
2869         if (!vsi->rx_rings)
2870                 return;
2871
2872         for (i = 0; i < vsi->num_queue_pairs; i++)
2873                 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
2874                         i40e_free_rx_resources(vsi->rx_rings[i]);
2875 #ifdef I40E_FCOE
2876         i40e_fcoe_free_ddp_resources(vsi);
2877 #endif
2878 }
2879
2880 /**
2881  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
2882  * @ring: The Tx ring to configure
2883  *
2884  * This enables/disables XPS for a given Tx descriptor ring
2885  * based on the TCs enabled for the VSI that ring belongs to.
2886  **/
2887 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
2888 {
2889         struct i40e_vsi *vsi = ring->vsi;
2890         cpumask_var_t mask;
2891
2892         if (!ring->q_vector || !ring->netdev)
2893                 return;
2894
2895         /* Single TC mode enable XPS */
2896         if (vsi->tc_config.numtc <= 1) {
2897                 if (!test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state))
2898                         netif_set_xps_queue(ring->netdev,
2899                                             &ring->q_vector->affinity_mask,
2900                                             ring->queue_index);
2901         } else if (alloc_cpumask_var(&mask, GFP_KERNEL)) {
2902                 /* Disable XPS to allow selection based on TC */
2903                 bitmap_zero(cpumask_bits(mask), nr_cpumask_bits);
2904                 netif_set_xps_queue(ring->netdev, mask, ring->queue_index);
2905                 free_cpumask_var(mask);
2906         }
2907
2908         /* schedule our worker thread which will take care of
2909          * applying the new filter changes
2910          */
2911         i40e_service_event_schedule(vsi->back);
2912 }
2913
2914 /**
2915  * i40e_configure_tx_ring - Configure a transmit ring context and rest
2916  * @ring: The Tx ring to configure
2917  *
2918  * Configure the Tx descriptor ring in the HMC context.
2919  **/
2920 static int i40e_configure_tx_ring(struct i40e_ring *ring)
2921 {
2922         struct i40e_vsi *vsi = ring->vsi;
2923         u16 pf_q = vsi->base_queue + ring->queue_index;
2924         struct i40e_hw *hw = &vsi->back->hw;
2925         struct i40e_hmc_obj_txq tx_ctx;
2926         i40e_status err = 0;
2927         u32 qtx_ctl = 0;
2928
2929         /* some ATR related tx ring init */
2930         if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
2931                 ring->atr_sample_rate = vsi->back->atr_sample_rate;
2932                 ring->atr_count = 0;
2933         } else {
2934                 ring->atr_sample_rate = 0;
2935         }
2936
2937         /* configure XPS */
2938         i40e_config_xps_tx_ring(ring);
2939
2940         /* clear the context structure first */
2941         memset(&tx_ctx, 0, sizeof(tx_ctx));
2942
2943         tx_ctx.new_context = 1;
2944         tx_ctx.base = (ring->dma / 128);
2945         tx_ctx.qlen = ring->count;
2946         tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
2947                                                I40E_FLAG_FD_ATR_ENABLED));
2948 #ifdef I40E_FCOE
2949         tx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE);
2950 #endif
2951         tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
2952         /* FDIR VSI tx ring can still use RS bit and writebacks */
2953         if (vsi->type != I40E_VSI_FDIR)
2954                 tx_ctx.head_wb_ena = 1;
2955         tx_ctx.head_wb_addr = ring->dma +
2956                               (ring->count * sizeof(struct i40e_tx_desc));
2957
2958         /* As part of VSI creation/update, FW allocates certain
2959          * Tx arbitration queue sets for each TC enabled for
2960          * the VSI. The FW returns the handles to these queue
2961          * sets as part of the response buffer to Add VSI,
2962          * Update VSI, etc. AQ commands. It is expected that
2963          * these queue set handles be associated with the Tx
2964          * queues by the driver as part of the TX queue context
2965          * initialization. This has to be done regardless of
2966          * DCB as by default everything is mapped to TC0.
2967          */
2968         tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
2969         tx_ctx.rdylist_act = 0;
2970
2971         /* clear the context in the HMC */
2972         err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2973         if (err) {
2974                 dev_info(&vsi->back->pdev->dev,
2975                          "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
2976                          ring->queue_index, pf_q, err);
2977                 return -ENOMEM;
2978         }
2979
2980         /* set the context in the HMC */
2981         err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2982         if (err) {
2983                 dev_info(&vsi->back->pdev->dev,
2984                          "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
2985                          ring->queue_index, pf_q, err);
2986                 return -ENOMEM;
2987         }
2988
2989         /* Now associate this queue with this PCI function */
2990         if (vsi->type == I40E_VSI_VMDQ2) {
2991                 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
2992                 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
2993                            I40E_QTX_CTL_VFVM_INDX_MASK;
2994         } else {
2995                 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2996         }
2997
2998         qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2999                     I40E_QTX_CTL_PF_INDX_MASK);
3000         wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3001         i40e_flush(hw);
3002
3003         /* cache tail off for easier writes later */
3004         ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3005
3006         return 0;
3007 }
3008
3009 /**
3010  * i40e_configure_rx_ring - Configure a receive ring context
3011  * @ring: The Rx ring to configure
3012  *
3013  * Configure the Rx descriptor ring in the HMC context.
3014  **/
3015 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3016 {
3017         struct i40e_vsi *vsi = ring->vsi;
3018         u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3019         u16 pf_q = vsi->base_queue + ring->queue_index;
3020         struct i40e_hw *hw = &vsi->back->hw;
3021         struct i40e_hmc_obj_rxq rx_ctx;
3022         i40e_status err = 0;
3023
3024         ring->state = 0;
3025
3026         /* clear the context structure first */
3027         memset(&rx_ctx, 0, sizeof(rx_ctx));
3028
3029         ring->rx_buf_len = vsi->rx_buf_len;
3030
3031         rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
3032
3033         rx_ctx.base = (ring->dma / 128);
3034         rx_ctx.qlen = ring->count;
3035
3036         /* use 32 byte descriptors */
3037         rx_ctx.dsize = 1;
3038
3039         /* descriptor type is always zero
3040          * rx_ctx.dtype = 0;
3041          */
3042         rx_ctx.hsplit_0 = 0;
3043
3044         rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3045         if (hw->revision_id == 0)
3046                 rx_ctx.lrxqthresh = 0;
3047         else
3048                 rx_ctx.lrxqthresh = 2;
3049         rx_ctx.crcstrip = 1;
3050         rx_ctx.l2tsel = 1;
3051         /* this controls whether VLAN is stripped from inner headers */
3052         rx_ctx.showiv = 0;
3053 #ifdef I40E_FCOE
3054         rx_ctx.fc_ena = (vsi->type == I40E_VSI_FCOE);
3055 #endif
3056         /* set the prefena field to 1 because the manual says to */
3057         rx_ctx.prefena = 1;
3058
3059         /* clear the context in the HMC */
3060         err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3061         if (err) {
3062                 dev_info(&vsi->back->pdev->dev,
3063                          "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3064                          ring->queue_index, pf_q, err);
3065                 return -ENOMEM;
3066         }
3067
3068         /* set the context in the HMC */
3069         err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3070         if (err) {
3071                 dev_info(&vsi->back->pdev->dev,
3072                          "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3073                          ring->queue_index, pf_q, err);
3074                 return -ENOMEM;
3075         }
3076
3077         /* cache tail for quicker writes, and clear the reg before use */
3078         ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3079         writel(0, ring->tail);
3080
3081         i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3082
3083         return 0;
3084 }
3085
3086 /**
3087  * i40e_vsi_configure_tx - Configure the VSI for Tx
3088  * @vsi: VSI structure describing this set of rings and resources
3089  *
3090  * Configure the Tx VSI for operation.
3091  **/
3092 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3093 {
3094         int err = 0;
3095         u16 i;
3096
3097         for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3098                 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3099
3100         return err;
3101 }
3102
3103 /**
3104  * i40e_vsi_configure_rx - Configure the VSI for Rx
3105  * @vsi: the VSI being configured
3106  *
3107  * Configure the Rx VSI for operation.
3108  **/
3109 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3110 {
3111         int err = 0;
3112         u16 i;
3113
3114         if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN))
3115                 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN
3116                                + ETH_FCS_LEN + VLAN_HLEN;
3117         else
3118                 vsi->max_frame = I40E_RXBUFFER_2048;
3119
3120         vsi->rx_buf_len = I40E_RXBUFFER_2048;
3121
3122 #ifdef I40E_FCOE
3123         /* setup rx buffer for FCoE */
3124         if ((vsi->type == I40E_VSI_FCOE) &&
3125             (vsi->back->flags & I40E_FLAG_FCOE_ENABLED)) {
3126                 vsi->rx_buf_len = I40E_RXBUFFER_3072;
3127                 vsi->max_frame = I40E_RXBUFFER_3072;
3128         }
3129
3130 #endif /* I40E_FCOE */
3131         /* round up for the chip's needs */
3132         vsi->rx_buf_len = ALIGN(vsi->rx_buf_len,
3133                                 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3134
3135         /* set up individual rings */
3136         for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3137                 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3138
3139         return err;
3140 }
3141
3142 /**
3143  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3144  * @vsi: ptr to the VSI
3145  **/
3146 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3147 {
3148         struct i40e_ring *tx_ring, *rx_ring;
3149         u16 qoffset, qcount;
3150         int i, n;
3151
3152         if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3153                 /* Reset the TC information */
3154                 for (i = 0; i < vsi->num_queue_pairs; i++) {
3155                         rx_ring = vsi->rx_rings[i];
3156                         tx_ring = vsi->tx_rings[i];
3157                         rx_ring->dcb_tc = 0;
3158                         tx_ring->dcb_tc = 0;
3159                 }
3160         }
3161
3162         for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3163                 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3164                         continue;
3165
3166                 qoffset = vsi->tc_config.tc_info[n].qoffset;
3167                 qcount = vsi->tc_config.tc_info[n].qcount;
3168                 for (i = qoffset; i < (qoffset + qcount); i++) {
3169                         rx_ring = vsi->rx_rings[i];
3170                         tx_ring = vsi->tx_rings[i];
3171                         rx_ring->dcb_tc = n;
3172                         tx_ring->dcb_tc = n;
3173                 }
3174         }
3175 }
3176
3177 /**
3178  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3179  * @vsi: ptr to the VSI
3180  **/
3181 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3182 {
3183         struct i40e_pf *pf = vsi->back;
3184         int err;
3185
3186         if (vsi->netdev)
3187                 i40e_set_rx_mode(vsi->netdev);
3188
3189         if (!!(pf->flags & I40E_FLAG_PF_MAC)) {
3190                 err = i40e_macaddr_init(vsi, pf->hw.mac.addr);
3191                 if (err) {
3192                         dev_warn(&pf->pdev->dev,
3193                                  "could not set up macaddr; err %d\n", err);
3194                 }
3195         }
3196 }
3197
3198 /**
3199  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3200  * @vsi: Pointer to the targeted VSI
3201  *
3202  * This function replays the hlist on the hw where all the SB Flow Director
3203  * filters were saved.
3204  **/
3205 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3206 {
3207         struct i40e_fdir_filter *filter;
3208         struct i40e_pf *pf = vsi->back;
3209         struct hlist_node *node;
3210
3211         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3212                 return;
3213
3214         hlist_for_each_entry_safe(filter, node,
3215                                   &pf->fdir_filter_list, fdir_node) {
3216                 i40e_add_del_fdir(vsi, filter, true);
3217         }
3218 }
3219
3220 /**
3221  * i40e_vsi_configure - Set up the VSI for action
3222  * @vsi: the VSI being configured
3223  **/
3224 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3225 {
3226         int err;
3227
3228         i40e_set_vsi_rx_mode(vsi);
3229         i40e_restore_vlan(vsi);
3230         i40e_vsi_config_dcb_rings(vsi);
3231         err = i40e_vsi_configure_tx(vsi);
3232         if (!err)
3233                 err = i40e_vsi_configure_rx(vsi);
3234
3235         return err;
3236 }
3237
3238 /**
3239  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3240  * @vsi: the VSI being configured
3241  **/
3242 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3243 {
3244         struct i40e_pf *pf = vsi->back;
3245         struct i40e_hw *hw = &pf->hw;
3246         u16 vector;
3247         int i, q;
3248         u32 qp;
3249
3250         /* The interrupt indexing is offset by 1 in the PFINT_ITRn
3251          * and PFINT_LNKLSTn registers, e.g.:
3252          *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3253          */
3254         qp = vsi->base_queue;
3255         vector = vsi->base_vector;
3256         for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3257                 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3258
3259                 q_vector->itr_countdown = ITR_COUNTDOWN_START;
3260                 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting);
3261                 q_vector->rx.latency_range = I40E_LOW_LATENCY;
3262                 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3263                      q_vector->rx.itr);
3264                 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting);
3265                 q_vector->tx.latency_range = I40E_LOW_LATENCY;
3266                 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3267                      q_vector->tx.itr);
3268                 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3269                      INTRL_USEC_TO_REG(vsi->int_rate_limit));
3270
3271                 /* Linked list for the queuepairs assigned to this vector */
3272                 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3273                 for (q = 0; q < q_vector->num_ringpairs; q++) {
3274                         u32 val;
3275
3276                         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3277                               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT)  |
3278                               (vector      << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3279                               (qp          << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3280                               (I40E_QUEUE_TYPE_TX
3281                                       << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3282
3283                         wr32(hw, I40E_QINT_RQCTL(qp), val);
3284
3285                         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3286                               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)  |
3287                               (vector      << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3288                               ((qp+1)      << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)|
3289                               (I40E_QUEUE_TYPE_RX
3290                                       << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3291
3292                         /* Terminate the linked list */
3293                         if (q == (q_vector->num_ringpairs - 1))
3294                                 val |= (I40E_QUEUE_END_OF_LIST
3295                                            << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3296
3297                         wr32(hw, I40E_QINT_TQCTL(qp), val);
3298                         qp++;
3299                 }
3300         }
3301
3302         i40e_flush(hw);
3303 }
3304
3305 /**
3306  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3307  * @hw: ptr to the hardware info
3308  **/
3309 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3310 {
3311         struct i40e_hw *hw = &pf->hw;
3312         u32 val;
3313
3314         /* clear things first */
3315         wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3316         rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3317
3318         val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3319               I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3320               I40E_PFINT_ICR0_ENA_GRST_MASK          |
3321               I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3322               I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3323               I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3324               I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3325               I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3326
3327         if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3328                 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3329
3330         if (pf->flags & I40E_FLAG_PTP)
3331                 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3332
3333         wr32(hw, I40E_PFINT_ICR0_ENA, val);
3334
3335         /* SW_ITR_IDX = 0, but don't change INTENA */
3336         wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3337                                         I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3338
3339         /* OTHER_ITR_IDX = 0 */
3340         wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3341 }
3342
3343 /**
3344  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3345  * @vsi: the VSI being configured
3346  **/
3347 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3348 {
3349         struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3350         struct i40e_pf *pf = vsi->back;
3351         struct i40e_hw *hw = &pf->hw;
3352         u32 val;
3353
3354         /* set the ITR configuration */
3355         q_vector->itr_countdown = ITR_COUNTDOWN_START;
3356         q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting);
3357         q_vector->rx.latency_range = I40E_LOW_LATENCY;
3358         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr);
3359         q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting);
3360         q_vector->tx.latency_range = I40E_LOW_LATENCY;
3361         wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr);
3362
3363         i40e_enable_misc_int_causes(pf);
3364
3365         /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3366         wr32(hw, I40E_PFINT_LNKLST0, 0);
3367
3368         /* Associate the queue pair to the vector and enable the queue int */
3369         val = I40E_QINT_RQCTL_CAUSE_ENA_MASK                  |
3370               (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3371               (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3372
3373         wr32(hw, I40E_QINT_RQCTL(0), val);
3374
3375         val = I40E_QINT_TQCTL_CAUSE_ENA_MASK                  |
3376               (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3377               (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3378
3379         wr32(hw, I40E_QINT_TQCTL(0), val);
3380         i40e_flush(hw);
3381 }
3382
3383 /**
3384  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
3385  * @pf: board private structure
3386  **/
3387 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3388 {
3389         struct i40e_hw *hw = &pf->hw;
3390
3391         wr32(hw, I40E_PFINT_DYN_CTL0,
3392              I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3393         i40e_flush(hw);
3394 }
3395
3396 /**
3397  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
3398  * @pf: board private structure
3399  * @clearpba: true when all pending interrupt events should be cleared
3400  **/
3401 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba)
3402 {
3403         struct i40e_hw *hw = &pf->hw;
3404         u32 val;
3405
3406         val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
3407               (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) |
3408               (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3409
3410         wr32(hw, I40E_PFINT_DYN_CTL0, val);
3411         i40e_flush(hw);
3412 }
3413
3414 /**
3415  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
3416  * @irq: interrupt number
3417  * @data: pointer to a q_vector
3418  **/
3419 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3420 {
3421         struct i40e_q_vector *q_vector = data;
3422
3423         if (!q_vector->tx.ring && !q_vector->rx.ring)
3424                 return IRQ_HANDLED;
3425
3426         napi_schedule_irqoff(&q_vector->napi);
3427
3428         return IRQ_HANDLED;
3429 }
3430
3431 /**
3432  * i40e_irq_affinity_notify - Callback for affinity changes
3433  * @notify: context as to what irq was changed
3434  * @mask: the new affinity mask
3435  *
3436  * This is a callback function used by the irq_set_affinity_notifier function
3437  * so that we may register to receive changes to the irq affinity masks.
3438  **/
3439 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3440                                      const cpumask_t *mask)
3441 {
3442         struct i40e_q_vector *q_vector =
3443                 container_of(notify, struct i40e_q_vector, affinity_notify);
3444
3445         q_vector->affinity_mask = *mask;
3446 }
3447
3448 /**
3449  * i40e_irq_affinity_release - Callback for affinity notifier release
3450  * @ref: internal core kernel usage
3451  *
3452  * This is a callback function used by the irq_set_affinity_notifier function
3453  * to inform the current notification subscriber that they will no longer
3454  * receive notifications.
3455  **/
3456 static void i40e_irq_affinity_release(struct kref *ref) {}
3457
3458 /**
3459  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
3460  * @vsi: the VSI being configured
3461  * @basename: name for the vector
3462  *
3463  * Allocates MSI-X vectors and requests interrupts from the kernel.
3464  **/
3465 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3466 {
3467         int q_vectors = vsi->num_q_vectors;
3468         struct i40e_pf *pf = vsi->back;
3469         int base = vsi->base_vector;
3470         int rx_int_idx = 0;
3471         int tx_int_idx = 0;
3472         int vector, err;
3473         int irq_num;
3474
3475         for (vector = 0; vector < q_vectors; vector++) {
3476                 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3477
3478                 irq_num = pf->msix_entries[base + vector].vector;
3479
3480                 if (q_vector->tx.ring && q_vector->rx.ring) {
3481                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3482                                  "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3483                         tx_int_idx++;
3484                 } else if (q_vector->rx.ring) {
3485                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3486                                  "%s-%s-%d", basename, "rx", rx_int_idx++);
3487                 } else if (q_vector->tx.ring) {
3488                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3489                                  "%s-%s-%d", basename, "tx", tx_int_idx++);
3490                 } else {
3491                         /* skip this unused q_vector */
3492                         continue;
3493                 }
3494                 err = request_irq(irq_num,
3495                                   vsi->irq_handler,
3496                                   0,
3497                                   q_vector->name,
3498                                   q_vector);
3499                 if (err) {
3500                         dev_info(&pf->pdev->dev,
3501                                  "MSIX request_irq failed, error: %d\n", err);
3502                         goto free_queue_irqs;
3503                 }
3504
3505                 /* register for affinity change notifications */
3506                 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3507                 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3508                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3509                 /* assign the mask for this irq */
3510                 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
3511         }
3512
3513         vsi->irqs_ready = true;
3514         return 0;
3515
3516 free_queue_irqs:
3517         while (vector) {
3518                 vector--;
3519                 irq_num = pf->msix_entries[base + vector].vector;
3520                 irq_set_affinity_notifier(irq_num, NULL);
3521                 irq_set_affinity_hint(irq_num, NULL);
3522                 free_irq(irq_num, &vsi->q_vectors[vector]);
3523         }
3524         return err;
3525 }
3526
3527 /**
3528  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
3529  * @vsi: the VSI being un-configured
3530  **/
3531 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3532 {
3533         struct i40e_pf *pf = vsi->back;
3534         struct i40e_hw *hw = &pf->hw;
3535         int base = vsi->base_vector;
3536         int i;
3537
3538         for (i = 0; i < vsi->num_queue_pairs; i++) {
3539                 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0);
3540                 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0);
3541         }
3542
3543         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3544                 for (i = vsi->base_vector;
3545                      i < (vsi->num_q_vectors + vsi->base_vector); i++)
3546                         wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3547
3548                 i40e_flush(hw);
3549                 for (i = 0; i < vsi->num_q_vectors; i++)
3550                         synchronize_irq(pf->msix_entries[i + base].vector);
3551         } else {
3552                 /* Legacy and MSI mode - this stops all interrupt handling */
3553                 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3554                 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3555                 i40e_flush(hw);
3556                 synchronize_irq(pf->pdev->irq);
3557         }
3558 }
3559
3560 /**
3561  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
3562  * @vsi: the VSI being configured
3563  **/
3564 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3565 {
3566         struct i40e_pf *pf = vsi->back;
3567         int i;
3568
3569         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3570                 for (i = 0; i < vsi->num_q_vectors; i++)
3571                         i40e_irq_dynamic_enable(vsi, i);
3572         } else {
3573                 i40e_irq_dynamic_enable_icr0(pf, true);
3574         }
3575
3576         i40e_flush(&pf->hw);
3577         return 0;
3578 }
3579
3580 /**
3581  * i40e_stop_misc_vector - Stop the vector that handles non-queue events
3582  * @pf: board private structure
3583  **/
3584 static void i40e_stop_misc_vector(struct i40e_pf *pf)
3585 {
3586         /* Disable ICR 0 */
3587         wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3588         i40e_flush(&pf->hw);
3589 }
3590
3591 /**
3592  * i40e_intr - MSI/Legacy and non-queue interrupt handler
3593  * @irq: interrupt number
3594  * @data: pointer to a q_vector
3595  *
3596  * This is the handler used for all MSI/Legacy interrupts, and deals
3597  * with both queue and non-queue interrupts.  This is also used in
3598  * MSIX mode to handle the non-queue interrupts.
3599  **/
3600 static irqreturn_t i40e_intr(int irq, void *data)
3601 {
3602         struct i40e_pf *pf = (struct i40e_pf *)data;
3603         struct i40e_hw *hw = &pf->hw;
3604         irqreturn_t ret = IRQ_NONE;
3605         u32 icr0, icr0_remaining;
3606         u32 val, ena_mask;
3607
3608         icr0 = rd32(hw, I40E_PFINT_ICR0);
3609         ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3610
3611         /* if sharing a legacy IRQ, we might get called w/o an intr pending */
3612         if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3613                 goto enable_intr;
3614
3615         /* if interrupt but no bits showing, must be SWINT */
3616         if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3617             (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3618                 pf->sw_int_count++;
3619
3620         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3621             (ena_mask & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3622                 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3623                 icr0 &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3624                 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3625         }
3626
3627         /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
3628         if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3629                 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3630                 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3631
3632                 /* We do not have a way to disarm Queue causes while leaving
3633                  * interrupt enabled for all other causes, ideally
3634                  * interrupt should be disabled while we are in NAPI but
3635                  * this is not a performance path and napi_schedule()
3636                  * can deal with rescheduling.
3637                  */
3638                 if (!test_bit(__I40E_DOWN, &pf->state))
3639                         napi_schedule_irqoff(&q_vector->napi);
3640         }
3641
3642         if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3643                 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3644                 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
3645                 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3646         }
3647
3648         if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3649                 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3650                 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
3651         }
3652
3653         if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3654                 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3655                 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state);
3656         }
3657
3658         if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3659                 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
3660                         set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
3661                 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3662                 val = rd32(hw, I40E_GLGEN_RSTAT);
3663                 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3664                        >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3665                 if (val == I40E_RESET_CORER) {
3666                         pf->corer_count++;
3667                 } else if (val == I40E_RESET_GLOBR) {
3668                         pf->globr_count++;
3669                 } else if (val == I40E_RESET_EMPR) {
3670                         pf->empr_count++;
3671                         set_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state);
3672                 }
3673         }
3674
3675         if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3676                 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3677                 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3678                 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3679                          rd32(hw, I40E_PFHMC_ERRORINFO),
3680                          rd32(hw, I40E_PFHMC_ERRORDATA));
3681         }
3682
3683         if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3684                 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3685
3686                 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
3687                         icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3688                         i40e_ptp_tx_hwtstamp(pf);
3689                 }
3690         }
3691
3692         /* If a critical error is pending we have no choice but to reset the
3693          * device.
3694          * Report and mask out any remaining unexpected interrupts.
3695          */
3696         icr0_remaining = icr0 & ena_mask;
3697         if (icr0_remaining) {
3698                 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
3699                          icr0_remaining);
3700                 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
3701                     (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
3702                     (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
3703                         dev_info(&pf->pdev->dev, "device will be reset\n");
3704                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
3705                         i40e_service_event_schedule(pf);
3706                 }
3707                 ena_mask &= ~icr0_remaining;
3708         }
3709         ret = IRQ_HANDLED;
3710
3711 enable_intr:
3712         /* re-enable interrupt causes */
3713         wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
3714         if (!test_bit(__I40E_DOWN, &pf->state)) {
3715                 i40e_service_event_schedule(pf);
3716                 i40e_irq_dynamic_enable_icr0(pf, false);
3717         }
3718
3719         return ret;
3720 }
3721
3722 /**
3723  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
3724  * @tx_ring:  tx ring to clean
3725  * @budget:   how many cleans we're allowed
3726  *
3727  * Returns true if there's any budget left (e.g. the clean is finished)
3728  **/
3729 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
3730 {
3731         struct i40e_vsi *vsi = tx_ring->vsi;
3732         u16 i = tx_ring->next_to_clean;
3733         struct i40e_tx_buffer *tx_buf;
3734         struct i40e_tx_desc *tx_desc;
3735
3736         tx_buf = &tx_ring->tx_bi[i];
3737         tx_desc = I40E_TX_DESC(tx_ring, i);
3738         i -= tx_ring->count;
3739
3740         do {
3741                 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
3742
3743                 /* if next_to_watch is not set then there is no work pending */
3744                 if (!eop_desc)
3745                         break;
3746
3747                 /* prevent any other reads prior to eop_desc */
3748                 read_barrier_depends();
3749
3750                 /* if the descriptor isn't done, no work yet to do */
3751                 if (!(eop_desc->cmd_type_offset_bsz &
3752                       cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
3753                         break;
3754
3755                 /* clear next_to_watch to prevent false hangs */
3756                 tx_buf->next_to_watch = NULL;
3757
3758                 tx_desc->buffer_addr = 0;
3759                 tx_desc->cmd_type_offset_bsz = 0;
3760                 /* move past filter desc */
3761                 tx_buf++;
3762                 tx_desc++;
3763                 i++;
3764                 if (unlikely(!i)) {
3765                         i -= tx_ring->count;
3766                         tx_buf = tx_ring->tx_bi;
3767                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3768                 }
3769                 /* unmap skb header data */
3770                 dma_unmap_single(tx_ring->dev,
3771                                  dma_unmap_addr(tx_buf, dma),
3772                                  dma_unmap_len(tx_buf, len),
3773                                  DMA_TO_DEVICE);
3774                 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
3775                         kfree(tx_buf->raw_buf);
3776
3777                 tx_buf->raw_buf = NULL;
3778                 tx_buf->tx_flags = 0;
3779                 tx_buf->next_to_watch = NULL;
3780                 dma_unmap_len_set(tx_buf, len, 0);
3781                 tx_desc->buffer_addr = 0;
3782                 tx_desc->cmd_type_offset_bsz = 0;
3783
3784                 /* move us past the eop_desc for start of next FD desc */
3785                 tx_buf++;
3786                 tx_desc++;
3787                 i++;
3788                 if (unlikely(!i)) {
3789                         i -= tx_ring->count;
3790                         tx_buf = tx_ring->tx_bi;
3791                         tx_desc = I40E_TX_DESC(tx_ring, 0);
3792                 }
3793
3794                 /* update budget accounting */
3795                 budget--;
3796         } while (likely(budget));
3797
3798         i += tx_ring->count;
3799         tx_ring->next_to_clean = i;
3800
3801         if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
3802                 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
3803
3804         return budget > 0;
3805 }
3806
3807 /**
3808  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
3809  * @irq: interrupt number
3810  * @data: pointer to a q_vector
3811  **/
3812 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
3813 {
3814         struct i40e_q_vector *q_vector = data;
3815         struct i40e_vsi *vsi;
3816
3817         if (!q_vector->tx.ring)
3818                 return IRQ_HANDLED;
3819
3820         vsi = q_vector->tx.ring->vsi;
3821         i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
3822
3823         return IRQ_HANDLED;
3824 }
3825
3826 /**
3827  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
3828  * @vsi: the VSI being configured
3829  * @v_idx: vector index
3830  * @qp_idx: queue pair index
3831  **/
3832 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
3833 {
3834         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
3835         struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
3836         struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
3837
3838         tx_ring->q_vector = q_vector;
3839         tx_ring->next = q_vector->tx.ring;
3840         q_vector->tx.ring = tx_ring;
3841         q_vector->tx.count++;
3842
3843         rx_ring->q_vector = q_vector;
3844         rx_ring->next = q_vector->rx.ring;
3845         q_vector->rx.ring = rx_ring;
3846         q_vector->rx.count++;
3847 }
3848
3849 /**
3850  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
3851  * @vsi: the VSI being configured
3852  *
3853  * This function maps descriptor rings to the queue-specific vectors
3854  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
3855  * one vector per queue pair, but on a constrained vector budget, we
3856  * group the queue pairs as "efficiently" as possible.
3857  **/
3858 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
3859 {
3860         int qp_remaining = vsi->num_queue_pairs;
3861         int q_vectors = vsi->num_q_vectors;
3862         int num_ringpairs;
3863         int v_start = 0;
3864         int qp_idx = 0;
3865
3866         /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
3867          * group them so there are multiple queues per vector.
3868          * It is also important to go through all the vectors available to be
3869          * sure that if we don't use all the vectors, that the remaining vectors
3870          * are cleared. This is especially important when decreasing the
3871          * number of queues in use.
3872          */
3873         for (; v_start < q_vectors; v_start++) {
3874                 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
3875
3876                 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
3877
3878                 q_vector->num_ringpairs = num_ringpairs;
3879
3880                 q_vector->rx.count = 0;
3881                 q_vector->tx.count = 0;
3882                 q_vector->rx.ring = NULL;
3883                 q_vector->tx.ring = NULL;
3884
3885                 while (num_ringpairs--) {
3886                         i40e_map_vector_to_qp(vsi, v_start, qp_idx);
3887                         qp_idx++;
3888                         qp_remaining--;
3889                 }
3890         }
3891 }
3892
3893 /**
3894  * i40e_vsi_request_irq - Request IRQ from the OS
3895  * @vsi: the VSI being configured
3896  * @basename: name for the vector
3897  **/
3898 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
3899 {
3900         struct i40e_pf *pf = vsi->back;
3901         int err;
3902
3903         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
3904                 err = i40e_vsi_request_irq_msix(vsi, basename);
3905         else if (pf->flags & I40E_FLAG_MSI_ENABLED)
3906                 err = request_irq(pf->pdev->irq, i40e_intr, 0,
3907                                   pf->int_name, pf);
3908         else
3909                 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
3910                                   pf->int_name, pf);
3911
3912         if (err)
3913                 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
3914
3915         return err;
3916 }
3917
3918 #ifdef CONFIG_NET_POLL_CONTROLLER
3919 /**
3920  * i40e_netpoll - A Polling 'interrupt' handler
3921  * @netdev: network interface device structure
3922  *
3923  * This is used by netconsole to send skbs without having to re-enable
3924  * interrupts.  It's not called while the normal interrupt routine is executing.
3925  **/
3926 #ifdef I40E_FCOE
3927 void i40e_netpoll(struct net_device *netdev)
3928 #else
3929 static void i40e_netpoll(struct net_device *netdev)
3930 #endif
3931 {
3932         struct i40e_netdev_priv *np = netdev_priv(netdev);
3933         struct i40e_vsi *vsi = np->vsi;
3934         struct i40e_pf *pf = vsi->back;
3935         int i;
3936
3937         /* if interface is down do nothing */
3938         if (test_bit(__I40E_DOWN, &vsi->state))
3939                 return;
3940
3941         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3942                 for (i = 0; i < vsi->num_q_vectors; i++)
3943                         i40e_msix_clean_rings(0, vsi->q_vectors[i]);
3944         } else {
3945                 i40e_intr(pf->pdev->irq, netdev);
3946         }
3947 }
3948 #endif
3949
3950 /**
3951  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
3952  * @pf: the PF being configured
3953  * @pf_q: the PF queue
3954  * @enable: enable or disable state of the queue
3955  *
3956  * This routine will wait for the given Tx queue of the PF to reach the
3957  * enabled or disabled state.
3958  * Returns -ETIMEDOUT in case of failing to reach the requested state after
3959  * multiple retries; else will return 0 in case of success.
3960  **/
3961 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
3962 {
3963         int i;
3964         u32 tx_reg;
3965
3966         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
3967                 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
3968                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
3969                         break;
3970
3971                 usleep_range(10, 20);
3972         }
3973         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
3974                 return -ETIMEDOUT;
3975
3976         return 0;
3977 }
3978
3979 /**
3980  * i40e_vsi_control_tx - Start or stop a VSI's rings
3981  * @vsi: the VSI being configured
3982  * @enable: start or stop the rings
3983  **/
3984 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
3985 {
3986         struct i40e_pf *pf = vsi->back;
3987         struct i40e_hw *hw = &pf->hw;
3988         int i, j, pf_q, ret = 0;
3989         u32 tx_reg;
3990
3991         pf_q = vsi->base_queue;
3992         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
3993
3994                 /* warn the TX unit of coming changes */
3995                 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
3996                 if (!enable)
3997                         usleep_range(10, 20);
3998
3999                 for (j = 0; j < 50; j++) {
4000                         tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4001                         if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4002                             ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4003                                 break;
4004                         usleep_range(1000, 2000);
4005                 }
4006                 /* Skip if the queue is already in the requested state */
4007                 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4008                         continue;
4009
4010                 /* turn on/off the queue */
4011                 if (enable) {
4012                         wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4013                         tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4014                 } else {
4015                         tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4016                 }
4017
4018                 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4019                 /* No waiting for the Tx queue to disable */
4020                 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state))
4021                         continue;
4022
4023                 /* wait for the change to finish */
4024                 ret = i40e_pf_txq_wait(pf, pf_q, enable);
4025                 if (ret) {
4026                         dev_info(&pf->pdev->dev,
4027                                  "VSI seid %d Tx ring %d %sable timeout\n",
4028                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
4029                         break;
4030                 }
4031         }
4032
4033         if (hw->revision_id == 0)
4034                 mdelay(50);
4035         return ret;
4036 }
4037
4038 /**
4039  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4040  * @pf: the PF being configured
4041  * @pf_q: the PF queue
4042  * @enable: enable or disable state of the queue
4043  *
4044  * This routine will wait for the given Rx queue of the PF to reach the
4045  * enabled or disabled state.
4046  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4047  * multiple retries; else will return 0 in case of success.
4048  **/
4049 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4050 {
4051         int i;
4052         u32 rx_reg;
4053
4054         for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4055                 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4056                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4057                         break;
4058
4059                 usleep_range(10, 20);
4060         }
4061         if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4062                 return -ETIMEDOUT;
4063
4064         return 0;
4065 }
4066
4067 /**
4068  * i40e_vsi_control_rx - Start or stop a VSI's rings
4069  * @vsi: the VSI being configured
4070  * @enable: start or stop the rings
4071  **/
4072 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4073 {
4074         struct i40e_pf *pf = vsi->back;
4075         struct i40e_hw *hw = &pf->hw;
4076         int i, j, pf_q, ret = 0;
4077         u32 rx_reg;
4078
4079         pf_q = vsi->base_queue;
4080         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4081                 for (j = 0; j < 50; j++) {
4082                         rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4083                         if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4084                             ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4085                                 break;
4086                         usleep_range(1000, 2000);
4087                 }
4088
4089                 /* Skip if the queue is already in the requested state */
4090                 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4091                         continue;
4092
4093                 /* turn on/off the queue */
4094                 if (enable)
4095                         rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4096                 else
4097                         rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4098                 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4099                 /* No waiting for the Tx queue to disable */
4100                 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state))
4101                         continue;
4102
4103                 /* wait for the change to finish */
4104                 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4105                 if (ret) {
4106                         dev_info(&pf->pdev->dev,
4107                                  "VSI seid %d Rx ring %d %sable timeout\n",
4108                                  vsi->seid, pf_q, (enable ? "en" : "dis"));
4109                         break;
4110                 }
4111         }
4112
4113         return ret;
4114 }
4115
4116 /**
4117  * i40e_vsi_start_rings - Start a VSI's rings
4118  * @vsi: the VSI being configured
4119  **/
4120 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4121 {
4122         int ret = 0;
4123
4124         /* do rx first for enable and last for disable */
4125         ret = i40e_vsi_control_rx(vsi, true);
4126         if (ret)
4127                 return ret;
4128         ret = i40e_vsi_control_tx(vsi, true);
4129
4130         return ret;
4131 }
4132
4133 /**
4134  * i40e_vsi_stop_rings - Stop a VSI's rings
4135  * @vsi: the VSI being configured
4136  **/
4137 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4138 {
4139         /* do rx first for enable and last for disable
4140          * Ignore return value, we need to shutdown whatever we can
4141          */
4142         i40e_vsi_control_tx(vsi, false);
4143         i40e_vsi_control_rx(vsi, false);
4144 }
4145
4146 /**
4147  * i40e_vsi_free_irq - Free the irq association with the OS
4148  * @vsi: the VSI being configured
4149  **/
4150 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4151 {
4152         struct i40e_pf *pf = vsi->back;
4153         struct i40e_hw *hw = &pf->hw;
4154         int base = vsi->base_vector;
4155         u32 val, qp;
4156         int i;
4157
4158         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4159                 if (!vsi->q_vectors)
4160                         return;
4161
4162                 if (!vsi->irqs_ready)
4163                         return;
4164
4165                 vsi->irqs_ready = false;
4166                 for (i = 0; i < vsi->num_q_vectors; i++) {
4167                         int irq_num;
4168                         u16 vector;
4169
4170                         vector = i + base;
4171                         irq_num = pf->msix_entries[vector].vector;
4172
4173                         /* free only the irqs that were actually requested */
4174                         if (!vsi->q_vectors[i] ||
4175                             !vsi->q_vectors[i]->num_ringpairs)
4176                                 continue;
4177
4178                         /* clear the affinity notifier in the IRQ descriptor */
4179                         irq_set_affinity_notifier(irq_num, NULL);
4180                         /* clear the affinity_mask in the IRQ descriptor */
4181                         irq_set_affinity_hint(irq_num, NULL);
4182                         synchronize_irq(irq_num);
4183                         free_irq(irq_num, vsi->q_vectors[i]);
4184
4185                         /* Tear down the interrupt queue link list
4186                          *
4187                          * We know that they come in pairs and always
4188                          * the Rx first, then the Tx.  To clear the
4189                          * link list, stick the EOL value into the
4190                          * next_q field of the registers.
4191                          */
4192                         val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4193                         qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4194                                 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4195                         val |= I40E_QUEUE_END_OF_LIST
4196                                 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4197                         wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4198
4199                         while (qp != I40E_QUEUE_END_OF_LIST) {
4200                                 u32 next;
4201
4202                                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4203
4204                                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4205                                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4206                                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4207                                          I40E_QINT_RQCTL_INTEVENT_MASK);
4208
4209                                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4210                                          I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4211
4212                                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4213
4214                                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4215
4216                                 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4217                                         >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4218
4219                                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4220                                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4221                                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4222                                          I40E_QINT_TQCTL_INTEVENT_MASK);
4223
4224                                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4225                                          I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4226
4227                                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4228                                 qp = next;
4229                         }
4230                 }
4231         } else {
4232                 free_irq(pf->pdev->irq, pf);
4233
4234                 val = rd32(hw, I40E_PFINT_LNKLST0);
4235                 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4236                         >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4237                 val |= I40E_QUEUE_END_OF_LIST
4238                         << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4239                 wr32(hw, I40E_PFINT_LNKLST0, val);
4240
4241                 val = rd32(hw, I40E_QINT_RQCTL(qp));
4242                 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
4243                          I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4244                          I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
4245                          I40E_QINT_RQCTL_INTEVENT_MASK);
4246
4247                 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4248                         I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4249
4250                 wr32(hw, I40E_QINT_RQCTL(qp), val);
4251
4252                 val = rd32(hw, I40E_QINT_TQCTL(qp));
4253
4254                 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
4255                          I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4256                          I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
4257                          I40E_QINT_TQCTL_INTEVENT_MASK);
4258
4259                 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4260                         I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4261
4262                 wr32(hw, I40E_QINT_TQCTL(qp), val);
4263         }
4264 }
4265
4266 /**
4267  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
4268  * @vsi: the VSI being configured
4269  * @v_idx: Index of vector to be freed
4270  *
4271  * This function frees the memory allocated to the q_vector.  In addition if
4272  * NAPI is enabled it will delete any references to the NAPI struct prior
4273  * to freeing the q_vector.
4274  **/
4275 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4276 {
4277         struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4278         struct i40e_ring *ring;
4279
4280         if (!q_vector)
4281                 return;
4282
4283         /* disassociate q_vector from rings */
4284         i40e_for_each_ring(ring, q_vector->tx)
4285                 ring->q_vector = NULL;
4286
4287         i40e_for_each_ring(ring, q_vector->rx)
4288                 ring->q_vector = NULL;
4289
4290         /* only VSI w/ an associated netdev is set up w/ NAPI */
4291         if (vsi->netdev)
4292                 netif_napi_del(&q_vector->napi);
4293
4294         vsi->q_vectors[v_idx] = NULL;
4295
4296         kfree_rcu(q_vector, rcu);
4297 }
4298
4299 /**
4300  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
4301  * @vsi: the VSI being un-configured
4302  *
4303  * This frees the memory allocated to the q_vectors and
4304  * deletes references to the NAPI struct.
4305  **/
4306 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4307 {
4308         int v_idx;
4309
4310         for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4311                 i40e_free_q_vector(vsi, v_idx);
4312 }
4313
4314 /**
4315  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
4316  * @pf: board private structure
4317  **/
4318 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4319 {
4320         /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
4321         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4322                 pci_disable_msix(pf->pdev);
4323                 kfree(pf->msix_entries);
4324                 pf->msix_entries = NULL;
4325                 kfree(pf->irq_pile);
4326                 pf->irq_pile = NULL;
4327         } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4328                 pci_disable_msi(pf->pdev);
4329         }
4330         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4331 }
4332
4333 /**
4334  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
4335  * @pf: board private structure
4336  *
4337  * We go through and clear interrupt specific resources and reset the structure
4338  * to pre-load conditions
4339  **/
4340 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4341 {
4342         int i;
4343
4344         i40e_stop_misc_vector(pf);
4345         if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
4346                 synchronize_irq(pf->msix_entries[0].vector);
4347                 free_irq(pf->msix_entries[0].vector, pf);
4348         }
4349
4350         i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4351                       I40E_IWARP_IRQ_PILE_ID);
4352
4353         i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4354         for (i = 0; i < pf->num_alloc_vsi; i++)
4355                 if (pf->vsi[i])
4356                         i40e_vsi_free_q_vectors(pf->vsi[i]);
4357         i40e_reset_interrupt_capability(pf);
4358 }
4359
4360 /**
4361  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4362  * @vsi: the VSI being configured
4363  **/
4364 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4365 {
4366         int q_idx;
4367
4368         if (!vsi->netdev)
4369                 return;
4370
4371         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4372                 napi_enable(&vsi->q_vectors[q_idx]->napi);
4373 }
4374
4375 /**
4376  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4377  * @vsi: the VSI being configured
4378  **/
4379 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4380 {
4381         int q_idx;
4382
4383         if (!vsi->netdev)
4384                 return;
4385
4386         for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4387                 napi_disable(&vsi->q_vectors[q_idx]->napi);
4388 }
4389
4390 /**
4391  * i40e_vsi_close - Shut down a VSI
4392  * @vsi: the vsi to be quelled
4393  **/
4394 static void i40e_vsi_close(struct i40e_vsi *vsi)
4395 {
4396         bool reset = false;
4397
4398         if (!test_and_set_bit(__I40E_DOWN, &vsi->state))
4399                 i40e_down(vsi);
4400         i40e_vsi_free_irq(vsi);
4401         i40e_vsi_free_tx_resources(vsi);
4402         i40e_vsi_free_rx_resources(vsi);
4403         vsi->current_netdev_flags = 0;
4404         if (test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
4405                 reset = true;
4406         i40e_notify_client_of_netdev_close(vsi, reset);
4407 }
4408
4409 /**
4410  * i40e_quiesce_vsi - Pause a given VSI
4411  * @vsi: the VSI being paused
4412  **/
4413 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4414 {
4415         if (test_bit(__I40E_DOWN, &vsi->state))
4416                 return;
4417
4418         /* No need to disable FCoE VSI when Tx suspended */
4419         if ((test_bit(__I40E_PORT_TX_SUSPENDED, &vsi->back->state)) &&
4420             vsi->type == I40E_VSI_FCOE) {
4421                 dev_dbg(&vsi->back->pdev->dev,
4422                          "VSI seid %d skipping FCoE VSI disable\n", vsi->seid);
4423                 return;
4424         }
4425
4426         set_bit(__I40E_NEEDS_RESTART, &vsi->state);
4427         if (vsi->netdev && netif_running(vsi->netdev))
4428                 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4429         else
4430                 i40e_vsi_close(vsi);
4431 }
4432
4433 /**
4434  * i40e_unquiesce_vsi - Resume a given VSI
4435  * @vsi: the VSI being resumed
4436  **/
4437 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4438 {
4439         if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state))
4440                 return;
4441
4442         clear_bit(__I40E_NEEDS_RESTART, &vsi->state);
4443         if (vsi->netdev && netif_running(vsi->netdev))
4444                 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4445         else
4446                 i40e_vsi_open(vsi);   /* this clears the DOWN bit */
4447 }
4448
4449 /**
4450  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
4451  * @pf: the PF
4452  **/
4453 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4454 {
4455         int v;
4456
4457         for (v = 0; v < pf->num_alloc_vsi; v++) {
4458                 if (pf->vsi[v])
4459                         i40e_quiesce_vsi(pf->vsi[v]);
4460         }
4461 }
4462
4463 /**
4464  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
4465  * @pf: the PF
4466  **/
4467 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4468 {
4469         int v;
4470
4471         for (v = 0; v < pf->num_alloc_vsi; v++) {
4472                 if (pf->vsi[v])
4473                         i40e_unquiesce_vsi(pf->vsi[v]);
4474         }
4475 }
4476
4477 #ifdef CONFIG_I40E_DCB
4478 /**
4479  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
4480  * @vsi: the VSI being configured
4481  *
4482  * This function waits for the given VSI's queues to be disabled.
4483  **/
4484 static int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4485 {
4486         struct i40e_pf *pf = vsi->back;
4487         int i, pf_q, ret;
4488
4489         pf_q = vsi->base_queue;
4490         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4491                 /* Check and wait for the disable status of the queue */
4492                 ret = i40e_pf_txq_wait(pf, pf_q, false);
4493                 if (ret) {
4494                         dev_info(&pf->pdev->dev,
4495                                  "VSI seid %d Tx ring %d disable timeout\n",
4496                                  vsi->seid, pf_q);
4497                         return ret;
4498                 }
4499         }
4500
4501         pf_q = vsi->base_queue;
4502         for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4503                 /* Check and wait for the disable status of the queue */
4504                 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4505                 if (ret) {
4506                         dev_info(&pf->pdev->dev,
4507                                  "VSI seid %d Rx ring %d disable timeout\n",
4508                                  vsi->seid, pf_q);
4509                         return ret;
4510                 }
4511         }
4512
4513         return 0;
4514 }
4515
4516 /**
4517  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
4518  * @pf: the PF
4519  *
4520  * This function waits for the queues to be in disabled state for all the
4521  * VSIs that are managed by this PF.
4522  **/
4523 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4524 {
4525         int v, ret = 0;
4526
4527         for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4528                 /* No need to wait for FCoE VSI queues */
4529                 if (pf->vsi[v] && pf->vsi[v]->type != I40E_VSI_FCOE) {
4530                         ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4531                         if (ret)
4532                                 break;
4533                 }
4534         }
4535
4536         return ret;
4537 }
4538
4539 #endif
4540
4541 /**
4542  * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue
4543  * @q_idx: TX queue number
4544  * @vsi: Pointer to VSI struct
4545  *
4546  * This function checks specified queue for given VSI. Detects hung condition.
4547  * Sets hung bit since it is two step process. Before next run of service task
4548  * if napi_poll runs, it reset 'hung' bit for respective q_vector. If not,
4549  * hung condition remain unchanged and during subsequent run, this function
4550  * issues SW interrupt to recover from hung condition.
4551  **/
4552 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi)
4553 {
4554         struct i40e_ring *tx_ring = NULL;
4555         struct i40e_pf  *pf;
4556         u32 head, val, tx_pending_hw;
4557         int i;
4558
4559         pf = vsi->back;
4560
4561         /* now that we have an index, find the tx_ring struct */
4562         for (i = 0; i < vsi->num_queue_pairs; i++) {
4563                 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
4564                         if (q_idx == vsi->tx_rings[i]->queue_index) {
4565                                 tx_ring = vsi->tx_rings[i];
4566                                 break;
4567                         }
4568                 }
4569         }
4570
4571         if (!tx_ring)
4572                 return;
4573
4574         /* Read interrupt register */
4575         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4576                 val = rd32(&pf->hw,
4577                            I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
4578                                                tx_ring->vsi->base_vector - 1));
4579         else
4580                 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
4581
4582         head = i40e_get_head(tx_ring);
4583
4584         tx_pending_hw = i40e_get_tx_pending(tx_ring, false);
4585
4586         /* HW is done executing descriptors, updated HEAD write back,
4587          * but SW hasn't processed those descriptors. If interrupt is
4588          * not generated from this point ON, it could result into
4589          * dev_watchdog detecting timeout on those netdev_queue,
4590          * hence proactively trigger SW interrupt.
4591          */
4592         if (tx_pending_hw && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) {
4593                 /* NAPI Poll didn't run and clear since it was set */
4594                 if (test_and_clear_bit(I40E_Q_VECTOR_HUNG_DETECT,
4595                                        &tx_ring->q_vector->hung_detected)) {
4596                         netdev_info(vsi->netdev, "VSI_seid %d, Hung TX queue %d, tx_pending_hw: %d, NTC:0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x\n",
4597                                     vsi->seid, q_idx, tx_pending_hw,
4598                                     tx_ring->next_to_clean, head,
4599                                     tx_ring->next_to_use,
4600                                     readl(tx_ring->tail));
4601                         netdev_info(vsi->netdev, "VSI_seid %d, Issuing force_wb for TX queue %d, Interrupt Reg: 0x%x\n",
4602                                     vsi->seid, q_idx, val);
4603                         i40e_force_wb(vsi, tx_ring->q_vector);
4604                 } else {
4605                         /* First Chance - detected possible hung */
4606                         set_bit(I40E_Q_VECTOR_HUNG_DETECT,
4607                                 &tx_ring->q_vector->hung_detected);
4608                 }
4609         }
4610
4611         /* This is the case where we have interrupts missing,
4612          * so the tx_pending in HW will most likely be 0, but we
4613          * will have tx_pending in SW since the WB happened but the
4614          * interrupt got lost.
4615          */
4616         if ((!tx_pending_hw) && i40e_get_tx_pending(tx_ring, true) &&
4617             (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) {
4618                 if (napi_reschedule(&tx_ring->q_vector->napi))
4619                         tx_ring->tx_stats.tx_lost_interrupt++;
4620         }
4621 }
4622
4623 /**
4624  * i40e_detect_recover_hung - Function to detect and recover hung_queues
4625  * @pf:  pointer to PF struct
4626  *
4627  * LAN VSI has netdev and netdev has TX queues. This function is to check
4628  * each of those TX queues if they are hung, trigger recovery by issuing
4629  * SW interrupt.
4630  **/
4631 static void i40e_detect_recover_hung(struct i40e_pf *pf)
4632 {
4633         struct net_device *netdev;
4634         struct i40e_vsi *vsi;
4635         int i;
4636
4637         /* Only for LAN VSI */
4638         vsi = pf->vsi[pf->lan_vsi];
4639
4640         if (!vsi)
4641                 return;
4642
4643         /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */
4644         if (test_bit(__I40E_DOWN, &vsi->back->state) ||
4645             test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state))
4646                 return;
4647
4648         /* Make sure type is MAIN VSI */
4649         if (vsi->type != I40E_VSI_MAIN)
4650                 return;
4651
4652         netdev = vsi->netdev;
4653         if (!netdev)
4654                 return;
4655
4656         /* Bail out if netif_carrier is not OK */
4657         if (!netif_carrier_ok(netdev))
4658                 return;
4659
4660         /* Go thru' TX queues for netdev */
4661         for (i = 0; i < netdev->num_tx_queues; i++) {
4662                 struct netdev_queue *q;
4663
4664                 q = netdev_get_tx_queue(netdev, i);
4665                 if (q)
4666                         i40e_detect_recover_hung_queue(i, vsi);
4667         }
4668 }
4669
4670 /**
4671  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
4672  * @pf: pointer to PF
4673  *
4674  * Get TC map for ISCSI PF type that will include iSCSI TC
4675  * and LAN TC.
4676  **/
4677 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4678 {
4679         struct i40e_dcb_app_priority_table app;
4680         struct i40e_hw *hw = &pf->hw;
4681         u8 enabled_tc = 1; /* TC0 is always enabled */
4682         u8 tc, i;
4683         /* Get the iSCSI APP TLV */
4684         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4685
4686         for (i = 0; i < dcbcfg->numapps; i++) {
4687                 app = dcbcfg->app[i];
4688                 if (app.selector == I40E_APP_SEL_TCPIP &&
4689                     app.protocolid == I40E_APP_PROTOID_ISCSI) {
4690                         tc = dcbcfg->etscfg.prioritytable[app.priority];
4691                         enabled_tc |= BIT(tc);
4692                         break;
4693                 }
4694         }
4695
4696         return enabled_tc;
4697 }
4698
4699 /**
4700  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
4701  * @dcbcfg: the corresponding DCBx configuration structure
4702  *
4703  * Return the number of TCs from given DCBx configuration
4704  **/
4705 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
4706 {
4707         int i, tc_unused = 0;
4708         u8 num_tc = 0;
4709         u8 ret = 0;
4710
4711         /* Scan the ETS Config Priority Table to find
4712          * traffic class enabled for a given priority
4713          * and create a bitmask of enabled TCs
4714          */
4715         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
4716                 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
4717
4718         /* Now scan the bitmask to check for
4719          * contiguous TCs starting with TC0
4720          */
4721         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4722                 if (num_tc & BIT(i)) {
4723                         if (!tc_unused) {
4724                                 ret++;
4725                         } else {
4726                                 pr_err("Non-contiguous TC - Disabling DCB\n");
4727                                 return 1;
4728                         }
4729                 } else {
4730                         tc_unused = 1;
4731                 }
4732         }
4733
4734         /* There is always at least TC0 */
4735         if (!ret)
4736                 ret = 1;
4737
4738         return ret;
4739 }
4740
4741 /**
4742  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
4743  * @dcbcfg: the corresponding DCBx configuration structure
4744  *
4745  * Query the current DCB configuration and return the number of
4746  * traffic classes enabled from the given DCBX config
4747  **/
4748 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
4749 {
4750         u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
4751         u8 enabled_tc = 1;
4752         u8 i;
4753
4754         for (i = 0; i < num_tc; i++)
4755                 enabled_tc |= BIT(i);
4756
4757         return enabled_tc;
4758 }
4759
4760 /**
4761  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
4762  * @pf: PF being queried
4763  *
4764  * Return number of traffic classes enabled for the given PF
4765  **/
4766 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
4767 {
4768         struct i40e_hw *hw = &pf->hw;
4769         u8 i, enabled_tc = 1;
4770         u8 num_tc = 0;
4771         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4772
4773         /* If DCB is not enabled then always in single TC */
4774         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4775                 return 1;
4776
4777         /* SFP mode will be enabled for all TCs on port */
4778         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4779                 return i40e_dcb_get_num_tc(dcbcfg);
4780
4781         /* MFP mode return count of enabled TCs for this PF */
4782         if (pf->hw.func_caps.iscsi)
4783                 enabled_tc =  i40e_get_iscsi_tc_map(pf);
4784         else
4785                 return 1; /* Only TC0 */
4786
4787         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4788                 if (enabled_tc & BIT(i))
4789                         num_tc++;
4790         }
4791         return num_tc;
4792 }
4793
4794 /**
4795  * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes
4796  * @pf: PF being queried
4797  *
4798  * Return a bitmap for enabled traffic classes for this PF.
4799  **/
4800 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
4801 {
4802         /* If DCB is not enabled for this PF then just return default TC */
4803         if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
4804                 return I40E_DEFAULT_TRAFFIC_CLASS;
4805
4806         /* SFP mode we want PF to be enabled for all TCs */
4807         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
4808                 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
4809
4810         /* MFP enabled and iSCSI PF type */
4811         if (pf->hw.func_caps.iscsi)
4812                 return i40e_get_iscsi_tc_map(pf);
4813         else
4814                 return I40E_DEFAULT_TRAFFIC_CLASS;
4815 }
4816
4817 /**
4818  * i40e_vsi_get_bw_info - Query VSI BW Information
4819  * @vsi: the VSI being queried
4820  *
4821  * Returns 0 on success, negative value on failure
4822  **/
4823 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
4824 {
4825         struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
4826         struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
4827         struct i40e_pf *pf = vsi->back;
4828         struct i40e_hw *hw = &pf->hw;
4829         i40e_status ret;
4830         u32 tc_bw_max;
4831         int i;
4832
4833         /* Get the VSI level BW configuration */
4834         ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
4835         if (ret) {
4836                 dev_info(&pf->pdev->dev,
4837                          "couldn't get PF vsi bw config, err %s aq_err %s\n",
4838                          i40e_stat_str(&pf->hw, ret),
4839                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4840                 return -EINVAL;
4841         }
4842
4843         /* Get the VSI level BW configuration per TC */
4844         ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
4845                                                NULL);
4846         if (ret) {
4847                 dev_info(&pf->pdev->dev,
4848                          "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
4849                          i40e_stat_str(&pf->hw, ret),
4850                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
4851                 return -EINVAL;
4852         }
4853
4854         if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
4855                 dev_info(&pf->pdev->dev,
4856                          "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
4857                          bw_config.tc_valid_bits,
4858                          bw_ets_config.tc_valid_bits);
4859                 /* Still continuing */
4860         }
4861
4862         vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
4863         vsi->bw_max_quanta = bw_config.max_bw;
4864         tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
4865                     (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
4866         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4867                 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
4868                 vsi->bw_ets_limit_credits[i] =
4869                                         le16_to_cpu(bw_ets_config.credits[i]);
4870                 /* 3 bits out of 4 for each TC */
4871                 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
4872         }
4873
4874         return 0;
4875 }
4876
4877 /**
4878  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
4879  * @vsi: the VSI being configured
4880  * @enabled_tc: TC bitmap
4881  * @bw_credits: BW shared credits per TC
4882  *
4883  * Returns 0 on success, negative value on failure
4884  **/
4885 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
4886                                        u8 *bw_share)
4887 {
4888         struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
4889         i40e_status ret;
4890         int i;
4891
4892         bw_data.tc_valid_bits = enabled_tc;
4893         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4894                 bw_data.tc_bw_credits[i] = bw_share[i];
4895
4896         ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data,
4897                                        NULL);
4898         if (ret) {
4899                 dev_info(&vsi->back->pdev->dev,
4900                          "AQ command Config VSI BW allocation per TC failed = %d\n",
4901                          vsi->back->hw.aq.asq_last_status);
4902                 return -EINVAL;
4903         }
4904
4905         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
4906                 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
4907
4908         return 0;
4909 }
4910
4911 /**
4912  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
4913  * @vsi: the VSI being configured
4914  * @enabled_tc: TC map to be enabled
4915  *
4916  **/
4917 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
4918 {
4919         struct net_device *netdev = vsi->netdev;
4920         struct i40e_pf *pf = vsi->back;
4921         struct i40e_hw *hw = &pf->hw;
4922         u8 netdev_tc = 0;
4923         int i;
4924         struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4925
4926         if (!netdev)
4927                 return;
4928
4929         if (!enabled_tc) {
4930                 netdev_reset_tc(netdev);
4931                 return;
4932         }
4933
4934         /* Set up actual enabled TCs on the VSI */
4935         if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
4936                 return;
4937
4938         /* set per TC queues for the VSI */
4939         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
4940                 /* Only set TC queues for enabled tcs
4941                  *
4942                  * e.g. For a VSI that has TC0 and TC3 enabled the
4943                  * enabled_tc bitmap would be 0x00001001; the driver
4944                  * will set the numtc for netdev as 2 that will be
4945                  * referenced by the netdev layer as TC 0 and 1.
4946                  */
4947                 if (vsi->tc_config.enabled_tc & BIT(i))
4948                         netdev_set_tc_queue(netdev,
4949                                         vsi->tc_config.tc_info[i].netdev_tc,
4950                                         vsi->tc_config.tc_info[i].qcount,
4951                                         vsi->tc_config.tc_info[i].qoffset);
4952         }
4953
4954         /* Assign UP2TC map for the VSI */
4955         for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
4956                 /* Get the actual TC# for the UP */
4957                 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
4958                 /* Get the mapped netdev TC# for the UP */
4959                 netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
4960                 netdev_set_prio_tc_map(netdev, i, netdev_tc);
4961         }
4962 }
4963
4964 /**
4965  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
4966  * @vsi: the VSI being configured
4967  * @ctxt: the ctxt buffer returned from AQ VSI update param command
4968  **/
4969 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
4970                                       struct i40e_vsi_context *ctxt)
4971 {
4972         /* copy just the sections touched not the entire info
4973          * since not all sections are valid as returned by
4974          * update vsi params
4975          */
4976         vsi->info.mapping_flags = ctxt->info.mapping_flags;
4977         memcpy(&vsi->info.queue_mapping,
4978                &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
4979         memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
4980                sizeof(vsi->info.tc_mapping));
4981 }
4982
4983 /**
4984  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
4985  * @vsi: VSI to be configured
4986  * @enabled_tc: TC bitmap
4987  *
4988  * This configures a particular VSI for TCs that are mapped to the
4989  * given TC bitmap. It uses default bandwidth share for TCs across
4990  * VSIs to configure TC for a particular VSI.
4991  *
4992  * NOTE:
4993  * It is expected that the VSI queues have been quisced before calling
4994  * this function.
4995  **/
4996 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
4997 {
4998         u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
4999         struct i40e_vsi_context ctxt;
5000         int ret = 0;
5001         int i;
5002
5003         /* Check if enabled_tc is same as existing or new TCs */
5004         if (vsi->tc_config.enabled_tc == enabled_tc)
5005                 return ret;
5006
5007         /* Enable ETS TCs with equal BW Share for now across all VSIs */
5008         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5009                 if (enabled_tc & BIT(i))
5010                         bw_share[i] = 1;
5011         }
5012
5013         ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5014         if (ret) {
5015                 dev_info(&vsi->back->pdev->dev,
5016                          "Failed configuring TC map %d for VSI %d\n",
5017                          enabled_tc, vsi->seid);
5018                 goto out;
5019         }
5020
5021         /* Update Queue Pairs Mapping for currently enabled UPs */
5022         ctxt.seid = vsi->seid;
5023         ctxt.pf_num = vsi->back->hw.pf_id;
5024         ctxt.vf_num = 0;
5025         ctxt.uplink_seid = vsi->uplink_seid;
5026         ctxt.info = vsi->info;
5027         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5028
5029         if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5030                 ctxt.info.valid_sections |=
5031                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5032                 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5033         }
5034
5035         /* Update the VSI after updating the VSI queue-mapping information */
5036         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
5037         if (ret) {
5038                 dev_info(&vsi->back->pdev->dev,
5039                          "Update vsi tc config failed, err %s aq_err %s\n",
5040                          i40e_stat_str(&vsi->back->hw, ret),
5041                          i40e_aq_str(&vsi->back->hw,
5042                                      vsi->back->hw.aq.asq_last_status));
5043                 goto out;
5044         }
5045         /* update the local VSI info with updated queue map */
5046         i40e_vsi_update_queue_map(vsi, &ctxt);
5047         vsi->info.valid_sections = 0;
5048
5049         /* Update current VSI BW information */
5050         ret = i40e_vsi_get_bw_info(vsi);
5051         if (ret) {
5052                 dev_info(&vsi->back->pdev->dev,
5053                          "Failed updating vsi bw info, err %s aq_err %s\n",
5054                          i40e_stat_str(&vsi->back->hw, ret),
5055                          i40e_aq_str(&vsi->back->hw,
5056                                      vsi->back->hw.aq.asq_last_status));
5057                 goto out;
5058         }
5059
5060         /* Update the netdev TC setup */
5061         i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5062 out:
5063         return ret;
5064 }
5065
5066 /**
5067  * i40e_veb_config_tc - Configure TCs for given VEB
5068  * @veb: given VEB
5069  * @enabled_tc: TC bitmap
5070  *
5071  * Configures given TC bitmap for VEB (switching) element
5072  **/
5073 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
5074 {
5075         struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
5076         struct i40e_pf *pf = veb->pf;
5077         int ret = 0;
5078         int i;
5079
5080         /* No TCs or already enabled TCs just return */
5081         if (!enabled_tc || veb->enabled_tc == enabled_tc)
5082                 return ret;
5083
5084         bw_data.tc_valid_bits = enabled_tc;
5085         /* bw_data.absolute_credits is not set (relative) */
5086
5087         /* Enable ETS TCs with equal BW Share for now */
5088         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5089                 if (enabled_tc & BIT(i))
5090                         bw_data.tc_bw_share_credits[i] = 1;
5091         }
5092
5093         ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
5094                                                    &bw_data, NULL);
5095         if (ret) {
5096                 dev_info(&pf->pdev->dev,
5097                          "VEB bw config failed, err %s aq_err %s\n",
5098                          i40e_stat_str(&pf->hw, ret),
5099                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5100                 goto out;
5101         }
5102
5103         /* Update the BW information */
5104         ret = i40e_veb_get_bw_info(veb);
5105         if (ret) {
5106                 dev_info(&pf->pdev->dev,
5107                          "Failed getting veb bw config, err %s aq_err %s\n",
5108                          i40e_stat_str(&pf->hw, ret),
5109                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5110         }
5111
5112 out:
5113         return ret;
5114 }
5115
5116 #ifdef CONFIG_I40E_DCB
5117 /**
5118  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
5119  * @pf: PF struct
5120  *
5121  * Reconfigure VEB/VSIs on a given PF; it is assumed that
5122  * the caller would've quiesce all the VSIs before calling
5123  * this function
5124  **/
5125 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
5126 {
5127         u8 tc_map = 0;
5128         int ret;
5129         u8 v;
5130
5131         /* Enable the TCs available on PF to all VEBs */
5132         tc_map = i40e_pf_get_tc_map(pf);
5133         for (v = 0; v < I40E_MAX_VEB; v++) {
5134                 if (!pf->veb[v])
5135                         continue;
5136                 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
5137                 if (ret) {
5138                         dev_info(&pf->pdev->dev,
5139                                  "Failed configuring TC for VEB seid=%d\n",
5140                                  pf->veb[v]->seid);
5141                         /* Will try to configure as many components */
5142                 }
5143         }
5144
5145         /* Update each VSI */
5146         for (v = 0; v < pf->num_alloc_vsi; v++) {
5147                 if (!pf->vsi[v])
5148                         continue;
5149
5150                 /* - Enable all TCs for the LAN VSI
5151 #ifdef I40E_FCOE
5152                  * - For FCoE VSI only enable the TC configured
5153                  *   as per the APP TLV
5154 #endif
5155                  * - For all others keep them at TC0 for now
5156                  */
5157                 if (v == pf->lan_vsi)
5158                         tc_map = i40e_pf_get_tc_map(pf);
5159                 else
5160                         tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
5161 #ifdef I40E_FCOE
5162                 if (pf->vsi[v]->type == I40E_VSI_FCOE)
5163                         tc_map = i40e_get_fcoe_tc_map(pf);
5164 #endif /* #ifdef I40E_FCOE */
5165
5166                 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
5167                 if (ret) {
5168                         dev_info(&pf->pdev->dev,
5169                                  "Failed configuring TC for VSI seid=%d\n",
5170                                  pf->vsi[v]->seid);
5171                         /* Will try to configure as many components */
5172                 } else {
5173                         /* Re-configure VSI vectors based on updated TC map */
5174                         i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
5175                         if (pf->vsi[v]->netdev)
5176                                 i40e_dcbnl_set_all(pf->vsi[v]);
5177                 }
5178         }
5179 }
5180
5181 /**
5182  * i40e_resume_port_tx - Resume port Tx
5183  * @pf: PF struct
5184  *
5185  * Resume a port's Tx and issue a PF reset in case of failure to
5186  * resume.
5187  **/
5188 static int i40e_resume_port_tx(struct i40e_pf *pf)
5189 {
5190         struct i40e_hw *hw = &pf->hw;
5191         int ret;
5192
5193         ret = i40e_aq_resume_port_tx(hw, NULL);
5194         if (ret) {
5195                 dev_info(&pf->pdev->dev,
5196                          "Resume Port Tx failed, err %s aq_err %s\n",
5197                           i40e_stat_str(&pf->hw, ret),
5198                           i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5199                 /* Schedule PF reset to recover */
5200                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
5201                 i40e_service_event_schedule(pf);
5202         }
5203
5204         return ret;
5205 }
5206
5207 /**
5208  * i40e_init_pf_dcb - Initialize DCB configuration
5209  * @pf: PF being configured
5210  *
5211  * Query the current DCB configuration and cache it
5212  * in the hardware structure
5213  **/
5214 static int i40e_init_pf_dcb(struct i40e_pf *pf)
5215 {
5216         struct i40e_hw *hw = &pf->hw;
5217         int err = 0;
5218
5219         /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */
5220         if (pf->flags & I40E_FLAG_NO_DCB_SUPPORT)
5221                 goto out;
5222
5223         /* Get the initial DCB configuration */
5224         err = i40e_init_dcb(hw);
5225         if (!err) {
5226                 /* Device/Function is not DCBX capable */
5227                 if ((!hw->func_caps.dcb) ||
5228                     (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
5229                         dev_info(&pf->pdev->dev,
5230                                  "DCBX offload is not supported or is disabled for this PF.\n");
5231
5232                         if (pf->flags & I40E_FLAG_MFP_ENABLED)
5233                                 goto out;
5234
5235                 } else {
5236                         /* When status is not DISABLED then DCBX in FW */
5237                         pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
5238                                        DCB_CAP_DCBX_VER_IEEE;
5239
5240                         pf->flags |= I40E_FLAG_DCB_CAPABLE;
5241                         /* Enable DCB tagging only when more than one TC
5242                          * or explicitly disable if only one TC
5243                          */
5244                         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5245                                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5246                         else
5247                                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5248                         dev_dbg(&pf->pdev->dev,
5249                                 "DCBX offload is supported for this PF.\n");
5250                 }
5251         } else {
5252                 dev_info(&pf->pdev->dev,
5253                          "Query for DCB configuration failed, err %s aq_err %s\n",
5254                          i40e_stat_str(&pf->hw, err),
5255                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5256         }
5257
5258 out:
5259         return err;
5260 }
5261 #endif /* CONFIG_I40E_DCB */
5262 #define SPEED_SIZE 14
5263 #define FC_SIZE 8
5264 /**
5265  * i40e_print_link_message - print link up or down
5266  * @vsi: the VSI for which link needs a message
5267  */
5268 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
5269 {
5270         enum i40e_aq_link_speed new_speed;
5271         char *speed = "Unknown";
5272         char *fc = "Unknown";
5273
5274         new_speed = vsi->back->hw.phy.link_info.link_speed;
5275
5276         if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
5277                 return;
5278         vsi->current_isup = isup;
5279         vsi->current_speed = new_speed;
5280         if (!isup) {
5281                 netdev_info(vsi->netdev, "NIC Link is Down\n");
5282                 return;
5283         }
5284
5285         /* Warn user if link speed on NPAR enabled partition is not at
5286          * least 10GB
5287          */
5288         if (vsi->back->hw.func_caps.npar_enable &&
5289             (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
5290              vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
5291                 netdev_warn(vsi->netdev,
5292                             "The partition detected link speed that is less than 10Gbps\n");
5293
5294         switch (vsi->back->hw.phy.link_info.link_speed) {
5295         case I40E_LINK_SPEED_40GB:
5296                 speed = "40 G";
5297                 break;
5298         case I40E_LINK_SPEED_20GB:
5299                 speed = "20 G";
5300                 break;
5301         case I40E_LINK_SPEED_25GB:
5302                 speed = "25 G";
5303                 break;
5304         case I40E_LINK_SPEED_10GB:
5305                 speed = "10 G";
5306                 break;
5307         case I40E_LINK_SPEED_1GB:
5308                 speed = "1000 M";
5309                 break;
5310         case I40E_LINK_SPEED_100MB:
5311                 speed = "100 M";
5312                 break;
5313         default:
5314                 break;
5315         }
5316
5317         switch (vsi->back->hw.fc.current_mode) {
5318         case I40E_FC_FULL:
5319                 fc = "RX/TX";
5320                 break;
5321         case I40E_FC_TX_PAUSE:
5322                 fc = "TX";
5323                 break;
5324         case I40E_FC_RX_PAUSE:
5325                 fc = "RX";
5326                 break;
5327         default:
5328                 fc = "None";
5329                 break;
5330         }
5331
5332         netdev_info(vsi->netdev, "NIC Link is Up %sbps Full Duplex, Flow Control: %s\n",
5333                     speed, fc);
5334 }
5335
5336 /**
5337  * i40e_up_complete - Finish the last steps of bringing up a connection
5338  * @vsi: the VSI being configured
5339  **/
5340 static int i40e_up_complete(struct i40e_vsi *vsi)
5341 {
5342         struct i40e_pf *pf = vsi->back;
5343         int err;
5344
5345         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
5346                 i40e_vsi_configure_msix(vsi);
5347         else
5348                 i40e_configure_msi_and_legacy(vsi);
5349
5350         /* start rings */
5351         err = i40e_vsi_start_rings(vsi);
5352         if (err)
5353                 return err;
5354
5355         clear_bit(__I40E_DOWN, &vsi->state);
5356         i40e_napi_enable_all(vsi);
5357         i40e_vsi_enable_irq(vsi);
5358
5359         if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
5360             (vsi->netdev)) {
5361                 i40e_print_link_message(vsi, true);
5362                 netif_tx_start_all_queues(vsi->netdev);
5363                 netif_carrier_on(vsi->netdev);
5364         } else if (vsi->netdev) {
5365                 i40e_print_link_message(vsi, false);
5366                 /* need to check for qualified module here*/
5367                 if ((pf->hw.phy.link_info.link_info &
5368                         I40E_AQ_MEDIA_AVAILABLE) &&
5369                     (!(pf->hw.phy.link_info.an_info &
5370                         I40E_AQ_QUALIFIED_MODULE)))
5371                         netdev_err(vsi->netdev,
5372                                    "the driver failed to link because an unqualified module was detected.");
5373         }
5374
5375         /* replay FDIR SB filters */
5376         if (vsi->type == I40E_VSI_FDIR) {
5377                 /* reset fd counters */
5378                 pf->fd_add_err = pf->fd_atr_cnt = 0;
5379                 if (pf->fd_tcp_rule > 0) {
5380                         pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED;
5381                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
5382                                 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 exist\n");
5383                         pf->fd_tcp_rule = 0;
5384                 }
5385                 i40e_fdir_filter_restore(vsi);
5386         }
5387
5388         /* On the next run of the service_task, notify any clients of the new
5389          * opened netdev
5390          */
5391         pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED;
5392         i40e_service_event_schedule(pf);
5393
5394         return 0;
5395 }
5396
5397 /**
5398  * i40e_vsi_reinit_locked - Reset the VSI
5399  * @vsi: the VSI being configured
5400  *
5401  * Rebuild the ring structs after some configuration
5402  * has changed, e.g. MTU size.
5403  **/
5404 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
5405 {
5406         struct i40e_pf *pf = vsi->back;
5407
5408         WARN_ON(in_interrupt());
5409         while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state))
5410                 usleep_range(1000, 2000);
5411         i40e_down(vsi);
5412
5413         i40e_up(vsi);
5414         clear_bit(__I40E_CONFIG_BUSY, &pf->state);
5415 }
5416
5417 /**
5418  * i40e_up - Bring the connection back up after being down
5419  * @vsi: the VSI being configured
5420  **/
5421 int i40e_up(struct i40e_vsi *vsi)
5422 {
5423         int err;
5424
5425         err = i40e_vsi_configure(vsi);
5426         if (!err)
5427                 err = i40e_up_complete(vsi);
5428
5429         return err;
5430 }
5431
5432 /**
5433  * i40e_down - Shutdown the connection processing
5434  * @vsi: the VSI being stopped
5435  **/
5436 void i40e_down(struct i40e_vsi *vsi)
5437 {
5438         int i;
5439
5440         /* It is assumed that the caller of this function
5441          * sets the vsi->state __I40E_DOWN bit.
5442          */
5443         if (vsi->netdev) {
5444                 netif_carrier_off(vsi->netdev);
5445                 netif_tx_disable(vsi->netdev);
5446         }
5447         i40e_vsi_disable_irq(vsi);
5448         i40e_vsi_stop_rings(vsi);
5449         i40e_napi_disable_all(vsi);
5450
5451         for (i = 0; i < vsi->num_queue_pairs; i++) {
5452                 i40e_clean_tx_ring(vsi->tx_rings[i]);
5453                 i40e_clean_rx_ring(vsi->rx_rings[i]);
5454         }
5455
5456         i40e_notify_client_of_netdev_close(vsi, false);
5457
5458 }
5459
5460 /**
5461  * i40e_setup_tc - configure multiple traffic classes
5462  * @netdev: net device to configure
5463  * @tc: number of traffic classes to enable
5464  **/
5465 static int i40e_setup_tc(struct net_device *netdev, u8 tc)
5466 {
5467         struct i40e_netdev_priv *np = netdev_priv(netdev);
5468         struct i40e_vsi *vsi = np->vsi;
5469         struct i40e_pf *pf = vsi->back;
5470         u8 enabled_tc = 0;
5471         int ret = -EINVAL;
5472         int i;
5473
5474         /* Check if DCB enabled to continue */
5475         if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5476                 netdev_info(netdev, "DCB is not enabled for adapter\n");
5477                 goto exit;
5478         }
5479
5480         /* Check if MFP enabled */
5481         if (pf->flags & I40E_FLAG_MFP_ENABLED) {
5482                 netdev_info(netdev, "Configuring TC not supported in MFP mode\n");
5483                 goto exit;
5484         }
5485
5486         /* Check whether tc count is within enabled limit */
5487         if (tc > i40e_pf_get_num_tc(pf)) {
5488                 netdev_info(netdev, "TC count greater than enabled on link for adapter\n");
5489                 goto exit;
5490         }
5491
5492         /* Generate TC map for number of tc requested */
5493         for (i = 0; i < tc; i++)
5494                 enabled_tc |= BIT(i);
5495
5496         /* Requesting same TC configuration as already enabled */
5497         if (enabled_tc == vsi->tc_config.enabled_tc)
5498                 return 0;
5499
5500         /* Quiesce VSI queues */
5501         i40e_quiesce_vsi(vsi);
5502
5503         /* Configure VSI for enabled TCs */
5504         ret = i40e_vsi_config_tc(vsi, enabled_tc);
5505         if (ret) {
5506                 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
5507                             vsi->seid);
5508                 goto exit;
5509         }
5510
5511         /* Unquiesce VSI */
5512         i40e_unquiesce_vsi(vsi);
5513
5514 exit:
5515         return ret;
5516 }
5517
5518 #ifdef I40E_FCOE
5519 int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto,
5520                     struct tc_to_netdev *tc)
5521 #else
5522 static int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto,
5523                            struct tc_to_netdev *tc)
5524 #endif
5525 {
5526         if (handle != TC_H_ROOT || tc->type != TC_SETUP_MQPRIO)
5527                 return -EINVAL;
5528         return i40e_setup_tc(netdev, tc->tc);
5529 }
5530
5531 /**
5532  * i40e_open - Called when a network interface is made active
5533  * @netdev: network interface device structure
5534  *
5535  * The open entry point is called when a network interface is made
5536  * active by the system (IFF_UP).  At this point all resources needed
5537  * for transmit and receive operations are allocated, the interrupt
5538  * handler is registered with the OS, the netdev watchdog subtask is
5539  * enabled, and the stack is notified that the interface is ready.
5540  *
5541  * Returns 0 on success, negative value on failure
5542  **/
5543 int i40e_open(struct net_device *netdev)
5544 {
5545         struct i40e_netdev_priv *np = netdev_priv(netdev);
5546         struct i40e_vsi *vsi = np->vsi;
5547         struct i40e_pf *pf = vsi->back;
5548         int err;
5549
5550         /* disallow open during test or if eeprom is broken */
5551         if (test_bit(__I40E_TESTING, &pf->state) ||
5552             test_bit(__I40E_BAD_EEPROM, &pf->state))
5553                 return -EBUSY;
5554
5555         netif_carrier_off(netdev);
5556
5557         err = i40e_vsi_open(vsi);
5558         if (err)
5559                 return err;
5560
5561         /* configure global TSO hardware offload settings */
5562         wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
5563                                                        TCP_FLAG_FIN) >> 16);
5564         wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
5565                                                        TCP_FLAG_FIN |
5566                                                        TCP_FLAG_CWR) >> 16);
5567         wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
5568
5569         udp_tunnel_get_rx_info(netdev);
5570
5571         return 0;
5572 }
5573
5574 /**
5575  * i40e_vsi_open -
5576  * @vsi: the VSI to open
5577  *
5578  * Finish initialization of the VSI.
5579  *
5580  * Returns 0 on success, negative value on failure
5581  **/
5582 int i40e_vsi_open(struct i40e_vsi *vsi)
5583 {
5584         struct i40e_pf *pf = vsi->back;
5585         char int_name[I40E_INT_NAME_STR_LEN];
5586         int err;
5587
5588         /* allocate descriptors */
5589         err = i40e_vsi_setup_tx_resources(vsi);
5590         if (err)
5591                 goto err_setup_tx;
5592         err = i40e_vsi_setup_rx_resources(vsi);
5593         if (err)
5594                 goto err_setup_rx;
5595
5596         err = i40e_vsi_configure(vsi);
5597         if (err)
5598                 goto err_setup_rx;
5599
5600         if (vsi->netdev) {
5601                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
5602                          dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
5603                 err = i40e_vsi_request_irq(vsi, int_name);
5604                 if (err)
5605                         goto err_setup_rx;
5606
5607                 /* Notify the stack of the actual queue counts. */
5608                 err = netif_set_real_num_tx_queues(vsi->netdev,
5609                                                    vsi->num_queue_pairs);
5610                 if (err)
5611                         goto err_set_queues;
5612
5613                 err = netif_set_real_num_rx_queues(vsi->netdev,
5614                                                    vsi->num_queue_pairs);
5615                 if (err)
5616                         goto err_set_queues;
5617
5618         } else if (vsi->type == I40E_VSI_FDIR) {
5619                 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
5620                          dev_driver_string(&pf->pdev->dev),
5621                          dev_name(&pf->pdev->dev));
5622                 err = i40e_vsi_request_irq(vsi, int_name);
5623
5624         } else {
5625                 err = -EINVAL;
5626                 goto err_setup_rx;
5627         }
5628
5629         err = i40e_up_complete(vsi);
5630         if (err)
5631                 goto err_up_complete;
5632
5633         return 0;
5634
5635 err_up_complete:
5636         i40e_down(vsi);
5637 err_set_queues:
5638         i40e_vsi_free_irq(vsi);
5639 err_setup_rx:
5640         i40e_vsi_free_rx_resources(vsi);
5641 err_setup_tx:
5642         i40e_vsi_free_tx_resources(vsi);
5643         if (vsi == pf->vsi[pf->lan_vsi])
5644                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED));
5645
5646         return err;
5647 }
5648
5649 /**
5650  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
5651  * @pf: Pointer to PF
5652  *
5653  * This function destroys the hlist where all the Flow Director
5654  * filters were saved.
5655  **/
5656 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
5657 {
5658         struct i40e_fdir_filter *filter;
5659         struct hlist_node *node2;
5660
5661         hlist_for_each_entry_safe(filter, node2,
5662                                   &pf->fdir_filter_list, fdir_node) {
5663                 hlist_del(&filter->fdir_node);
5664                 kfree(filter);
5665         }
5666         pf->fdir_pf_active_filters = 0;
5667 }
5668
5669 /**
5670  * i40e_close - Disables a network interface
5671  * @netdev: network interface device structure
5672  *
5673  * The close entry point is called when an interface is de-activated
5674  * by the OS.  The hardware is still under the driver's control, but
5675  * this netdev interface is disabled.
5676  *
5677  * Returns 0, this is not allowed to fail
5678  **/
5679 int i40e_close(struct net_device *netdev)
5680 {
5681         struct i40e_netdev_priv *np = netdev_priv(netdev);
5682         struct i40e_vsi *vsi = np->vsi;
5683
5684         i40e_vsi_close(vsi);
5685
5686         return 0;
5687 }
5688
5689 /**
5690  * i40e_do_reset - Start a PF or Core Reset sequence
5691  * @pf: board private structure
5692  * @reset_flags: which reset is requested
5693  *
5694  * The essential difference in resets is that the PF Reset
5695  * doesn't clear the packet buffers, doesn't reset the PE
5696  * firmware, and doesn't bother the other PFs on the chip.
5697  **/
5698 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags)
5699 {
5700         u32 val;
5701
5702         WARN_ON(in_interrupt());
5703
5704
5705         /* do the biggest reset indicated */
5706         if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
5707
5708                 /* Request a Global Reset
5709                  *
5710                  * This will start the chip's countdown to the actual full
5711                  * chip reset event, and a warning interrupt to be sent
5712                  * to all PFs, including the requestor.  Our handler
5713                  * for the warning interrupt will deal with the shutdown
5714                  * and recovery of the switch setup.
5715                  */
5716                 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
5717                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5718                 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
5719                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5720
5721         } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
5722
5723                 /* Request a Core Reset
5724                  *
5725                  * Same as Global Reset, except does *not* include the MAC/PHY
5726                  */
5727                 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
5728                 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
5729                 val |= I40E_GLGEN_RTRIG_CORER_MASK;
5730                 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
5731                 i40e_flush(&pf->hw);
5732
5733         } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) {
5734
5735                 /* Request a PF Reset
5736                  *
5737                  * Resets only the PF-specific registers
5738                  *
5739                  * This goes directly to the tear-down and rebuild of
5740                  * the switch, since we need to do all the recovery as
5741                  * for the Core Reset.
5742                  */
5743                 dev_dbg(&pf->pdev->dev, "PFR requested\n");
5744                 i40e_handle_reset_warning(pf);
5745
5746         } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
5747                 int v;
5748
5749                 /* Find the VSI(s) that requested a re-init */
5750                 dev_info(&pf->pdev->dev,
5751                          "VSI reinit requested\n");
5752                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5753                         struct i40e_vsi *vsi = pf->vsi[v];
5754
5755                         if (vsi != NULL &&
5756                             test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) {
5757                                 i40e_vsi_reinit_locked(pf->vsi[v]);
5758                                 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state);
5759                         }
5760                 }
5761         } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
5762                 int v;
5763
5764                 /* Find the VSI(s) that needs to be brought down */
5765                 dev_info(&pf->pdev->dev, "VSI down requested\n");
5766                 for (v = 0; v < pf->num_alloc_vsi; v++) {
5767                         struct i40e_vsi *vsi = pf->vsi[v];
5768
5769                         if (vsi != NULL &&
5770                             test_bit(__I40E_DOWN_REQUESTED, &vsi->state)) {
5771                                 set_bit(__I40E_DOWN, &vsi->state);
5772                                 i40e_down(vsi);
5773                                 clear_bit(__I40E_DOWN_REQUESTED, &vsi->state);
5774                         }
5775                 }
5776         } else {
5777                 dev_info(&pf->pdev->dev,
5778                          "bad reset request 0x%08x\n", reset_flags);
5779         }
5780 }
5781
5782 #ifdef CONFIG_I40E_DCB
5783 /**
5784  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
5785  * @pf: board private structure
5786  * @old_cfg: current DCB config
5787  * @new_cfg: new DCB config
5788  **/
5789 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
5790                             struct i40e_dcbx_config *old_cfg,
5791                             struct i40e_dcbx_config *new_cfg)
5792 {
5793         bool need_reconfig = false;
5794
5795         /* Check if ETS configuration has changed */
5796         if (memcmp(&new_cfg->etscfg,
5797                    &old_cfg->etscfg,
5798                    sizeof(new_cfg->etscfg))) {
5799                 /* If Priority Table has changed reconfig is needed */
5800                 if (memcmp(&new_cfg->etscfg.prioritytable,
5801                            &old_cfg->etscfg.prioritytable,
5802                            sizeof(new_cfg->etscfg.prioritytable))) {
5803                         need_reconfig = true;
5804                         dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
5805                 }
5806
5807                 if (memcmp(&new_cfg->etscfg.tcbwtable,
5808                            &old_cfg->etscfg.tcbwtable,
5809                            sizeof(new_cfg->etscfg.tcbwtable)))
5810                         dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
5811
5812                 if (memcmp(&new_cfg->etscfg.tsatable,
5813                            &old_cfg->etscfg.tsatable,
5814                            sizeof(new_cfg->etscfg.tsatable)))
5815                         dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
5816         }
5817
5818         /* Check if PFC configuration has changed */
5819         if (memcmp(&new_cfg->pfc,
5820                    &old_cfg->pfc,
5821                    sizeof(new_cfg->pfc))) {
5822                 need_reconfig = true;
5823                 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
5824         }
5825
5826         /* Check if APP Table has changed */
5827         if (memcmp(&new_cfg->app,
5828                    &old_cfg->app,
5829                    sizeof(new_cfg->app))) {
5830                 need_reconfig = true;
5831                 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
5832         }
5833
5834         dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
5835         return need_reconfig;
5836 }
5837
5838 /**
5839  * i40e_handle_lldp_event - Handle LLDP Change MIB event
5840  * @pf: board private structure
5841  * @e: event info posted on ARQ
5842  **/
5843 static int i40e_handle_lldp_event(struct i40e_pf *pf,
5844                                   struct i40e_arq_event_info *e)
5845 {
5846         struct i40e_aqc_lldp_get_mib *mib =
5847                 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
5848         struct i40e_hw *hw = &pf->hw;
5849         struct i40e_dcbx_config tmp_dcbx_cfg;
5850         bool need_reconfig = false;
5851         int ret = 0;
5852         u8 type;
5853
5854         /* Not DCB capable or capability disabled */
5855         if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
5856                 return ret;
5857
5858         /* Ignore if event is not for Nearest Bridge */
5859         type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
5860                 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
5861         dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
5862         if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
5863                 return ret;
5864
5865         /* Check MIB Type and return if event for Remote MIB update */
5866         type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
5867         dev_dbg(&pf->pdev->dev,
5868                 "LLDP event mib type %s\n", type ? "remote" : "local");
5869         if (type == I40E_AQ_LLDP_MIB_REMOTE) {
5870                 /* Update the remote cached instance and return */
5871                 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
5872                                 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
5873                                 &hw->remote_dcbx_config);
5874                 goto exit;
5875         }
5876
5877         /* Store the old configuration */
5878         tmp_dcbx_cfg = hw->local_dcbx_config;
5879
5880         /* Reset the old DCBx configuration data */
5881         memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
5882         /* Get updated DCBX data from firmware */
5883         ret = i40e_get_dcb_config(&pf->hw);
5884         if (ret) {
5885                 dev_info(&pf->pdev->dev,
5886                          "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
5887                          i40e_stat_str(&pf->hw, ret),
5888                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5889                 goto exit;
5890         }
5891
5892         /* No change detected in DCBX configs */
5893         if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
5894                     sizeof(tmp_dcbx_cfg))) {
5895                 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
5896                 goto exit;
5897         }
5898
5899         need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
5900                                                &hw->local_dcbx_config);
5901
5902         i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
5903
5904         if (!need_reconfig)
5905                 goto exit;
5906
5907         /* Enable DCB tagging only when more than one TC */
5908         if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
5909                 pf->flags |= I40E_FLAG_DCB_ENABLED;
5910         else
5911                 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
5912
5913         set_bit(__I40E_PORT_TX_SUSPENDED, &pf->state);
5914         /* Reconfiguration needed quiesce all VSIs */
5915         i40e_pf_quiesce_all_vsi(pf);
5916
5917         /* Changes in configuration update VEB/VSI */
5918         i40e_dcb_reconfigure(pf);
5919
5920         ret = i40e_resume_port_tx(pf);
5921
5922         clear_bit(__I40E_PORT_TX_SUSPENDED, &pf->state);
5923         /* In case of error no point in resuming VSIs */
5924         if (ret)
5925                 goto exit;
5926
5927         /* Wait for the PF's queues to be disabled */
5928         ret = i40e_pf_wait_queues_disabled(pf);
5929         if (ret) {
5930                 /* Schedule PF reset to recover */
5931                 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
5932                 i40e_service_event_schedule(pf);
5933         } else {
5934                 i40e_pf_unquiesce_all_vsi(pf);
5935                 /* Notify the client for the DCB changes */
5936                 i40e_notify_client_of_l2_param_changes(pf->vsi[pf->lan_vsi]);
5937         }
5938
5939 exit:
5940         return ret;
5941 }
5942 #endif /* CONFIG_I40E_DCB */
5943
5944 /**
5945  * i40e_do_reset_safe - Protected reset path for userland calls.
5946  * @pf: board private structure
5947  * @reset_flags: which reset is requested
5948  *
5949  **/
5950 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
5951 {
5952         rtnl_lock();
5953         i40e_do_reset(pf, reset_flags);
5954         rtnl_unlock();
5955 }
5956
5957 /**
5958  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
5959  * @pf: board private structure
5960  * @e: event info posted on ARQ
5961  *
5962  * Handler for LAN Queue Overflow Event generated by the firmware for PF
5963  * and VF queues
5964  **/
5965 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
5966                                            struct i40e_arq_event_info *e)
5967 {
5968         struct i40e_aqc_lan_overflow *data =
5969                 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
5970         u32 queue = le32_to_cpu(data->prtdcb_rupto);
5971         u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
5972         struct i40e_hw *hw = &pf->hw;
5973         struct i40e_vf *vf;
5974         u16 vf_id;
5975
5976         dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
5977                 queue, qtx_ctl);
5978
5979         /* Queue belongs to VF, find the VF and issue VF reset */
5980         if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
5981             >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
5982                 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
5983                          >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
5984                 vf_id -= hw->func_caps.vf_base_id;
5985                 vf = &pf->vf[vf_id];
5986                 i40e_vc_notify_vf_reset(vf);
5987                 /* Allow VF to process pending reset notification */
5988                 msleep(20);
5989                 i40e_reset_vf(vf, false);
5990         }
5991 }
5992
5993 /**
5994  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
5995  * @pf: board private structure
5996  **/
5997 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
5998 {
5999         u32 val, fcnt_prog;
6000
6001         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6002         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
6003         return fcnt_prog;
6004 }
6005
6006 /**
6007  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
6008  * @pf: board private structure
6009  **/
6010 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
6011 {
6012         u32 val, fcnt_prog;
6013
6014         val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
6015         fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
6016                     ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
6017                       I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
6018         return fcnt_prog;
6019 }
6020
6021 /**
6022  * i40e_get_global_fd_count - Get total FD filters programmed on device
6023  * @pf: board private structure
6024  **/
6025 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
6026 {
6027         u32 val, fcnt_prog;
6028
6029         val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
6030         fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
6031                     ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
6032                      I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
6033         return fcnt_prog;
6034 }
6035
6036 /**
6037  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
6038  * @pf: board private structure
6039  **/
6040 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
6041 {
6042         struct i40e_fdir_filter *filter;
6043         u32 fcnt_prog, fcnt_avail;
6044         struct hlist_node *node;
6045
6046         if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state))
6047                 return;
6048
6049         /* Check if, FD SB or ATR was auto disabled and if there is enough room
6050          * to re-enable
6051          */
6052         fcnt_prog = i40e_get_global_fd_count(pf);
6053         fcnt_avail = pf->fdir_pf_filter_count;
6054         if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
6055             (pf->fd_add_err == 0) ||
6056             (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) {
6057                 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
6058                     (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) {
6059                         pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
6060                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
6061                                 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
6062                 }
6063         }
6064
6065         /* Wait for some more space to be available to turn on ATR. We also
6066          * must check that no existing ntuple rules for TCP are in effect
6067          */
6068         if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) {
6069                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
6070                     (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED) &&
6071                     (pf->fd_tcp_rule == 0)) {
6072                         pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
6073                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
6074                                 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
6075                 }
6076         }
6077
6078         /* if hw had a problem adding a filter, delete it */
6079         if (pf->fd_inv > 0) {
6080                 hlist_for_each_entry_safe(filter, node,
6081                                           &pf->fdir_filter_list, fdir_node) {
6082                         if (filter->fd_id == pf->fd_inv) {
6083                                 hlist_del(&filter->fdir_node);
6084                                 kfree(filter);
6085                                 pf->fdir_pf_active_filters--;
6086                         }
6087                 }
6088         }
6089 }
6090
6091 #define I40E_MIN_FD_FLUSH_INTERVAL 10
6092 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
6093 /**
6094  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
6095  * @pf: board private structure
6096  **/
6097 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
6098 {
6099         unsigned long min_flush_time;
6100         int flush_wait_retry = 50;
6101         bool disable_atr = false;
6102         int fd_room;
6103         int reg;
6104
6105         if (!time_after(jiffies, pf->fd_flush_timestamp +
6106                                  (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
6107                 return;
6108
6109         /* If the flush is happening too quick and we have mostly SB rules we
6110          * should not re-enable ATR for some time.
6111          */
6112         min_flush_time = pf->fd_flush_timestamp +
6113                          (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
6114         fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
6115
6116         if (!(time_after(jiffies, min_flush_time)) &&
6117             (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
6118                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6119                         dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
6120                 disable_atr = true;
6121         }
6122
6123         pf->fd_flush_timestamp = jiffies;
6124         pf->auto_disable_flags |= I40E_FLAG_FD_ATR_ENABLED;
6125         /* flush all filters */
6126         wr32(&pf->hw, I40E_PFQF_CTL_1,
6127              I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
6128         i40e_flush(&pf->hw);
6129         pf->fd_flush_cnt++;
6130         pf->fd_add_err = 0;
6131         do {
6132                 /* Check FD flush status every 5-6msec */
6133                 usleep_range(5000, 6000);
6134                 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
6135                 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
6136                         break;
6137         } while (flush_wait_retry--);
6138         if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
6139                 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
6140         } else {
6141                 /* replay sideband filters */
6142                 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
6143                 if (!disable_atr)
6144                         pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
6145                 clear_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state);
6146                 if (I40E_DEBUG_FD & pf->hw.debug_mask)
6147                         dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
6148         }
6149 }
6150
6151 /**
6152  * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed
6153  * @pf: board private structure
6154  **/
6155 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
6156 {
6157         return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
6158 }
6159
6160 /* We can see up to 256 filter programming desc in transit if the filters are
6161  * being applied really fast; before we see the first
6162  * filter miss error on Rx queue 0. Accumulating enough error messages before
6163  * reacting will make sure we don't cause flush too often.
6164  */
6165 #define I40E_MAX_FD_PROGRAM_ERROR 256
6166
6167 /**
6168  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
6169  * @pf: board private structure
6170  **/
6171 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
6172 {
6173
6174         /* if interface is down do nothing */
6175         if (test_bit(__I40E_DOWN, &pf->state))
6176                 return;
6177
6178         if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state))
6179                 i40e_fdir_flush_and_replay(pf);
6180
6181         i40e_fdir_check_and_reenable(pf);
6182
6183 }
6184
6185 /**
6186  * i40e_vsi_link_event - notify VSI of a link event
6187  * @vsi: vsi to be notified
6188  * @link_up: link up or down
6189  **/
6190 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
6191 {
6192         if (!vsi || test_bit(__I40E_DOWN, &vsi->state))
6193                 return;
6194
6195         switch (vsi->type) {
6196         case I40E_VSI_MAIN:
6197 #ifdef I40E_FCOE
6198         case I40E_VSI_FCOE:
6199 #endif
6200                 if (!vsi->netdev || !vsi->netdev_registered)
6201                         break;
6202
6203                 if (link_up) {
6204                         netif_carrier_on(vsi->netdev);
6205                         netif_tx_wake_all_queues(vsi->netdev);
6206                 } else {
6207                         netif_carrier_off(vsi->netdev);
6208                         netif_tx_stop_all_queues(vsi->netdev);
6209                 }
6210                 break;
6211
6212         case I40E_VSI_SRIOV:
6213         case I40E_VSI_VMDQ2:
6214         case I40E_VSI_CTRL:
6215         case I40E_VSI_IWARP:
6216         case I40E_VSI_MIRROR:
6217         default:
6218                 /* there is no notification for other VSIs */
6219                 break;
6220         }
6221 }
6222
6223 /**
6224  * i40e_veb_link_event - notify elements on the veb of a link event
6225  * @veb: veb to be notified
6226  * @link_up: link up or down
6227  **/
6228 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
6229 {
6230         struct i40e_pf *pf;
6231         int i;
6232
6233         if (!veb || !veb->pf)
6234                 return;
6235         pf = veb->pf;
6236
6237         /* depth first... */
6238         for (i = 0; i < I40E_MAX_VEB; i++)
6239                 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
6240                         i40e_veb_link_event(pf->veb[i], link_up);
6241
6242         /* ... now the local VSIs */
6243         for (i = 0; i < pf->num_alloc_vsi; i++)
6244                 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
6245                         i40e_vsi_link_event(pf->vsi[i], link_up);
6246 }
6247
6248 /**
6249  * i40e_link_event - Update netif_carrier status
6250  * @pf: board private structure
6251  **/
6252 static void i40e_link_event(struct i40e_pf *pf)
6253 {
6254         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6255         u8 new_link_speed, old_link_speed;
6256         i40e_status status;
6257         bool new_link, old_link;
6258
6259         /* save off old link status information */
6260         pf->hw.phy.link_info_old = pf->hw.phy.link_info;
6261
6262         /* set this to force the get_link_status call to refresh state */
6263         pf->hw.phy.get_link_info = true;
6264
6265         old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
6266
6267         status = i40e_get_link_status(&pf->hw, &new_link);
6268         if (status) {
6269                 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
6270                         status);
6271                 return;
6272         }
6273
6274         old_link_speed = pf->hw.phy.link_info_old.link_speed;
6275         new_link_speed = pf->hw.phy.link_info.link_speed;
6276
6277         if (new_link == old_link &&
6278             new_link_speed == old_link_speed &&
6279             (test_bit(__I40E_DOWN, &vsi->state) ||
6280              new_link == netif_carrier_ok(vsi->netdev)))
6281                 return;
6282
6283         if (!test_bit(__I40E_DOWN, &vsi->state))
6284                 i40e_print_link_message(vsi, new_link);
6285
6286         /* Notify the base of the switch tree connected to
6287          * the link.  Floating VEBs are not notified.
6288          */
6289         if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
6290                 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
6291         else
6292                 i40e_vsi_link_event(vsi, new_link);
6293
6294         if (pf->vf)
6295                 i40e_vc_notify_link_state(pf);
6296
6297         if (pf->flags & I40E_FLAG_PTP)
6298                 i40e_ptp_set_increment(pf);
6299 }
6300
6301 /**
6302  * i40e_watchdog_subtask - periodic checks not using event driven response
6303  * @pf: board private structure
6304  **/
6305 static void i40e_watchdog_subtask(struct i40e_pf *pf)
6306 {
6307         int i;
6308
6309         /* if interface is down do nothing */
6310         if (test_bit(__I40E_DOWN, &pf->state) ||
6311             test_bit(__I40E_CONFIG_BUSY, &pf->state))
6312                 return;
6313
6314         /* make sure we don't do these things too often */
6315         if (time_before(jiffies, (pf->service_timer_previous +
6316                                   pf->service_timer_period)))
6317                 return;
6318         pf->service_timer_previous = jiffies;
6319
6320         if (pf->flags & I40E_FLAG_LINK_POLLING_ENABLED)
6321                 i40e_link_event(pf);
6322
6323         /* Update the stats for active netdevs so the network stack
6324          * can look at updated numbers whenever it cares to
6325          */
6326         for (i = 0; i < pf->num_alloc_vsi; i++)
6327                 if (pf->vsi[i] && pf->vsi[i]->netdev)
6328                         i40e_update_stats(pf->vsi[i]);
6329
6330         if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
6331                 /* Update the stats for the active switching components */
6332                 for (i = 0; i < I40E_MAX_VEB; i++)
6333                         if (pf->veb[i])
6334                                 i40e_update_veb_stats(pf->veb[i]);
6335         }
6336
6337         i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]);
6338 }
6339
6340 /**
6341  * i40e_reset_subtask - Set up for resetting the device and driver
6342  * @pf: board private structure
6343  **/
6344 static void i40e_reset_subtask(struct i40e_pf *pf)
6345 {
6346         u32 reset_flags = 0;
6347
6348         rtnl_lock();
6349         if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) {
6350                 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
6351                 clear_bit(__I40E_REINIT_REQUESTED, &pf->state);
6352         }
6353         if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) {
6354                 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
6355                 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
6356         }
6357         if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) {
6358                 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
6359                 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state);
6360         }
6361         if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) {
6362                 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
6363                 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state);
6364         }
6365         if (test_bit(__I40E_DOWN_REQUESTED, &pf->state)) {
6366                 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
6367                 clear_bit(__I40E_DOWN_REQUESTED, &pf->state);
6368         }
6369
6370         /* If there's a recovery already waiting, it takes
6371          * precedence before starting a new reset sequence.
6372          */
6373         if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) {
6374                 i40e_handle_reset_warning(pf);
6375                 goto unlock;
6376         }
6377
6378         /* If we're already down or resetting, just bail */
6379         if (reset_flags &&
6380             !test_bit(__I40E_DOWN, &pf->state) &&
6381             !test_bit(__I40E_CONFIG_BUSY, &pf->state))
6382                 i40e_do_reset(pf, reset_flags);
6383
6384 unlock:
6385         rtnl_unlock();
6386 }
6387
6388 /**
6389  * i40e_handle_link_event - Handle link event
6390  * @pf: board private structure
6391  * @e: event info posted on ARQ
6392  **/
6393 static void i40e_handle_link_event(struct i40e_pf *pf,
6394                                    struct i40e_arq_event_info *e)
6395 {
6396         struct i40e_aqc_get_link_status *status =
6397                 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
6398
6399         /* Do a new status request to re-enable LSE reporting
6400          * and load new status information into the hw struct
6401          * This completely ignores any state information
6402          * in the ARQ event info, instead choosing to always
6403          * issue the AQ update link status command.
6404          */
6405         i40e_link_event(pf);
6406
6407         /* check for unqualified module, if link is down */
6408         if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
6409             (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
6410             (!(status->link_info & I40E_AQ_LINK_UP)))
6411                 dev_err(&pf->pdev->dev,
6412                         "The driver failed to link because an unqualified module was detected.\n");
6413 }
6414
6415 /**
6416  * i40e_clean_adminq_subtask - Clean the AdminQ rings
6417  * @pf: board private structure
6418  **/
6419 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
6420 {
6421         struct i40e_arq_event_info event;
6422         struct i40e_hw *hw = &pf->hw;
6423         u16 pending, i = 0;
6424         i40e_status ret;
6425         u16 opcode;
6426         u32 oldval;
6427         u32 val;
6428
6429         /* Do not run clean AQ when PF reset fails */
6430         if (test_bit(__I40E_RESET_FAILED, &pf->state))
6431                 return;
6432
6433         /* check for error indications */
6434         val = rd32(&pf->hw, pf->hw.aq.arq.len);
6435         oldval = val;
6436         if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
6437                 if (hw->debug_mask & I40E_DEBUG_AQ)
6438                         dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
6439                 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
6440         }
6441         if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
6442                 if (hw->debug_mask & I40E_DEBUG_AQ)
6443                         dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
6444                 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
6445                 pf->arq_overflows++;
6446         }
6447         if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
6448                 if (hw->debug_mask & I40E_DEBUG_AQ)
6449                         dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
6450                 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
6451         }
6452         if (oldval != val)
6453                 wr32(&pf->hw, pf->hw.aq.arq.len, val);
6454
6455         val = rd32(&pf->hw, pf->hw.aq.asq.len);
6456         oldval = val;
6457         if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
6458                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6459                         dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
6460                 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
6461         }
6462         if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
6463                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6464                         dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
6465                 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
6466         }
6467         if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
6468                 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
6469                         dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
6470                 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
6471         }
6472         if (oldval != val)
6473                 wr32(&pf->hw, pf->hw.aq.asq.len, val);
6474
6475         event.buf_len = I40E_MAX_AQ_BUF_SIZE;
6476         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
6477         if (!event.msg_buf)
6478                 return;
6479
6480         do {
6481                 ret = i40e_clean_arq_element(hw, &event, &pending);
6482                 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
6483                         break;
6484                 else if (ret) {
6485                         dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
6486                         break;
6487                 }
6488
6489                 opcode = le16_to_cpu(event.desc.opcode);
6490                 switch (opcode) {
6491
6492                 case i40e_aqc_opc_get_link_status:
6493                         i40e_handle_link_event(pf, &event);
6494                         break;
6495                 case i40e_aqc_opc_send_msg_to_pf:
6496                         ret = i40e_vc_process_vf_msg(pf,
6497                                         le16_to_cpu(event.desc.retval),
6498                                         le32_to_cpu(event.desc.cookie_high),
6499                                         le32_to_cpu(event.desc.cookie_low),
6500                                         event.msg_buf,
6501                                         event.msg_len);
6502                         break;
6503                 case i40e_aqc_opc_lldp_update_mib:
6504                         dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
6505 #ifdef CONFIG_I40E_DCB
6506                         rtnl_lock();
6507                         ret = i40e_handle_lldp_event(pf, &event);
6508                         rtnl_unlock();
6509 #endif /* CONFIG_I40E_DCB */
6510                         break;
6511                 case i40e_aqc_opc_event_lan_overflow:
6512                         dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
6513                         i40e_handle_lan_overflow_event(pf, &event);
6514                         break;
6515                 case i40e_aqc_opc_send_msg_to_peer:
6516                         dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
6517                         break;
6518                 case i40e_aqc_opc_nvm_erase:
6519                 case i40e_aqc_opc_nvm_update:
6520                 case i40e_aqc_opc_oem_post_update:
6521                         i40e_debug(&pf->hw, I40E_DEBUG_NVM,
6522                                    "ARQ NVM operation 0x%04x completed\n",
6523                                    opcode);
6524                         break;
6525                 default:
6526                         dev_info(&pf->pdev->dev,
6527                                  "ARQ: Unknown event 0x%04x ignored\n",
6528                                  opcode);
6529                         break;
6530                 }
6531         } while (pending && (i++ < pf->adminq_work_limit));
6532
6533         clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state);
6534         /* re-enable Admin queue interrupt cause */
6535         val = rd32(hw, I40E_PFINT_ICR0_ENA);
6536         val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
6537         wr32(hw, I40E_PFINT_ICR0_ENA, val);
6538         i40e_flush(hw);
6539
6540         kfree(event.msg_buf);
6541 }
6542
6543 /**
6544  * i40e_verify_eeprom - make sure eeprom is good to use
6545  * @pf: board private structure
6546  **/
6547 static void i40e_verify_eeprom(struct i40e_pf *pf)
6548 {
6549         int err;
6550
6551         err = i40e_diag_eeprom_test(&pf->hw);
6552         if (err) {
6553                 /* retry in case of garbage read */
6554                 err = i40e_diag_eeprom_test(&pf->hw);
6555                 if (err) {
6556                         dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
6557                                  err);
6558                         set_bit(__I40E_BAD_EEPROM, &pf->state);
6559                 }
6560         }
6561
6562         if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) {
6563                 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
6564                 clear_bit(__I40E_BAD_EEPROM, &pf->state);
6565         }
6566 }
6567
6568 /**
6569  * i40e_enable_pf_switch_lb
6570  * @pf: pointer to the PF structure
6571  *
6572  * enable switch loop back or die - no point in a return value
6573  **/
6574 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
6575 {
6576         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6577         struct i40e_vsi_context ctxt;
6578         int ret;
6579
6580         ctxt.seid = pf->main_vsi_seid;
6581         ctxt.pf_num = pf->hw.pf_id;
6582         ctxt.vf_num = 0;
6583         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6584         if (ret) {
6585                 dev_info(&pf->pdev->dev,
6586                          "couldn't get PF vsi config, err %s aq_err %s\n",
6587                          i40e_stat_str(&pf->hw, ret),
6588                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6589                 return;
6590         }
6591         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6592         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6593         ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6594
6595         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6596         if (ret) {
6597                 dev_info(&pf->pdev->dev,
6598                          "update vsi switch failed, err %s aq_err %s\n",
6599                          i40e_stat_str(&pf->hw, ret),
6600                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6601         }
6602 }
6603
6604 /**
6605  * i40e_disable_pf_switch_lb
6606  * @pf: pointer to the PF structure
6607  *
6608  * disable switch loop back or die - no point in a return value
6609  **/
6610 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
6611 {
6612         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
6613         struct i40e_vsi_context ctxt;
6614         int ret;
6615
6616         ctxt.seid = pf->main_vsi_seid;
6617         ctxt.pf_num = pf->hw.pf_id;
6618         ctxt.vf_num = 0;
6619         ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
6620         if (ret) {
6621                 dev_info(&pf->pdev->dev,
6622                          "couldn't get PF vsi config, err %s aq_err %s\n",
6623                          i40e_stat_str(&pf->hw, ret),
6624                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6625                 return;
6626         }
6627         ctxt.flags = I40E_AQ_VSI_TYPE_PF;
6628         ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6629         ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6630
6631         ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
6632         if (ret) {
6633                 dev_info(&pf->pdev->dev,
6634                          "update vsi switch failed, err %s aq_err %s\n",
6635                          i40e_stat_str(&pf->hw, ret),
6636                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6637         }
6638 }
6639
6640 /**
6641  * i40e_config_bridge_mode - Configure the HW bridge mode
6642  * @veb: pointer to the bridge instance
6643  *
6644  * Configure the loop back mode for the LAN VSI that is downlink to the
6645  * specified HW bridge instance. It is expected this function is called
6646  * when a new HW bridge is instantiated.
6647  **/
6648 static void i40e_config_bridge_mode(struct i40e_veb *veb)
6649 {
6650         struct i40e_pf *pf = veb->pf;
6651
6652         if (pf->hw.debug_mask & I40E_DEBUG_LAN)
6653                 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
6654                          veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
6655         if (veb->bridge_mode & BRIDGE_MODE_VEPA)
6656                 i40e_disable_pf_switch_lb(pf);
6657         else
6658                 i40e_enable_pf_switch_lb(pf);
6659 }
6660
6661 /**
6662  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
6663  * @veb: pointer to the VEB instance
6664  *
6665  * This is a recursive function that first builds the attached VSIs then
6666  * recurses in to build the next layer of VEB.  We track the connections
6667  * through our own index numbers because the seid's from the HW could
6668  * change across the reset.
6669  **/
6670 static int i40e_reconstitute_veb(struct i40e_veb *veb)
6671 {
6672         struct i40e_vsi *ctl_vsi = NULL;
6673         struct i40e_pf *pf = veb->pf;
6674         int v, veb_idx;
6675         int ret;
6676
6677         /* build VSI that owns this VEB, temporarily attached to base VEB */
6678         for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
6679                 if (pf->vsi[v] &&
6680                     pf->vsi[v]->veb_idx == veb->idx &&
6681                     pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
6682                         ctl_vsi = pf->vsi[v];
6683                         break;
6684                 }
6685         }
6686         if (!ctl_vsi) {
6687                 dev_info(&pf->pdev->dev,
6688                          "missing owner VSI for veb_idx %d\n", veb->idx);
6689                 ret = -ENOENT;
6690                 goto end_reconstitute;
6691         }
6692         if (ctl_vsi != pf->vsi[pf->lan_vsi])
6693                 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6694         ret = i40e_add_vsi(ctl_vsi);
6695         if (ret) {
6696                 dev_info(&pf->pdev->dev,
6697                          "rebuild of veb_idx %d owner VSI failed: %d\n",
6698                          veb->idx, ret);
6699                 goto end_reconstitute;
6700         }
6701         i40e_vsi_reset_stats(ctl_vsi);
6702
6703         /* create the VEB in the switch and move the VSI onto the VEB */
6704         ret = i40e_add_veb(veb, ctl_vsi);
6705         if (ret)
6706                 goto end_reconstitute;
6707
6708         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
6709                 veb->bridge_mode = BRIDGE_MODE_VEB;
6710         else
6711                 veb->bridge_mode = BRIDGE_MODE_VEPA;
6712         i40e_config_bridge_mode(veb);
6713
6714         /* create the remaining VSIs attached to this VEB */
6715         for (v = 0; v < pf->num_alloc_vsi; v++) {
6716                 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
6717                         continue;
6718
6719                 if (pf->vsi[v]->veb_idx == veb->idx) {
6720                         struct i40e_vsi *vsi = pf->vsi[v];
6721
6722                         vsi->uplink_seid = veb->seid;
6723                         ret = i40e_add_vsi(vsi);
6724                         if (ret) {
6725                                 dev_info(&pf->pdev->dev,
6726                                          "rebuild of vsi_idx %d failed: %d\n",
6727                                          v, ret);
6728                                 goto end_reconstitute;
6729                         }
6730                         i40e_vsi_reset_stats(vsi);
6731                 }
6732         }
6733
6734         /* create any VEBs attached to this VEB - RECURSION */
6735         for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
6736                 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
6737                         pf->veb[veb_idx]->uplink_seid = veb->seid;
6738                         ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
6739                         if (ret)
6740                                 break;
6741                 }
6742         }
6743
6744 end_reconstitute:
6745         return ret;
6746 }
6747
6748 /**
6749  * i40e_get_capabilities - get info about the HW
6750  * @pf: the PF struct
6751  **/
6752 static int i40e_get_capabilities(struct i40e_pf *pf)
6753 {
6754         struct i40e_aqc_list_capabilities_element_resp *cap_buf;
6755         u16 data_size;
6756         int buf_len;
6757         int err;
6758
6759         buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
6760         do {
6761                 cap_buf = kzalloc(buf_len, GFP_KERNEL);
6762                 if (!cap_buf)
6763                         return -ENOMEM;
6764
6765                 /* this loads the data into the hw struct for us */
6766                 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
6767                                             &data_size,
6768                                             i40e_aqc_opc_list_func_capabilities,
6769                                             NULL);
6770                 /* data loaded, buffer no longer needed */
6771                 kfree(cap_buf);
6772
6773                 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
6774                         /* retry with a larger buffer */
6775                         buf_len = data_size;
6776                 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
6777                         dev_info(&pf->pdev->dev,
6778                                  "capability discovery failed, err %s aq_err %s\n",
6779                                  i40e_stat_str(&pf->hw, err),
6780                                  i40e_aq_str(&pf->hw,
6781                                              pf->hw.aq.asq_last_status));
6782                         return -ENODEV;
6783                 }
6784         } while (err);
6785
6786         if (pf->hw.debug_mask & I40E_DEBUG_USER)
6787                 dev_info(&pf->pdev->dev,
6788                          "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
6789                          pf->hw.pf_id, pf->hw.func_caps.num_vfs,
6790                          pf->hw.func_caps.num_msix_vectors,
6791                          pf->hw.func_caps.num_msix_vectors_vf,
6792                          pf->hw.func_caps.fd_filters_guaranteed,
6793                          pf->hw.func_caps.fd_filters_best_effort,
6794                          pf->hw.func_caps.num_tx_qp,
6795                          pf->hw.func_caps.num_vsis);
6796
6797 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
6798                        + pf->hw.func_caps.num_vfs)
6799         if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) {
6800                 dev_info(&pf->pdev->dev,
6801                          "got num_vsis %d, setting num_vsis to %d\n",
6802                          pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
6803                 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
6804         }
6805
6806         return 0;
6807 }
6808
6809 static int i40e_vsi_clear(struct i40e_vsi *vsi);
6810
6811 /**
6812  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
6813  * @pf: board private structure
6814  **/
6815 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
6816 {
6817         struct i40e_vsi *vsi;
6818
6819         /* quick workaround for an NVM issue that leaves a critical register
6820          * uninitialized
6821          */
6822         if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
6823                 static const u32 hkey[] = {
6824                         0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
6825                         0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
6826                         0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
6827                         0x95b3a76d};
6828                 int i;
6829
6830                 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
6831                         wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
6832         }
6833
6834         if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
6835                 return;
6836
6837         /* find existing VSI and see if it needs configuring */
6838         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
6839
6840         /* create a new VSI if none exists */
6841         if (!vsi) {
6842                 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
6843                                      pf->vsi[pf->lan_vsi]->seid, 0);
6844                 if (!vsi) {
6845                         dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
6846                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
6847                         return;
6848                 }
6849         }
6850
6851         i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
6852 }
6853
6854 /**
6855  * i40e_fdir_teardown - release the Flow Director resources
6856  * @pf: board private structure
6857  **/
6858 static void i40e_fdir_teardown(struct i40e_pf *pf)
6859 {
6860         struct i40e_vsi *vsi;
6861
6862         i40e_fdir_filter_exit(pf);
6863         vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
6864         if (vsi)
6865                 i40e_vsi_release(vsi);
6866 }
6867
6868 /**
6869  * i40e_prep_for_reset - prep for the core to reset
6870  * @pf: board private structure
6871  *
6872  * Close up the VFs and other things in prep for PF Reset.
6873   **/
6874 static void i40e_prep_for_reset(struct i40e_pf *pf)
6875 {
6876         struct i40e_hw *hw = &pf->hw;
6877         i40e_status ret = 0;
6878         u32 v;
6879
6880         clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state);
6881         if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state))
6882                 return;
6883         if (i40e_check_asq_alive(&pf->hw))
6884                 i40e_vc_notify_reset(pf);
6885
6886         dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
6887
6888         /* quiesce the VSIs and their queues that are not already DOWN */
6889         i40e_pf_quiesce_all_vsi(pf);
6890
6891         for (v = 0; v < pf->num_alloc_vsi; v++) {
6892                 if (pf->vsi[v])
6893                         pf->vsi[v]->seid = 0;
6894         }
6895
6896         i40e_shutdown_adminq(&pf->hw);
6897
6898         /* call shutdown HMC */
6899         if (hw->hmc.hmc_obj) {
6900                 ret = i40e_shutdown_lan_hmc(hw);
6901                 if (ret)
6902                         dev_warn(&pf->pdev->dev,
6903                                  "shutdown_lan_hmc failed: %d\n", ret);
6904         }
6905 }
6906
6907 /**
6908  * i40e_send_version - update firmware with driver version
6909  * @pf: PF struct
6910  */
6911 static void i40e_send_version(struct i40e_pf *pf)
6912 {
6913         struct i40e_driver_version dv;
6914
6915         dv.major_version = DRV_VERSION_MAJOR;
6916         dv.minor_version = DRV_VERSION_MINOR;
6917         dv.build_version = DRV_VERSION_BUILD;
6918         dv.subbuild_version = 0;
6919         strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
6920         i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
6921 }
6922
6923 /**
6924  * i40e_reset_and_rebuild - reset and rebuild using a saved config
6925  * @pf: board private structure
6926  * @reinit: if the Main VSI needs to re-initialized.
6927  **/
6928 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit)
6929 {
6930         struct i40e_hw *hw = &pf->hw;
6931         u8 set_fc_aq_fail = 0;
6932         i40e_status ret;
6933         u32 val;
6934         u32 v;
6935
6936         /* Now we wait for GRST to settle out.
6937          * We don't have to delete the VEBs or VSIs from the hw switch
6938          * because the reset will make them disappear.
6939          */
6940         ret = i40e_pf_reset(hw);
6941         if (ret) {
6942                 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
6943                 set_bit(__I40E_RESET_FAILED, &pf->state);
6944                 goto clear_recovery;
6945         }
6946         pf->pfr_count++;
6947
6948         if (test_bit(__I40E_DOWN, &pf->state))
6949                 goto clear_recovery;
6950         dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
6951
6952         /* rebuild the basics for the AdminQ, HMC, and initial HW switch */
6953         ret = i40e_init_adminq(&pf->hw);
6954         if (ret) {
6955                 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
6956                          i40e_stat_str(&pf->hw, ret),
6957                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6958                 goto clear_recovery;
6959         }
6960
6961         /* re-verify the eeprom if we just had an EMP reset */
6962         if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state))
6963                 i40e_verify_eeprom(pf);
6964
6965         i40e_clear_pxe_mode(hw);
6966         ret = i40e_get_capabilities(pf);
6967         if (ret)
6968                 goto end_core_reset;
6969
6970         ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
6971                                 hw->func_caps.num_rx_qp,
6972                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
6973         if (ret) {
6974                 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
6975                 goto end_core_reset;
6976         }
6977         ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
6978         if (ret) {
6979                 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
6980                 goto end_core_reset;
6981         }
6982
6983 #ifdef CONFIG_I40E_DCB
6984         ret = i40e_init_pf_dcb(pf);
6985         if (ret) {
6986                 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
6987                 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
6988                 /* Continue without DCB enabled */
6989         }
6990 #endif /* CONFIG_I40E_DCB */
6991 #ifdef I40E_FCOE
6992         i40e_init_pf_fcoe(pf);
6993
6994 #endif
6995         /* do basic switch setup */
6996         ret = i40e_setup_pf_switch(pf, reinit);
6997         if (ret)
6998                 goto end_core_reset;
6999
7000         /* The driver only wants link up/down and module qualification
7001          * reports from firmware.  Note the negative logic.
7002          */
7003         ret = i40e_aq_set_phy_int_mask(&pf->hw,
7004                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
7005                                          I40E_AQ_EVENT_MEDIA_NA |
7006                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
7007         if (ret)
7008                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
7009                          i40e_stat_str(&pf->hw, ret),
7010                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7011
7012         /* make sure our flow control settings are restored */
7013         ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
7014         if (ret)
7015                 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
7016                         i40e_stat_str(&pf->hw, ret),
7017                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7018
7019         /* Rebuild the VSIs and VEBs that existed before reset.
7020          * They are still in our local switch element arrays, so only
7021          * need to rebuild the switch model in the HW.
7022          *
7023          * If there were VEBs but the reconstitution failed, we'll try
7024          * try to recover minimal use by getting the basic PF VSI working.
7025          */
7026         if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) {
7027                 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
7028                 /* find the one VEB connected to the MAC, and find orphans */
7029                 for (v = 0; v < I40E_MAX_VEB; v++) {
7030                         if (!pf->veb[v])
7031                                 continue;
7032
7033                         if (pf->veb[v]->uplink_seid == pf->mac_seid ||
7034                             pf->veb[v]->uplink_seid == 0) {
7035                                 ret = i40e_reconstitute_veb(pf->veb[v]);
7036
7037                                 if (!ret)
7038                                         continue;
7039
7040                                 /* If Main VEB failed, we're in deep doodoo,
7041                                  * so give up rebuilding the switch and set up
7042                                  * for minimal rebuild of PF VSI.
7043                                  * If orphan failed, we'll report the error
7044                                  * but try to keep going.
7045                                  */
7046                                 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
7047                                         dev_info(&pf->pdev->dev,
7048                                                  "rebuild of switch failed: %d, will try to set up simple PF connection\n",
7049                                                  ret);
7050                                         pf->vsi[pf->lan_vsi]->uplink_seid
7051                                                                 = pf->mac_seid;
7052                                         break;
7053                                 } else if (pf->veb[v]->uplink_seid == 0) {
7054                                         dev_info(&pf->pdev->dev,
7055                                                  "rebuild of orphan VEB failed: %d\n",
7056                                                  ret);
7057                                 }
7058                         }
7059                 }
7060         }
7061
7062         if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) {
7063                 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
7064                 /* no VEB, so rebuild only the Main VSI */
7065                 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]);
7066                 if (ret) {
7067                         dev_info(&pf->pdev->dev,
7068                                  "rebuild of Main VSI failed: %d\n", ret);
7069                         goto end_core_reset;
7070                 }
7071         }
7072
7073         /* Reconfigure hardware for allowing smaller MSS in the case
7074          * of TSO, so that we avoid the MDD being fired and causing
7075          * a reset in the case of small MSS+TSO.
7076          */
7077 #define I40E_REG_MSS          0x000E64DC
7078 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
7079 #define I40E_64BYTE_MSS       0x400000
7080         val = rd32(hw, I40E_REG_MSS);
7081         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
7082                 val &= ~I40E_REG_MSS_MIN_MASK;
7083                 val |= I40E_64BYTE_MSS;
7084                 wr32(hw, I40E_REG_MSS, val);
7085         }
7086
7087         if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
7088                 msleep(75);
7089                 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
7090                 if (ret)
7091                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
7092                                  i40e_stat_str(&pf->hw, ret),
7093                                  i40e_aq_str(&pf->hw,
7094                                              pf->hw.aq.asq_last_status));
7095         }
7096         /* reinit the misc interrupt */
7097         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7098                 ret = i40e_setup_misc_vector(pf);
7099
7100         /* Add a filter to drop all Flow control frames from any VSI from being
7101          * transmitted. By doing so we stop a malicious VF from sending out
7102          * PAUSE or PFC frames and potentially controlling traffic for other
7103          * PF/VF VSIs.
7104          * The FW can still send Flow control frames if enabled.
7105          */
7106         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
7107                                                        pf->main_vsi_seid);
7108
7109         /* restart the VSIs that were rebuilt and running before the reset */
7110         i40e_pf_unquiesce_all_vsi(pf);
7111
7112         if (pf->num_alloc_vfs) {
7113                 for (v = 0; v < pf->num_alloc_vfs; v++)
7114                         i40e_reset_vf(&pf->vf[v], true);
7115         }
7116
7117         /* tell the firmware that we're starting */
7118         i40e_send_version(pf);
7119
7120 end_core_reset:
7121         clear_bit(__I40E_RESET_FAILED, &pf->state);
7122 clear_recovery:
7123         clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state);
7124 }
7125
7126 /**
7127  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
7128  * @pf: board private structure
7129  *
7130  * Close up the VFs and other things in prep for a Core Reset,
7131  * then get ready to rebuild the world.
7132  **/
7133 static void i40e_handle_reset_warning(struct i40e_pf *pf)
7134 {
7135         i40e_prep_for_reset(pf);
7136         i40e_reset_and_rebuild(pf, false);
7137 }
7138
7139 /**
7140  * i40e_handle_mdd_event
7141  * @pf: pointer to the PF structure
7142  *
7143  * Called from the MDD irq handler to identify possibly malicious vfs
7144  **/
7145 static void i40e_handle_mdd_event(struct i40e_pf *pf)
7146 {
7147         struct i40e_hw *hw = &pf->hw;
7148         bool mdd_detected = false;
7149         bool pf_mdd_detected = false;
7150         struct i40e_vf *vf;
7151         u32 reg;
7152         int i;
7153
7154         if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state))
7155                 return;
7156
7157         /* find what triggered the MDD event */
7158         reg = rd32(hw, I40E_GL_MDET_TX);
7159         if (reg & I40E_GL_MDET_TX_VALID_MASK) {
7160                 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
7161                                 I40E_GL_MDET_TX_PF_NUM_SHIFT;
7162                 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
7163                                 I40E_GL_MDET_TX_VF_NUM_SHIFT;
7164                 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
7165                                 I40E_GL_MDET_TX_EVENT_SHIFT;
7166                 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
7167                                 I40E_GL_MDET_TX_QUEUE_SHIFT) -
7168                                 pf->hw.func_caps.base_queue;
7169                 if (netif_msg_tx_err(pf))
7170                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
7171                                  event, queue, pf_num, vf_num);
7172                 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
7173                 mdd_detected = true;
7174         }
7175         reg = rd32(hw, I40E_GL_MDET_RX);
7176         if (reg & I40E_GL_MDET_RX_VALID_MASK) {
7177                 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
7178                                 I40E_GL_MDET_RX_FUNCTION_SHIFT;
7179                 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
7180                                 I40E_GL_MDET_RX_EVENT_SHIFT;
7181                 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
7182                                 I40E_GL_MDET_RX_QUEUE_SHIFT) -
7183                                 pf->hw.func_caps.base_queue;
7184                 if (netif_msg_rx_err(pf))
7185                         dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
7186                                  event, queue, func);
7187                 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
7188                 mdd_detected = true;
7189         }
7190
7191         if (mdd_detected) {
7192                 reg = rd32(hw, I40E_PF_MDET_TX);
7193                 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
7194                         wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
7195                         dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n");
7196                         pf_mdd_detected = true;
7197                 }
7198                 reg = rd32(hw, I40E_PF_MDET_RX);
7199                 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
7200                         wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
7201                         dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n");
7202                         pf_mdd_detected = true;
7203                 }
7204                 /* Queue belongs to the PF, initiate a reset */
7205                 if (pf_mdd_detected) {
7206                         set_bit(__I40E_PF_RESET_REQUESTED, &pf->state);
7207                         i40e_service_event_schedule(pf);
7208                 }
7209         }
7210
7211         /* see if one of the VFs needs its hand slapped */
7212         for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
7213                 vf = &(pf->vf[i]);
7214                 reg = rd32(hw, I40E_VP_MDET_TX(i));
7215                 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
7216                         wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
7217                         vf->num_mdd_events++;
7218                         dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
7219                                  i);
7220                 }
7221
7222                 reg = rd32(hw, I40E_VP_MDET_RX(i));
7223                 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
7224                         wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
7225                         vf->num_mdd_events++;
7226                         dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
7227                                  i);
7228                 }
7229
7230                 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) {
7231                         dev_info(&pf->pdev->dev,
7232                                  "Too many MDD events on VF %d, disabled\n", i);
7233                         dev_info(&pf->pdev->dev,
7234                                  "Use PF Control I/F to re-enable the VF\n");
7235                         set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states);
7236                 }
7237         }
7238
7239         /* re-enable mdd interrupt cause */
7240         clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state);
7241         reg = rd32(hw, I40E_PFINT_ICR0_ENA);
7242         reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
7243         wr32(hw, I40E_PFINT_ICR0_ENA, reg);
7244         i40e_flush(hw);
7245 }
7246
7247 /**
7248  * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW
7249  * @pf: board private structure
7250  **/
7251 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
7252 {
7253         struct i40e_hw *hw = &pf->hw;
7254         i40e_status ret;
7255         __be16 port;
7256         int i;
7257
7258         if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC))
7259                 return;
7260
7261         pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC;
7262
7263         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
7264                 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
7265                         pf->pending_udp_bitmap &= ~BIT_ULL(i);
7266                         port = pf->udp_ports[i].index;
7267                         if (port)
7268                                 ret = i40e_aq_add_udp_tunnel(hw, port,
7269                                                         pf->udp_ports[i].type,
7270                                                         NULL, NULL);
7271                         else
7272                                 ret = i40e_aq_del_udp_tunnel(hw, i, NULL);
7273
7274                         if (ret) {
7275                                 dev_dbg(&pf->pdev->dev,
7276                                         "%s %s port %d, index %d failed, err %s aq_err %s\n",
7277                                         pf->udp_ports[i].type ? "vxlan" : "geneve",
7278                                         port ? "add" : "delete",
7279                                         ntohs(port), i,
7280                                         i40e_stat_str(&pf->hw, ret),
7281                                         i40e_aq_str(&pf->hw,
7282                                                     pf->hw.aq.asq_last_status));
7283                                 pf->udp_ports[i].index = 0;
7284                         }
7285                 }
7286         }
7287 }
7288
7289 /**
7290  * i40e_service_task - Run the driver's async subtasks
7291  * @work: pointer to work_struct containing our data
7292  **/
7293 static void i40e_service_task(struct work_struct *work)
7294 {
7295         struct i40e_pf *pf = container_of(work,
7296                                           struct i40e_pf,
7297                                           service_task);
7298         unsigned long start_time = jiffies;
7299
7300         /* don't bother with service tasks if a reset is in progress */
7301         if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
7302                 return;
7303         }
7304
7305         if (test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state))
7306                 return;
7307
7308         i40e_detect_recover_hung(pf);
7309         i40e_sync_filters_subtask(pf);
7310         i40e_reset_subtask(pf);
7311         i40e_handle_mdd_event(pf);
7312         i40e_vc_process_vflr_event(pf);
7313         i40e_watchdog_subtask(pf);
7314         i40e_fdir_reinit_subtask(pf);
7315         i40e_client_subtask(pf);
7316         i40e_sync_filters_subtask(pf);
7317         i40e_sync_udp_filters_subtask(pf);
7318         i40e_clean_adminq_subtask(pf);
7319
7320         /* flush memory to make sure state is correct before next watchdog */
7321         smp_mb__before_atomic();
7322         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
7323
7324         /* If the tasks have taken longer than one timer cycle or there
7325          * is more work to be done, reschedule the service task now
7326          * rather than wait for the timer to tick again.
7327          */
7328         if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
7329             test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state)            ||
7330             test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)               ||
7331             test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state))
7332                 i40e_service_event_schedule(pf);
7333 }
7334
7335 /**
7336  * i40e_service_timer - timer callback
7337  * @data: pointer to PF struct
7338  **/
7339 static void i40e_service_timer(unsigned long data)
7340 {
7341         struct i40e_pf *pf = (struct i40e_pf *)data;
7342
7343         mod_timer(&pf->service_timer,
7344                   round_jiffies(jiffies + pf->service_timer_period));
7345         i40e_service_event_schedule(pf);
7346 }
7347
7348 /**
7349  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
7350  * @vsi: the VSI being configured
7351  **/
7352 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
7353 {
7354         struct i40e_pf *pf = vsi->back;
7355
7356         switch (vsi->type) {
7357         case I40E_VSI_MAIN:
7358                 vsi->alloc_queue_pairs = pf->num_lan_qps;
7359                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7360                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7361                 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7362                         vsi->num_q_vectors = pf->num_lan_msix;
7363                 else
7364                         vsi->num_q_vectors = 1;
7365
7366                 break;
7367
7368         case I40E_VSI_FDIR:
7369                 vsi->alloc_queue_pairs = 1;
7370                 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT,
7371                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7372                 vsi->num_q_vectors = pf->num_fdsb_msix;
7373                 break;
7374
7375         case I40E_VSI_VMDQ2:
7376                 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
7377                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7378                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7379                 vsi->num_q_vectors = pf->num_vmdq_msix;
7380                 break;
7381
7382         case I40E_VSI_SRIOV:
7383                 vsi->alloc_queue_pairs = pf->num_vf_qps;
7384                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7385                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7386                 break;
7387
7388 #ifdef I40E_FCOE
7389         case I40E_VSI_FCOE:
7390                 vsi->alloc_queue_pairs = pf->num_fcoe_qps;
7391                 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
7392                                       I40E_REQ_DESCRIPTOR_MULTIPLE);
7393                 vsi->num_q_vectors = pf->num_fcoe_msix;
7394                 break;
7395
7396 #endif /* I40E_FCOE */
7397         default:
7398                 WARN_ON(1);
7399                 return -ENODATA;
7400         }
7401
7402         return 0;
7403 }
7404
7405 /**
7406  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
7407  * @type: VSI pointer
7408  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
7409  *
7410  * On error: returns error code (negative)
7411  * On success: returns 0
7412  **/
7413 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
7414 {
7415         int size;
7416         int ret = 0;
7417
7418         /* allocate memory for both Tx and Rx ring pointers */
7419         size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2;
7420         vsi->tx_rings = kzalloc(size, GFP_KERNEL);
7421         if (!vsi->tx_rings)
7422                 return -ENOMEM;
7423         vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs];
7424
7425         if (alloc_qvectors) {
7426                 /* allocate memory for q_vector pointers */
7427                 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
7428                 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
7429                 if (!vsi->q_vectors) {
7430                         ret = -ENOMEM;
7431                         goto err_vectors;
7432                 }
7433         }
7434         return ret;
7435
7436 err_vectors:
7437         kfree(vsi->tx_rings);
7438         return ret;
7439 }
7440
7441 /**
7442  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
7443  * @pf: board private structure
7444  * @type: type of VSI
7445  *
7446  * On error: returns error code (negative)
7447  * On success: returns vsi index in PF (positive)
7448  **/
7449 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
7450 {
7451         int ret = -ENODEV;
7452         struct i40e_vsi *vsi;
7453         int vsi_idx;
7454         int i;
7455
7456         /* Need to protect the allocation of the VSIs at the PF level */
7457         mutex_lock(&pf->switch_mutex);
7458
7459         /* VSI list may be fragmented if VSI creation/destruction has
7460          * been happening.  We can afford to do a quick scan to look
7461          * for any free VSIs in the list.
7462          *
7463          * find next empty vsi slot, looping back around if necessary
7464          */
7465         i = pf->next_vsi;
7466         while (i < pf->num_alloc_vsi && pf->vsi[i])
7467                 i++;
7468         if (i >= pf->num_alloc_vsi) {
7469                 i = 0;
7470                 while (i < pf->next_vsi && pf->vsi[i])
7471                         i++;
7472         }
7473
7474         if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
7475                 vsi_idx = i;             /* Found one! */
7476         } else {
7477                 ret = -ENODEV;
7478                 goto unlock_pf;  /* out of VSI slots! */
7479         }
7480         pf->next_vsi = ++i;
7481
7482         vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
7483         if (!vsi) {
7484                 ret = -ENOMEM;
7485                 goto unlock_pf;
7486         }
7487         vsi->type = type;
7488         vsi->back = pf;
7489         set_bit(__I40E_DOWN, &vsi->state);
7490         vsi->flags = 0;
7491         vsi->idx = vsi_idx;
7492         vsi->int_rate_limit = 0;
7493         vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
7494                                 pf->rss_table_size : 64;
7495         vsi->netdev_registered = false;
7496         vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
7497         hash_init(vsi->mac_filter_hash);
7498         vsi->irqs_ready = false;
7499
7500         ret = i40e_set_num_rings_in_vsi(vsi);
7501         if (ret)
7502                 goto err_rings;
7503
7504         ret = i40e_vsi_alloc_arrays(vsi, true);
7505         if (ret)
7506                 goto err_rings;
7507
7508         /* Setup default MSIX irq handler for VSI */
7509         i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
7510
7511         /* Initialize VSI lock */
7512         spin_lock_init(&vsi->mac_filter_hash_lock);
7513         pf->vsi[vsi_idx] = vsi;
7514         ret = vsi_idx;
7515         goto unlock_pf;
7516
7517 err_rings:
7518         pf->next_vsi = i - 1;
7519         kfree(vsi);
7520 unlock_pf:
7521         mutex_unlock(&pf->switch_mutex);
7522         return ret;
7523 }
7524
7525 /**
7526  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
7527  * @type: VSI pointer
7528  * @free_qvectors: a bool to specify if q_vectors need to be freed.
7529  *
7530  * On error: returns error code (negative)
7531  * On success: returns 0
7532  **/
7533 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
7534 {
7535         /* free the ring and vector containers */
7536         if (free_qvectors) {
7537                 kfree(vsi->q_vectors);
7538                 vsi->q_vectors = NULL;
7539         }
7540         kfree(vsi->tx_rings);
7541         vsi->tx_rings = NULL;
7542         vsi->rx_rings = NULL;
7543 }
7544
7545 /**
7546  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
7547  * and lookup table
7548  * @vsi: Pointer to VSI structure
7549  */
7550 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
7551 {
7552         if (!vsi)
7553                 return;
7554
7555         kfree(vsi->rss_hkey_user);
7556         vsi->rss_hkey_user = NULL;
7557
7558         kfree(vsi->rss_lut_user);
7559         vsi->rss_lut_user = NULL;
7560 }
7561
7562 /**
7563  * i40e_vsi_clear - Deallocate the VSI provided
7564  * @vsi: the VSI being un-configured
7565  **/
7566 static int i40e_vsi_clear(struct i40e_vsi *vsi)
7567 {
7568         struct i40e_pf *pf;
7569
7570         if (!vsi)
7571                 return 0;
7572
7573         if (!vsi->back)
7574                 goto free_vsi;
7575         pf = vsi->back;
7576
7577         mutex_lock(&pf->switch_mutex);
7578         if (!pf->vsi[vsi->idx]) {
7579                 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n",
7580                         vsi->idx, vsi->idx, vsi, vsi->type);
7581                 goto unlock_vsi;
7582         }
7583
7584         if (pf->vsi[vsi->idx] != vsi) {
7585                 dev_err(&pf->pdev->dev,
7586                         "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n",
7587                         pf->vsi[vsi->idx]->idx,
7588                         pf->vsi[vsi->idx],
7589                         pf->vsi[vsi->idx]->type,
7590                         vsi->idx, vsi, vsi->type);
7591                 goto unlock_vsi;
7592         }
7593
7594         /* updates the PF for this cleared vsi */
7595         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
7596         i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
7597
7598         i40e_vsi_free_arrays(vsi, true);
7599         i40e_clear_rss_config_user(vsi);
7600
7601         pf->vsi[vsi->idx] = NULL;
7602         if (vsi->idx < pf->next_vsi)
7603                 pf->next_vsi = vsi->idx;
7604
7605 unlock_vsi:
7606         mutex_unlock(&pf->switch_mutex);
7607 free_vsi:
7608         kfree(vsi);
7609
7610         return 0;
7611 }
7612
7613 /**
7614  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
7615  * @vsi: the VSI being cleaned
7616  **/
7617 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
7618 {
7619         int i;
7620
7621         if (vsi->tx_rings && vsi->tx_rings[0]) {
7622                 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7623                         kfree_rcu(vsi->tx_rings[i], rcu);
7624                         vsi->tx_rings[i] = NULL;
7625                         vsi->rx_rings[i] = NULL;
7626                 }
7627         }
7628 }
7629
7630 /**
7631  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
7632  * @vsi: the VSI being configured
7633  **/
7634 static int i40e_alloc_rings(struct i40e_vsi *vsi)
7635 {
7636         struct i40e_ring *tx_ring, *rx_ring;
7637         struct i40e_pf *pf = vsi->back;
7638         int i;
7639
7640         /* Set basic values in the rings to be used later during open() */
7641         for (i = 0; i < vsi->alloc_queue_pairs; i++) {
7642                 /* allocate space for both Tx and Rx in one shot */
7643                 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL);
7644                 if (!tx_ring)
7645                         goto err_out;
7646
7647                 tx_ring->queue_index = i;
7648                 tx_ring->reg_idx = vsi->base_queue + i;
7649                 tx_ring->ring_active = false;
7650                 tx_ring->vsi = vsi;
7651                 tx_ring->netdev = vsi->netdev;
7652                 tx_ring->dev = &pf->pdev->dev;
7653                 tx_ring->count = vsi->num_desc;
7654                 tx_ring->size = 0;
7655                 tx_ring->dcb_tc = 0;
7656                 if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE)
7657                         tx_ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
7658                 tx_ring->tx_itr_setting = pf->tx_itr_default;
7659                 vsi->tx_rings[i] = tx_ring;
7660
7661                 rx_ring = &tx_ring[1];
7662                 rx_ring->queue_index = i;
7663                 rx_ring->reg_idx = vsi->base_queue + i;
7664                 rx_ring->ring_active = false;
7665                 rx_ring->vsi = vsi;
7666                 rx_ring->netdev = vsi->netdev;
7667                 rx_ring->dev = &pf->pdev->dev;
7668                 rx_ring->count = vsi->num_desc;
7669                 rx_ring->size = 0;
7670                 rx_ring->dcb_tc = 0;
7671                 rx_ring->rx_itr_setting = pf->rx_itr_default;
7672                 vsi->rx_rings[i] = rx_ring;
7673         }
7674
7675         return 0;
7676
7677 err_out:
7678         i40e_vsi_clear_rings(vsi);
7679         return -ENOMEM;
7680 }
7681
7682 /**
7683  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
7684  * @pf: board private structure
7685  * @vectors: the number of MSI-X vectors to request
7686  *
7687  * Returns the number of vectors reserved, or error
7688  **/
7689 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
7690 {
7691         vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
7692                                         I40E_MIN_MSIX, vectors);
7693         if (vectors < 0) {
7694                 dev_info(&pf->pdev->dev,
7695                          "MSI-X vector reservation failed: %d\n", vectors);
7696                 vectors = 0;
7697         }
7698
7699         return vectors;
7700 }
7701
7702 /**
7703  * i40e_init_msix - Setup the MSIX capability
7704  * @pf: board private structure
7705  *
7706  * Work with the OS to set up the MSIX vectors needed.
7707  *
7708  * Returns the number of vectors reserved or negative on failure
7709  **/
7710 static int i40e_init_msix(struct i40e_pf *pf)
7711 {
7712         struct i40e_hw *hw = &pf->hw;
7713         int vectors_left;
7714         int v_budget, i;
7715         int v_actual;
7716         int iwarp_requested = 0;
7717
7718         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7719                 return -ENODEV;
7720
7721         /* The number of vectors we'll request will be comprised of:
7722          *   - Add 1 for "other" cause for Admin Queue events, etc.
7723          *   - The number of LAN queue pairs
7724          *      - Queues being used for RSS.
7725          *              We don't need as many as max_rss_size vectors.
7726          *              use rss_size instead in the calculation since that
7727          *              is governed by number of cpus in the system.
7728          *      - assumes symmetric Tx/Rx pairing
7729          *   - The number of VMDq pairs
7730          *   - The CPU count within the NUMA node if iWARP is enabled
7731 #ifdef I40E_FCOE
7732          *   - The number of FCOE qps.
7733 #endif
7734          * Once we count this up, try the request.
7735          *
7736          * If we can't get what we want, we'll simplify to nearly nothing
7737          * and try again.  If that still fails, we punt.
7738          */
7739         vectors_left = hw->func_caps.num_msix_vectors;
7740         v_budget = 0;
7741
7742         /* reserve one vector for miscellaneous handler */
7743         if (vectors_left) {
7744                 v_budget++;
7745                 vectors_left--;
7746         }
7747
7748         /* reserve vectors for the main PF traffic queues */
7749         pf->num_lan_msix = min_t(int, num_online_cpus(), vectors_left);
7750         vectors_left -= pf->num_lan_msix;
7751         v_budget += pf->num_lan_msix;
7752
7753         /* reserve one vector for sideband flow director */
7754         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
7755                 if (vectors_left) {
7756                         pf->num_fdsb_msix = 1;
7757                         v_budget++;
7758                         vectors_left--;
7759                 } else {
7760                         pf->num_fdsb_msix = 0;
7761                 }
7762         }
7763
7764 #ifdef I40E_FCOE
7765         /* can we reserve enough for FCoE? */
7766         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
7767                 if (!vectors_left)
7768                         pf->num_fcoe_msix = 0;
7769                 else if (vectors_left >= pf->num_fcoe_qps)
7770                         pf->num_fcoe_msix = pf->num_fcoe_qps;
7771                 else
7772                         pf->num_fcoe_msix = 1;
7773                 v_budget += pf->num_fcoe_msix;
7774                 vectors_left -= pf->num_fcoe_msix;
7775         }
7776
7777 #endif
7778         /* can we reserve enough for iWARP? */
7779         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
7780                 iwarp_requested = pf->num_iwarp_msix;
7781
7782                 if (!vectors_left)
7783                         pf->num_iwarp_msix = 0;
7784                 else if (vectors_left < pf->num_iwarp_msix)
7785                         pf->num_iwarp_msix = 1;
7786                 v_budget += pf->num_iwarp_msix;
7787                 vectors_left -= pf->num_iwarp_msix;
7788         }
7789
7790         /* any vectors left over go for VMDq support */
7791         if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
7792                 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps;
7793                 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted);
7794
7795                 if (!vectors_left) {
7796                         pf->num_vmdq_msix = 0;
7797                         pf->num_vmdq_qps = 0;
7798                 } else {
7799                         /* if we're short on vectors for what's desired, we limit
7800                          * the queues per vmdq.  If this is still more than are
7801                          * available, the user will need to change the number of
7802                          * queues/vectors used by the PF later with the ethtool
7803                          * channels command
7804                          */
7805                         if (vmdq_vecs < vmdq_vecs_wanted)
7806                                 pf->num_vmdq_qps = 1;
7807                         pf->num_vmdq_msix = pf->num_vmdq_qps;
7808
7809                         v_budget += vmdq_vecs;
7810                         vectors_left -= vmdq_vecs;
7811                 }
7812         }
7813
7814         pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
7815                                    GFP_KERNEL);
7816         if (!pf->msix_entries)
7817                 return -ENOMEM;
7818
7819         for (i = 0; i < v_budget; i++)
7820                 pf->msix_entries[i].entry = i;
7821         v_actual = i40e_reserve_msix_vectors(pf, v_budget);
7822
7823         if (v_actual < I40E_MIN_MSIX) {
7824                 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
7825                 kfree(pf->msix_entries);
7826                 pf->msix_entries = NULL;
7827                 pci_disable_msix(pf->pdev);
7828                 return -ENODEV;
7829
7830         } else if (v_actual == I40E_MIN_MSIX) {
7831                 /* Adjust for minimal MSIX use */
7832                 pf->num_vmdq_vsis = 0;
7833                 pf->num_vmdq_qps = 0;
7834                 pf->num_lan_qps = 1;
7835                 pf->num_lan_msix = 1;
7836
7837         } else if (!vectors_left) {
7838                 /* If we have limited resources, we will start with no vectors
7839                  * for the special features and then allocate vectors to some
7840                  * of these features based on the policy and at the end disable
7841                  * the features that did not get any vectors.
7842                  */
7843                 int vec;
7844
7845                 dev_info(&pf->pdev->dev,
7846                          "MSI-X vector limit reached, attempting to redistribute vectors\n");
7847                 /* reserve the misc vector */
7848                 vec = v_actual - 1;
7849
7850                 /* Scale vector usage down */
7851                 pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
7852                 pf->num_vmdq_vsis = 1;
7853                 pf->num_vmdq_qps = 1;
7854 #ifdef I40E_FCOE
7855                 pf->num_fcoe_qps = 0;
7856                 pf->num_fcoe_msix = 0;
7857 #endif
7858
7859                 /* partition out the remaining vectors */
7860                 switch (vec) {
7861                 case 2:
7862                         pf->num_lan_msix = 1;
7863                         break;
7864                 case 3:
7865                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
7866                                 pf->num_lan_msix = 1;
7867                                 pf->num_iwarp_msix = 1;
7868                         } else {
7869                                 pf->num_lan_msix = 2;
7870                         }
7871 #ifdef I40E_FCOE
7872                         /* give one vector to FCoE */
7873                         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
7874                                 pf->num_lan_msix = 1;
7875                                 pf->num_fcoe_msix = 1;
7876                         }
7877 #endif
7878                         break;
7879                 default:
7880                         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
7881                                 pf->num_iwarp_msix = min_t(int, (vec / 3),
7882                                                  iwarp_requested);
7883                                 pf->num_vmdq_vsis = min_t(int, (vec / 3),
7884                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
7885                         } else {
7886                                 pf->num_vmdq_vsis = min_t(int, (vec / 2),
7887                                                   I40E_DEFAULT_NUM_VMDQ_VSI);
7888                         }
7889                         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
7890                                 pf->num_fdsb_msix = 1;
7891                                 vec--;
7892                         }
7893                         pf->num_lan_msix = min_t(int,
7894                                (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
7895                                                               pf->num_lan_msix);
7896                         pf->num_lan_qps = pf->num_lan_msix;
7897 #ifdef I40E_FCOE
7898                         /* give one vector to FCoE */
7899                         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
7900                                 pf->num_fcoe_msix = 1;
7901                                 vec--;
7902                         }
7903 #endif
7904                         break;
7905                 }
7906         }
7907
7908         if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
7909             (pf->num_fdsb_msix == 0)) {
7910                 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
7911                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
7912         }
7913         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
7914             (pf->num_vmdq_msix == 0)) {
7915                 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
7916                 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
7917         }
7918
7919         if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
7920             (pf->num_iwarp_msix == 0)) {
7921                 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
7922                 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
7923         }
7924 #ifdef I40E_FCOE
7925
7926         if ((pf->flags & I40E_FLAG_FCOE_ENABLED) && (pf->num_fcoe_msix == 0)) {
7927                 dev_info(&pf->pdev->dev, "FCOE disabled, not enough MSI-X vectors\n");
7928                 pf->flags &= ~I40E_FLAG_FCOE_ENABLED;
7929         }
7930 #endif
7931         i40e_debug(&pf->hw, I40E_DEBUG_INIT,
7932                    "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
7933                    pf->num_lan_msix,
7934                    pf->num_vmdq_msix * pf->num_vmdq_vsis,
7935                    pf->num_fdsb_msix,
7936                    pf->num_iwarp_msix);
7937
7938         return v_actual;
7939 }
7940
7941 /**
7942  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
7943  * @vsi: the VSI being configured
7944  * @v_idx: index of the vector in the vsi struct
7945  * @cpu: cpu to be used on affinity_mask
7946  *
7947  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
7948  **/
7949 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
7950 {
7951         struct i40e_q_vector *q_vector;
7952
7953         /* allocate q_vector */
7954         q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
7955         if (!q_vector)
7956                 return -ENOMEM;
7957
7958         q_vector->vsi = vsi;
7959         q_vector->v_idx = v_idx;
7960         cpumask_set_cpu(cpu, &q_vector->affinity_mask);
7961
7962         if (vsi->netdev)
7963                 netif_napi_add(vsi->netdev, &q_vector->napi,
7964                                i40e_napi_poll, NAPI_POLL_WEIGHT);
7965
7966         q_vector->rx.latency_range = I40E_LOW_LATENCY;
7967         q_vector->tx.latency_range = I40E_LOW_LATENCY;
7968
7969         /* tie q_vector and vsi together */
7970         vsi->q_vectors[v_idx] = q_vector;
7971
7972         return 0;
7973 }
7974
7975 /**
7976  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
7977  * @vsi: the VSI being configured
7978  *
7979  * We allocate one q_vector per queue interrupt.  If allocation fails we
7980  * return -ENOMEM.
7981  **/
7982 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
7983 {
7984         struct i40e_pf *pf = vsi->back;
7985         int err, v_idx, num_q_vectors, current_cpu;
7986
7987         /* if not MSIX, give the one vector only to the LAN VSI */
7988         if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7989                 num_q_vectors = vsi->num_q_vectors;
7990         else if (vsi == pf->vsi[pf->lan_vsi])
7991                 num_q_vectors = 1;
7992         else
7993                 return -EINVAL;
7994
7995         current_cpu = cpumask_first(cpu_online_mask);
7996
7997         for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
7998                 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
7999                 if (err)
8000                         goto err_out;
8001                 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
8002                 if (unlikely(current_cpu >= nr_cpu_ids))
8003                         current_cpu = cpumask_first(cpu_online_mask);
8004         }
8005
8006         return 0;
8007
8008 err_out:
8009         while (v_idx--)
8010                 i40e_free_q_vector(vsi, v_idx);
8011
8012         return err;
8013 }
8014
8015 /**
8016  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
8017  * @pf: board private structure to initialize
8018  **/
8019 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
8020 {
8021         int vectors = 0;
8022         ssize_t size;
8023
8024         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
8025                 vectors = i40e_init_msix(pf);
8026                 if (vectors < 0) {
8027                         pf->flags &= ~(I40E_FLAG_MSIX_ENABLED   |
8028                                        I40E_FLAG_IWARP_ENABLED  |
8029 #ifdef I40E_FCOE
8030                                        I40E_FLAG_FCOE_ENABLED   |
8031 #endif
8032                                        I40E_FLAG_RSS_ENABLED    |
8033                                        I40E_FLAG_DCB_CAPABLE    |
8034                                        I40E_FLAG_DCB_ENABLED    |
8035                                        I40E_FLAG_SRIOV_ENABLED  |
8036                                        I40E_FLAG_FD_SB_ENABLED  |
8037                                        I40E_FLAG_FD_ATR_ENABLED |
8038                                        I40E_FLAG_VMDQ_ENABLED);
8039
8040                         /* rework the queue expectations without MSIX */
8041                         i40e_determine_queue_usage(pf);
8042                 }
8043         }
8044
8045         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
8046             (pf->flags & I40E_FLAG_MSI_ENABLED)) {
8047                 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
8048                 vectors = pci_enable_msi(pf->pdev);
8049                 if (vectors < 0) {
8050                         dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
8051                                  vectors);
8052                         pf->flags &= ~I40E_FLAG_MSI_ENABLED;
8053                 }
8054                 vectors = 1;  /* one MSI or Legacy vector */
8055         }
8056
8057         if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
8058                 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
8059
8060         /* set up vector assignment tracking */
8061         size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
8062         pf->irq_pile = kzalloc(size, GFP_KERNEL);
8063         if (!pf->irq_pile) {
8064                 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n");
8065                 return -ENOMEM;
8066         }
8067         pf->irq_pile->num_entries = vectors;
8068         pf->irq_pile->search_hint = 0;
8069
8070         /* track first vector for misc interrupts, ignore return */
8071         (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
8072
8073         return 0;
8074 }
8075
8076 /**
8077  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
8078  * @pf: board private structure
8079  *
8080  * This sets up the handler for MSIX 0, which is used to manage the
8081  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
8082  * when in MSI or Legacy interrupt mode.
8083  **/
8084 static int i40e_setup_misc_vector(struct i40e_pf *pf)
8085 {
8086         struct i40e_hw *hw = &pf->hw;
8087         int err = 0;
8088
8089         /* Only request the irq if this is the first time through, and
8090          * not when we're rebuilding after a Reset
8091          */
8092         if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) {
8093                 err = request_irq(pf->msix_entries[0].vector,
8094                                   i40e_intr, 0, pf->int_name, pf);
8095                 if (err) {
8096                         dev_info(&pf->pdev->dev,
8097                                  "request_irq for %s failed: %d\n",
8098                                  pf->int_name, err);
8099                         return -EFAULT;
8100                 }
8101         }
8102
8103         i40e_enable_misc_int_causes(pf);
8104
8105         /* associate no queues to the misc vector */
8106         wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
8107         wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
8108
8109         i40e_flush(hw);
8110
8111         i40e_irq_dynamic_enable_icr0(pf, true);
8112
8113         return err;
8114 }
8115
8116 /**
8117  * i40e_config_rss_aq - Prepare for RSS using AQ commands
8118  * @vsi: vsi structure
8119  * @seed: RSS hash seed
8120  **/
8121 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8122                               u8 *lut, u16 lut_size)
8123 {
8124         struct i40e_pf *pf = vsi->back;
8125         struct i40e_hw *hw = &pf->hw;
8126         int ret = 0;
8127
8128         if (seed) {
8129                 struct i40e_aqc_get_set_rss_key_data *seed_dw =
8130                         (struct i40e_aqc_get_set_rss_key_data *)seed;
8131                 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
8132                 if (ret) {
8133                         dev_info(&pf->pdev->dev,
8134                                  "Cannot set RSS key, err %s aq_err %s\n",
8135                                  i40e_stat_str(hw, ret),
8136                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8137                         return ret;
8138                 }
8139         }
8140         if (lut) {
8141                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8142
8143                 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8144                 if (ret) {
8145                         dev_info(&pf->pdev->dev,
8146                                  "Cannot set RSS lut, err %s aq_err %s\n",
8147                                  i40e_stat_str(hw, ret),
8148                                  i40e_aq_str(hw, hw->aq.asq_last_status));
8149                         return ret;
8150                 }
8151         }
8152         return ret;
8153 }
8154
8155 /**
8156  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
8157  * @vsi: Pointer to vsi structure
8158  * @seed: Buffter to store the hash keys
8159  * @lut: Buffer to store the lookup table entries
8160  * @lut_size: Size of buffer to store the lookup table entries
8161  *
8162  * Return 0 on success, negative on failure
8163  */
8164 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
8165                            u8 *lut, u16 lut_size)
8166 {
8167         struct i40e_pf *pf = vsi->back;
8168         struct i40e_hw *hw = &pf->hw;
8169         int ret = 0;
8170
8171         if (seed) {
8172                 ret = i40e_aq_get_rss_key(hw, vsi->id,
8173                         (struct i40e_aqc_get_set_rss_key_data *)seed);
8174                 if (ret) {
8175                         dev_info(&pf->pdev->dev,
8176                                  "Cannot get RSS key, err %s aq_err %s\n",
8177                                  i40e_stat_str(&pf->hw, ret),
8178                                  i40e_aq_str(&pf->hw,
8179                                              pf->hw.aq.asq_last_status));
8180                         return ret;
8181                 }
8182         }
8183
8184         if (lut) {
8185                 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
8186
8187                 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
8188                 if (ret) {
8189                         dev_info(&pf->pdev->dev,
8190                                  "Cannot get RSS lut, err %s aq_err %s\n",
8191                                  i40e_stat_str(&pf->hw, ret),
8192                                  i40e_aq_str(&pf->hw,
8193                                              pf->hw.aq.asq_last_status));
8194                         return ret;
8195                 }
8196         }
8197
8198         return ret;
8199 }
8200
8201 /**
8202  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
8203  * @vsi: VSI structure
8204  **/
8205 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
8206 {
8207         u8 seed[I40E_HKEY_ARRAY_SIZE];
8208         struct i40e_pf *pf = vsi->back;
8209         u8 *lut;
8210         int ret;
8211
8212         if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE))
8213                 return 0;
8214
8215         if (!vsi->rss_size)
8216                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8217                                       vsi->num_queue_pairs);
8218         if (!vsi->rss_size)
8219                 return -EINVAL;
8220
8221         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8222         if (!lut)
8223                 return -ENOMEM;
8224         /* Use the user configured hash keys and lookup table if there is one,
8225          * otherwise use default
8226          */
8227         if (vsi->rss_lut_user)
8228                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8229         else
8230                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8231         if (vsi->rss_hkey_user)
8232                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8233         else
8234                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8235         ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
8236         kfree(lut);
8237
8238         return ret;
8239 }
8240
8241 /**
8242  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
8243  * @vsi: Pointer to vsi structure
8244  * @seed: RSS hash seed
8245  * @lut: Lookup table
8246  * @lut_size: Lookup table size
8247  *
8248  * Returns 0 on success, negative on failure
8249  **/
8250 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
8251                                const u8 *lut, u16 lut_size)
8252 {
8253         struct i40e_pf *pf = vsi->back;
8254         struct i40e_hw *hw = &pf->hw;
8255         u16 vf_id = vsi->vf_id;
8256         u8 i;
8257
8258         /* Fill out hash function seed */
8259         if (seed) {
8260                 u32 *seed_dw = (u32 *)seed;
8261
8262                 if (vsi->type == I40E_VSI_MAIN) {
8263                         for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8264                                 i40e_write_rx_ctl(hw, I40E_PFQF_HKEY(i),
8265                                                   seed_dw[i]);
8266                 } else if (vsi->type == I40E_VSI_SRIOV) {
8267                         for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
8268                                 i40e_write_rx_ctl(hw,
8269                                                   I40E_VFQF_HKEY1(i, vf_id),
8270                                                   seed_dw[i]);
8271                 } else {
8272                         dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
8273                 }
8274         }
8275
8276         if (lut) {
8277                 u32 *lut_dw = (u32 *)lut;
8278
8279                 if (vsi->type == I40E_VSI_MAIN) {
8280                         if (lut_size != I40E_HLUT_ARRAY_SIZE)
8281                                 return -EINVAL;
8282                         for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8283                                 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
8284                 } else if (vsi->type == I40E_VSI_SRIOV) {
8285                         if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
8286                                 return -EINVAL;
8287                         for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
8288                                 i40e_write_rx_ctl(hw,
8289                                                   I40E_VFQF_HLUT1(i, vf_id),
8290                                                   lut_dw[i]);
8291                 } else {
8292                         dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
8293                 }
8294         }
8295         i40e_flush(hw);
8296
8297         return 0;
8298 }
8299
8300 /**
8301  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
8302  * @vsi: Pointer to VSI structure
8303  * @seed: Buffer to store the keys
8304  * @lut: Buffer to store the lookup table entries
8305  * @lut_size: Size of buffer to store the lookup table entries
8306  *
8307  * Returns 0 on success, negative on failure
8308  */
8309 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
8310                             u8 *lut, u16 lut_size)
8311 {
8312         struct i40e_pf *pf = vsi->back;
8313         struct i40e_hw *hw = &pf->hw;
8314         u16 i;
8315
8316         if (seed) {
8317                 u32 *seed_dw = (u32 *)seed;
8318
8319                 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
8320                         seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
8321         }
8322         if (lut) {
8323                 u32 *lut_dw = (u32 *)lut;
8324
8325                 if (lut_size != I40E_HLUT_ARRAY_SIZE)
8326                         return -EINVAL;
8327                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8328                         lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
8329         }
8330
8331         return 0;
8332 }
8333
8334 /**
8335  * i40e_config_rss - Configure RSS keys and lut
8336  * @vsi: Pointer to VSI structure
8337  * @seed: RSS hash seed
8338  * @lut: Lookup table
8339  * @lut_size: Lookup table size
8340  *
8341  * Returns 0 on success, negative on failure
8342  */
8343 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8344 {
8345         struct i40e_pf *pf = vsi->back;
8346
8347         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
8348                 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
8349         else
8350                 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
8351 }
8352
8353 /**
8354  * i40e_get_rss - Get RSS keys and lut
8355  * @vsi: Pointer to VSI structure
8356  * @seed: Buffer to store the keys
8357  * @lut: Buffer to store the lookup table entries
8358  * lut_size: Size of buffer to store the lookup table entries
8359  *
8360  * Returns 0 on success, negative on failure
8361  */
8362 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
8363 {
8364         struct i40e_pf *pf = vsi->back;
8365
8366         if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)
8367                 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
8368         else
8369                 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
8370 }
8371
8372 /**
8373  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
8374  * @pf: Pointer to board private structure
8375  * @lut: Lookup table
8376  * @rss_table_size: Lookup table size
8377  * @rss_size: Range of queue number for hashing
8378  */
8379 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
8380                        u16 rss_table_size, u16 rss_size)
8381 {
8382         u16 i;
8383
8384         for (i = 0; i < rss_table_size; i++)
8385                 lut[i] = i % rss_size;
8386 }
8387
8388 /**
8389  * i40e_pf_config_rss - Prepare for RSS if used
8390  * @pf: board private structure
8391  **/
8392 static int i40e_pf_config_rss(struct i40e_pf *pf)
8393 {
8394         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8395         u8 seed[I40E_HKEY_ARRAY_SIZE];
8396         u8 *lut;
8397         struct i40e_hw *hw = &pf->hw;
8398         u32 reg_val;
8399         u64 hena;
8400         int ret;
8401
8402         /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
8403         hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
8404                 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
8405         hena |= i40e_pf_get_default_rss_hena(pf);
8406
8407         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
8408         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
8409
8410         /* Determine the RSS table size based on the hardware capabilities */
8411         reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
8412         reg_val = (pf->rss_table_size == 512) ?
8413                         (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
8414                         (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
8415         i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
8416
8417         /* Determine the RSS size of the VSI */
8418         if (!vsi->rss_size)
8419                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8420                                       vsi->num_queue_pairs);
8421         if (!vsi->rss_size)
8422                 return -EINVAL;
8423
8424         lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
8425         if (!lut)
8426                 return -ENOMEM;
8427
8428         /* Use user configured lut if there is one, otherwise use default */
8429         if (vsi->rss_lut_user)
8430                 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
8431         else
8432                 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
8433
8434         /* Use user configured hash key if there is one, otherwise
8435          * use default.
8436          */
8437         if (vsi->rss_hkey_user)
8438                 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
8439         else
8440                 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
8441         ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
8442         kfree(lut);
8443
8444         return ret;
8445 }
8446
8447 /**
8448  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
8449  * @pf: board private structure
8450  * @queue_count: the requested queue count for rss.
8451  *
8452  * returns 0 if rss is not enabled, if enabled returns the final rss queue
8453  * count which may be different from the requested queue count.
8454  **/
8455 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
8456 {
8457         struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
8458         int new_rss_size;
8459
8460         if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
8461                 return 0;
8462
8463         new_rss_size = min_t(int, queue_count, pf->rss_size_max);
8464
8465         if (queue_count != vsi->num_queue_pairs) {
8466                 vsi->req_queue_pairs = queue_count;
8467                 i40e_prep_for_reset(pf);
8468
8469                 pf->alloc_rss_size = new_rss_size;
8470
8471                 i40e_reset_and_rebuild(pf, true);
8472
8473                 /* Discard the user configured hash keys and lut, if less
8474                  * queues are enabled.
8475                  */
8476                 if (queue_count < vsi->rss_size) {
8477                         i40e_clear_rss_config_user(vsi);
8478                         dev_dbg(&pf->pdev->dev,
8479                                 "discard user configured hash keys and lut\n");
8480                 }
8481
8482                 /* Reset vsi->rss_size, as number of enabled queues changed */
8483                 vsi->rss_size = min_t(int, pf->alloc_rss_size,
8484                                       vsi->num_queue_pairs);
8485
8486                 i40e_pf_config_rss(pf);
8487         }
8488         dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
8489                  vsi->req_queue_pairs, pf->rss_size_max);
8490         return pf->alloc_rss_size;
8491 }
8492
8493 /**
8494  * i40e_get_npar_bw_setting - Retrieve BW settings for this PF partition
8495  * @pf: board private structure
8496  **/
8497 i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf)
8498 {
8499         i40e_status status;
8500         bool min_valid, max_valid;
8501         u32 max_bw, min_bw;
8502
8503         status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
8504                                            &min_valid, &max_valid);
8505
8506         if (!status) {
8507                 if (min_valid)
8508                         pf->npar_min_bw = min_bw;
8509                 if (max_valid)
8510                         pf->npar_max_bw = max_bw;
8511         }
8512
8513         return status;
8514 }
8515
8516 /**
8517  * i40e_set_npar_bw_setting - Set BW settings for this PF partition
8518  * @pf: board private structure
8519  **/
8520 i40e_status i40e_set_npar_bw_setting(struct i40e_pf *pf)
8521 {
8522         struct i40e_aqc_configure_partition_bw_data bw_data;
8523         i40e_status status;
8524
8525         /* Set the valid bit for this PF */
8526         bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
8527         bw_data.max_bw[pf->hw.pf_id] = pf->npar_max_bw & I40E_ALT_BW_VALUE_MASK;
8528         bw_data.min_bw[pf->hw.pf_id] = pf->npar_min_bw & I40E_ALT_BW_VALUE_MASK;
8529
8530         /* Set the new bandwidths */
8531         status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
8532
8533         return status;
8534 }
8535
8536 /**
8537  * i40e_commit_npar_bw_setting - Commit BW settings for this PF partition
8538  * @pf: board private structure
8539  **/
8540 i40e_status i40e_commit_npar_bw_setting(struct i40e_pf *pf)
8541 {
8542         /* Commit temporary BW setting to permanent NVM image */
8543         enum i40e_admin_queue_err last_aq_status;
8544         i40e_status ret;
8545         u16 nvm_word;
8546
8547         if (pf->hw.partition_id != 1) {
8548                 dev_info(&pf->pdev->dev,
8549                          "Commit BW only works on partition 1! This is partition %d",
8550                          pf->hw.partition_id);
8551                 ret = I40E_NOT_SUPPORTED;
8552                 goto bw_commit_out;
8553         }
8554
8555         /* Acquire NVM for read access */
8556         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
8557         last_aq_status = pf->hw.aq.asq_last_status;
8558         if (ret) {
8559                 dev_info(&pf->pdev->dev,
8560                          "Cannot acquire NVM for read access, err %s aq_err %s\n",
8561                          i40e_stat_str(&pf->hw, ret),
8562                          i40e_aq_str(&pf->hw, last_aq_status));
8563                 goto bw_commit_out;
8564         }
8565
8566         /* Read word 0x10 of NVM - SW compatibility word 1 */
8567         ret = i40e_aq_read_nvm(&pf->hw,
8568                                I40E_SR_NVM_CONTROL_WORD,
8569                                0x10, sizeof(nvm_word), &nvm_word,
8570                                false, NULL);
8571         /* Save off last admin queue command status before releasing
8572          * the NVM
8573          */
8574         last_aq_status = pf->hw.aq.asq_last_status;
8575         i40e_release_nvm(&pf->hw);
8576         if (ret) {
8577                 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
8578                          i40e_stat_str(&pf->hw, ret),
8579                          i40e_aq_str(&pf->hw, last_aq_status));
8580                 goto bw_commit_out;
8581         }
8582
8583         /* Wait a bit for NVM release to complete */
8584         msleep(50);
8585
8586         /* Acquire NVM for write access */
8587         ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
8588         last_aq_status = pf->hw.aq.asq_last_status;
8589         if (ret) {
8590                 dev_info(&pf->pdev->dev,
8591                          "Cannot acquire NVM for write access, err %s aq_err %s\n",
8592                          i40e_stat_str(&pf->hw, ret),
8593                          i40e_aq_str(&pf->hw, last_aq_status));
8594                 goto bw_commit_out;
8595         }
8596         /* Write it back out unchanged to initiate update NVM,
8597          * which will force a write of the shadow (alt) RAM to
8598          * the NVM - thus storing the bandwidth values permanently.
8599          */
8600         ret = i40e_aq_update_nvm(&pf->hw,
8601                                  I40E_SR_NVM_CONTROL_WORD,
8602                                  0x10, sizeof(nvm_word),
8603                                  &nvm_word, true, NULL);
8604         /* Save off last admin queue command status before releasing
8605          * the NVM
8606          */
8607         last_aq_status = pf->hw.aq.asq_last_status;
8608         i40e_release_nvm(&pf->hw);
8609         if (ret)
8610                 dev_info(&pf->pdev->dev,
8611                          "BW settings NOT SAVED, err %s aq_err %s\n",
8612                          i40e_stat_str(&pf->hw, ret),
8613                          i40e_aq_str(&pf->hw, last_aq_status));
8614 bw_commit_out:
8615
8616         return ret;
8617 }
8618
8619 /**
8620  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
8621  * @pf: board private structure to initialize
8622  *
8623  * i40e_sw_init initializes the Adapter private data structure.
8624  * Fields are initialized based on PCI device information and
8625  * OS network device settings (MTU size).
8626  **/
8627 static int i40e_sw_init(struct i40e_pf *pf)
8628 {
8629         int err = 0;
8630         int size;
8631
8632         /* Set default capability flags */
8633         pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
8634                     I40E_FLAG_MSI_ENABLED     |
8635                     I40E_FLAG_MSIX_ENABLED;
8636
8637         /* Set default ITR */
8638         pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF;
8639         pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF;
8640
8641         /* Depending on PF configurations, it is possible that the RSS
8642          * maximum might end up larger than the available queues
8643          */
8644         pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
8645         pf->alloc_rss_size = 1;
8646         pf->rss_table_size = pf->hw.func_caps.rss_table_size;
8647         pf->rss_size_max = min_t(int, pf->rss_size_max,
8648                                  pf->hw.func_caps.num_tx_qp);
8649         if (pf->hw.func_caps.rss) {
8650                 pf->flags |= I40E_FLAG_RSS_ENABLED;
8651                 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
8652                                            num_online_cpus());
8653         }
8654
8655         /* MFP mode enabled */
8656         if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
8657                 pf->flags |= I40E_FLAG_MFP_ENABLED;
8658                 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
8659                 if (i40e_get_npar_bw_setting(pf))
8660                         dev_warn(&pf->pdev->dev,
8661                                  "Could not get NPAR bw settings\n");
8662                 else
8663                         dev_info(&pf->pdev->dev,
8664                                  "Min BW = %8.8x, Max BW = %8.8x\n",
8665                                  pf->npar_min_bw, pf->npar_max_bw);
8666         }
8667
8668         /* FW/NVM is not yet fixed in this regard */
8669         if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
8670             (pf->hw.func_caps.fd_filters_best_effort > 0)) {
8671                 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
8672                 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
8673                 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
8674                     pf->hw.num_partitions > 1)
8675                         dev_info(&pf->pdev->dev,
8676                                  "Flow Director Sideband mode Disabled in MFP mode\n");
8677                 else
8678                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8679                 pf->fdir_pf_filter_count =
8680                                  pf->hw.func_caps.fd_filters_guaranteed;
8681                 pf->hw.fdir_shared_filter_count =
8682                                  pf->hw.func_caps.fd_filters_best_effort;
8683         }
8684
8685         if (i40e_is_mac_710(&pf->hw) &&
8686             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
8687             (pf->hw.aq.fw_maj_ver < 4))) {
8688                 pf->flags |= I40E_FLAG_RESTART_AUTONEG;
8689                 /* No DCB support  for FW < v4.33 */
8690                 pf->flags |= I40E_FLAG_NO_DCB_SUPPORT;
8691         }
8692
8693         /* Disable FW LLDP if FW < v4.3 */
8694         if (i40e_is_mac_710(&pf->hw) &&
8695             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
8696             (pf->hw.aq.fw_maj_ver < 4)))
8697                 pf->flags |= I40E_FLAG_STOP_FW_LLDP;
8698
8699         /* Use the FW Set LLDP MIB API if FW > v4.40 */
8700         if (i40e_is_mac_710(&pf->hw) &&
8701             (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
8702             (pf->hw.aq.fw_maj_ver >= 5)))
8703                 pf->flags |= I40E_FLAG_USE_SET_LLDP_MIB;
8704
8705         if (pf->hw.func_caps.vmdq) {
8706                 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
8707                 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
8708                 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
8709         }
8710
8711         if (pf->hw.func_caps.iwarp) {
8712                 pf->flags |= I40E_FLAG_IWARP_ENABLED;
8713                 /* IWARP needs one extra vector for CQP just like MISC.*/
8714                 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
8715         }
8716
8717 #ifdef I40E_FCOE
8718         i40e_init_pf_fcoe(pf);
8719
8720 #endif /* I40E_FCOE */
8721 #ifdef CONFIG_PCI_IOV
8722         if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
8723                 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
8724                 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
8725                 pf->num_req_vfs = min_t(int,
8726                                         pf->hw.func_caps.num_vfs,
8727                                         I40E_MAX_VF_COUNT);
8728         }
8729 #endif /* CONFIG_PCI_IOV */
8730         if (pf->hw.mac.type == I40E_MAC_X722) {
8731                 pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE |
8732                              I40E_FLAG_128_QP_RSS_CAPABLE |
8733                              I40E_FLAG_HW_ATR_EVICT_CAPABLE |
8734                              I40E_FLAG_OUTER_UDP_CSUM_CAPABLE |
8735                              I40E_FLAG_WB_ON_ITR_CAPABLE |
8736                              I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE |
8737                              I40E_FLAG_NO_PCI_LINK_CHECK |
8738                              I40E_FLAG_USE_SET_LLDP_MIB |
8739                              I40E_FLAG_GENEVE_OFFLOAD_CAPABLE |
8740                              I40E_FLAG_PTP_L4_CAPABLE;
8741         } else if ((pf->hw.aq.api_maj_ver > 1) ||
8742                    ((pf->hw.aq.api_maj_ver == 1) &&
8743                     (pf->hw.aq.api_min_ver > 4))) {
8744                 /* Supported in FW API version higher than 1.4 */
8745                 pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE;
8746                 pf->auto_disable_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE;
8747         } else {
8748                 pf->auto_disable_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE;
8749         }
8750
8751         pf->eeprom_version = 0xDEAD;
8752         pf->lan_veb = I40E_NO_VEB;
8753         pf->lan_vsi = I40E_NO_VSI;
8754
8755         /* By default FW has this off for performance reasons */
8756         pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
8757
8758         /* set up queue assignment tracking */
8759         size = sizeof(struct i40e_lump_tracking)
8760                 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
8761         pf->qp_pile = kzalloc(size, GFP_KERNEL);
8762         if (!pf->qp_pile) {
8763                 err = -ENOMEM;
8764                 goto sw_init_done;
8765         }
8766         pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
8767         pf->qp_pile->search_hint = 0;
8768
8769         pf->tx_timeout_recovery_level = 1;
8770
8771         mutex_init(&pf->switch_mutex);
8772
8773         /* If NPAR is enabled nudge the Tx scheduler */
8774         if (pf->hw.func_caps.npar_enable && (!i40e_get_npar_bw_setting(pf)))
8775                 i40e_set_npar_bw_setting(pf);
8776
8777 sw_init_done:
8778         return err;
8779 }
8780
8781 /**
8782  * i40e_set_ntuple - set the ntuple feature flag and take action
8783  * @pf: board private structure to initialize
8784  * @features: the feature set that the stack is suggesting
8785  *
8786  * returns a bool to indicate if reset needs to happen
8787  **/
8788 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
8789 {
8790         bool need_reset = false;
8791
8792         /* Check if Flow Director n-tuple support was enabled or disabled.  If
8793          * the state changed, we need to reset.
8794          */
8795         if (features & NETIF_F_NTUPLE) {
8796                 /* Enable filters and mark for reset */
8797                 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
8798                         need_reset = true;
8799                 /* enable FD_SB only if there is MSI-X vector */
8800                 if (pf->num_fdsb_msix > 0)
8801                         pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8802         } else {
8803                 /* turn off filters, mark for reset and clear SW filter list */
8804                 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
8805                         need_reset = true;
8806                         i40e_fdir_filter_exit(pf);
8807                 }
8808                 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8809                 pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED;
8810                 /* reset fd counters */
8811                 pf->fd_add_err = pf->fd_atr_cnt = pf->fd_tcp_rule = 0;
8812                 pf->fdir_pf_active_filters = 0;
8813                 /* if ATR was auto disabled it can be re-enabled. */
8814                 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
8815                     (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) {
8816                         pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED;
8817                         if (I40E_DEBUG_FD & pf->hw.debug_mask)
8818                                 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
8819                 }
8820         }
8821         return need_reset;
8822 }
8823
8824 /**
8825  * i40e_clear_rss_lut - clear the rx hash lookup table
8826  * @vsi: the VSI being configured
8827  **/
8828 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
8829 {
8830         struct i40e_pf *pf = vsi->back;
8831         struct i40e_hw *hw = &pf->hw;
8832         u16 vf_id = vsi->vf_id;
8833         u8 i;
8834
8835         if (vsi->type == I40E_VSI_MAIN) {
8836                 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
8837                         wr32(hw, I40E_PFQF_HLUT(i), 0);
8838         } else if (vsi->type == I40E_VSI_SRIOV) {
8839                 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
8840                         i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
8841         } else {
8842                 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
8843         }
8844 }
8845
8846 /**
8847  * i40e_set_features - set the netdev feature flags
8848  * @netdev: ptr to the netdev being adjusted
8849  * @features: the feature set that the stack is suggesting
8850  **/
8851 static int i40e_set_features(struct net_device *netdev,
8852                              netdev_features_t features)
8853 {
8854         struct i40e_netdev_priv *np = netdev_priv(netdev);
8855         struct i40e_vsi *vsi = np->vsi;
8856         struct i40e_pf *pf = vsi->back;
8857         bool need_reset;
8858
8859         if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
8860                 i40e_pf_config_rss(pf);
8861         else if (!(features & NETIF_F_RXHASH) &&
8862                  netdev->features & NETIF_F_RXHASH)
8863                 i40e_clear_rss_lut(vsi);
8864
8865         if (features & NETIF_F_HW_VLAN_CTAG_RX)
8866                 i40e_vlan_stripping_enable(vsi);
8867         else
8868                 i40e_vlan_stripping_disable(vsi);
8869
8870         need_reset = i40e_set_ntuple(pf, features);
8871
8872         if (need_reset)
8873                 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED));
8874
8875         return 0;
8876 }
8877
8878 /**
8879  * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port
8880  * @pf: board private structure
8881  * @port: The UDP port to look up
8882  *
8883  * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found
8884  **/
8885 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, __be16 port)
8886 {
8887         u8 i;
8888
8889         for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
8890                 if (pf->udp_ports[i].index == port)
8891                         return i;
8892         }
8893
8894         return i;
8895 }
8896
8897 /**
8898  * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up
8899  * @netdev: This physical port's netdev
8900  * @ti: Tunnel endpoint information
8901  **/
8902 static void i40e_udp_tunnel_add(struct net_device *netdev,
8903                                 struct udp_tunnel_info *ti)
8904 {
8905         struct i40e_netdev_priv *np = netdev_priv(netdev);
8906         struct i40e_vsi *vsi = np->vsi;
8907         struct i40e_pf *pf = vsi->back;
8908         __be16 port = ti->port;
8909         u8 next_idx;
8910         u8 idx;
8911
8912         idx = i40e_get_udp_port_idx(pf, port);
8913
8914         /* Check if port already exists */
8915         if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8916                 netdev_info(netdev, "port %d already offloaded\n",
8917                             ntohs(port));
8918                 return;
8919         }
8920
8921         /* Now check if there is space to add the new port */
8922         next_idx = i40e_get_udp_port_idx(pf, 0);
8923
8924         if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
8925                 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
8926                             ntohs(port));
8927                 return;
8928         }
8929
8930         switch (ti->type) {
8931         case UDP_TUNNEL_TYPE_VXLAN:
8932                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
8933                 break;
8934         case UDP_TUNNEL_TYPE_GENEVE:
8935                 if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE))
8936                         return;
8937                 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
8938                 break;
8939         default:
8940                 return;
8941         }
8942
8943         /* New port: add it and mark its index in the bitmap */
8944         pf->udp_ports[next_idx].index = port;
8945         pf->pending_udp_bitmap |= BIT_ULL(next_idx);
8946         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
8947 }
8948
8949 /**
8950  * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away
8951  * @netdev: This physical port's netdev
8952  * @ti: Tunnel endpoint information
8953  **/
8954 static void i40e_udp_tunnel_del(struct net_device *netdev,
8955                                 struct udp_tunnel_info *ti)
8956 {
8957         struct i40e_netdev_priv *np = netdev_priv(netdev);
8958         struct i40e_vsi *vsi = np->vsi;
8959         struct i40e_pf *pf = vsi->back;
8960         __be16 port = ti->port;
8961         u8 idx;
8962
8963         idx = i40e_get_udp_port_idx(pf, port);
8964
8965         /* Check if port already exists */
8966         if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
8967                 goto not_found;
8968
8969         switch (ti->type) {
8970         case UDP_TUNNEL_TYPE_VXLAN:
8971                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
8972                         goto not_found;
8973                 break;
8974         case UDP_TUNNEL_TYPE_GENEVE:
8975                 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
8976                         goto not_found;
8977                 break;
8978         default:
8979                 goto not_found;
8980         }
8981
8982         /* if port exists, set it to 0 (mark for deletion)
8983          * and make it pending
8984          */
8985         pf->udp_ports[idx].index = 0;
8986         pf->pending_udp_bitmap |= BIT_ULL(idx);
8987         pf->flags |= I40E_FLAG_UDP_FILTER_SYNC;
8988
8989         return;
8990 not_found:
8991         netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
8992                     ntohs(port));
8993 }
8994
8995 static int i40e_get_phys_port_id(struct net_device *netdev,
8996                                  struct netdev_phys_item_id *ppid)
8997 {
8998         struct i40e_netdev_priv *np = netdev_priv(netdev);
8999         struct i40e_pf *pf = np->vsi->back;
9000         struct i40e_hw *hw = &pf->hw;
9001
9002         if (!(pf->flags & I40E_FLAG_PORT_ID_VALID))
9003                 return -EOPNOTSUPP;
9004
9005         ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
9006         memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
9007
9008         return 0;
9009 }
9010
9011 /**
9012  * i40e_ndo_fdb_add - add an entry to the hardware database
9013  * @ndm: the input from the stack
9014  * @tb: pointer to array of nladdr (unused)
9015  * @dev: the net device pointer
9016  * @addr: the MAC address entry being added
9017  * @flags: instructions from stack about fdb operation
9018  */
9019 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
9020                             struct net_device *dev,
9021                             const unsigned char *addr, u16 vid,
9022                             u16 flags)
9023 {
9024         struct i40e_netdev_priv *np = netdev_priv(dev);
9025         struct i40e_pf *pf = np->vsi->back;
9026         int err = 0;
9027
9028         if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
9029                 return -EOPNOTSUPP;
9030
9031         if (vid) {
9032                 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
9033                 return -EINVAL;
9034         }
9035
9036         /* Hardware does not support aging addresses so if a
9037          * ndm_state is given only allow permanent addresses
9038          */
9039         if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
9040                 netdev_info(dev, "FDB only supports static addresses\n");
9041                 return -EINVAL;
9042         }
9043
9044         if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
9045                 err = dev_uc_add_excl(dev, addr);
9046         else if (is_multicast_ether_addr(addr))
9047                 err = dev_mc_add_excl(dev, addr);
9048         else
9049                 err = -EINVAL;
9050
9051         /* Only return duplicate errors if NLM_F_EXCL is set */
9052         if (err == -EEXIST && !(flags & NLM_F_EXCL))
9053                 err = 0;
9054
9055         return err;
9056 }
9057
9058 /**
9059  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
9060  * @dev: the netdev being configured
9061  * @nlh: RTNL message
9062  *
9063  * Inserts a new hardware bridge if not already created and
9064  * enables the bridging mode requested (VEB or VEPA). If the
9065  * hardware bridge has already been inserted and the request
9066  * is to change the mode then that requires a PF reset to
9067  * allow rebuild of the components with required hardware
9068  * bridge mode enabled.
9069  **/
9070 static int i40e_ndo_bridge_setlink(struct net_device *dev,
9071                                    struct nlmsghdr *nlh,
9072                                    u16 flags)
9073 {
9074         struct i40e_netdev_priv *np = netdev_priv(dev);
9075         struct i40e_vsi *vsi = np->vsi;
9076         struct i40e_pf *pf = vsi->back;
9077         struct i40e_veb *veb = NULL;
9078         struct nlattr *attr, *br_spec;
9079         int i, rem;
9080
9081         /* Only for PF VSI for now */
9082         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9083                 return -EOPNOTSUPP;
9084
9085         /* Find the HW bridge for PF VSI */
9086         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9087                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9088                         veb = pf->veb[i];
9089         }
9090
9091         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
9092
9093         nla_for_each_nested(attr, br_spec, rem) {
9094                 __u16 mode;
9095
9096                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
9097                         continue;
9098
9099                 mode = nla_get_u16(attr);
9100                 if ((mode != BRIDGE_MODE_VEPA) &&
9101                     (mode != BRIDGE_MODE_VEB))
9102                         return -EINVAL;
9103
9104                 /* Insert a new HW bridge */
9105                 if (!veb) {
9106                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
9107                                              vsi->tc_config.enabled_tc);
9108                         if (veb) {
9109                                 veb->bridge_mode = mode;
9110                                 i40e_config_bridge_mode(veb);
9111                         } else {
9112                                 /* No Bridge HW offload available */
9113                                 return -ENOENT;
9114                         }
9115                         break;
9116                 } else if (mode != veb->bridge_mode) {
9117                         /* Existing HW bridge but different mode needs reset */
9118                         veb->bridge_mode = mode;
9119                         /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
9120                         if (mode == BRIDGE_MODE_VEB)
9121                                 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
9122                         else
9123                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
9124                         i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED));
9125                         break;
9126                 }
9127         }
9128
9129         return 0;
9130 }
9131
9132 /**
9133  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
9134  * @skb: skb buff
9135  * @pid: process id
9136  * @seq: RTNL message seq #
9137  * @dev: the netdev being configured
9138  * @filter_mask: unused
9139  * @nlflags: netlink flags passed in
9140  *
9141  * Return the mode in which the hardware bridge is operating in
9142  * i.e VEB or VEPA.
9143  **/
9144 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
9145                                    struct net_device *dev,
9146                                    u32 __always_unused filter_mask,
9147                                    int nlflags)
9148 {
9149         struct i40e_netdev_priv *np = netdev_priv(dev);
9150         struct i40e_vsi *vsi = np->vsi;
9151         struct i40e_pf *pf = vsi->back;
9152         struct i40e_veb *veb = NULL;
9153         int i;
9154
9155         /* Only for PF VSI for now */
9156         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
9157                 return -EOPNOTSUPP;
9158
9159         /* Find the HW bridge for the PF VSI */
9160         for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9161                 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9162                         veb = pf->veb[i];
9163         }
9164
9165         if (!veb)
9166                 return 0;
9167
9168         return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
9169                                        0, 0, nlflags, filter_mask, NULL);
9170 }
9171
9172 /**
9173  * i40e_features_check - Validate encapsulated packet conforms to limits
9174  * @skb: skb buff
9175  * @dev: This physical port's netdev
9176  * @features: Offload features that the stack believes apply
9177  **/
9178 static netdev_features_t i40e_features_check(struct sk_buff *skb,
9179                                              struct net_device *dev,
9180                                              netdev_features_t features)
9181 {
9182         size_t len;
9183
9184         /* No point in doing any of this if neither checksum nor GSO are
9185          * being requested for this frame.  We can rule out both by just
9186          * checking for CHECKSUM_PARTIAL
9187          */
9188         if (skb->ip_summed != CHECKSUM_PARTIAL)
9189                 return features;
9190
9191         /* We cannot support GSO if the MSS is going to be less than
9192          * 64 bytes.  If it is then we need to drop support for GSO.
9193          */
9194         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
9195                 features &= ~NETIF_F_GSO_MASK;
9196
9197         /* MACLEN can support at most 63 words */
9198         len = skb_network_header(skb) - skb->data;
9199         if (len & ~(63 * 2))
9200                 goto out_err;
9201
9202         /* IPLEN and EIPLEN can support at most 127 dwords */
9203         len = skb_transport_header(skb) - skb_network_header(skb);
9204         if (len & ~(127 * 4))
9205                 goto out_err;
9206
9207         if (skb->encapsulation) {
9208                 /* L4TUNLEN can support 127 words */
9209                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
9210                 if (len & ~(127 * 2))
9211                         goto out_err;
9212
9213                 /* IPLEN can support at most 127 dwords */
9214                 len = skb_inner_transport_header(skb) -
9215                       skb_inner_network_header(skb);
9216                 if (len & ~(127 * 4))
9217                         goto out_err;
9218         }
9219
9220         /* No need to validate L4LEN as TCP is the only protocol with a
9221          * a flexible value and we support all possible values supported
9222          * by TCP, which is at most 15 dwords
9223          */
9224
9225         return features;
9226 out_err:
9227         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9228 }
9229
9230 static const struct net_device_ops i40e_netdev_ops = {
9231         .ndo_open               = i40e_open,
9232         .ndo_stop               = i40e_close,
9233         .ndo_start_xmit         = i40e_lan_xmit_frame,
9234         .ndo_get_stats64        = i40e_get_netdev_stats_struct,
9235         .ndo_set_rx_mode        = i40e_set_rx_mode,
9236         .ndo_validate_addr      = eth_validate_addr,
9237         .ndo_set_mac_address    = i40e_set_mac,
9238         .ndo_change_mtu         = i40e_change_mtu,
9239         .ndo_do_ioctl           = i40e_ioctl,
9240         .ndo_tx_timeout         = i40e_tx_timeout,
9241         .ndo_vlan_rx_add_vid    = i40e_vlan_rx_add_vid,
9242         .ndo_vlan_rx_kill_vid   = i40e_vlan_rx_kill_vid,
9243 #ifdef CONFIG_NET_POLL_CONTROLLER
9244         .ndo_poll_controller    = i40e_netpoll,
9245 #endif
9246         .ndo_setup_tc           = __i40e_setup_tc,
9247 #ifdef I40E_FCOE
9248         .ndo_fcoe_enable        = i40e_fcoe_enable,
9249         .ndo_fcoe_disable       = i40e_fcoe_disable,
9250 #endif
9251         .ndo_set_features       = i40e_set_features,
9252         .ndo_set_vf_mac         = i40e_ndo_set_vf_mac,
9253         .ndo_set_vf_vlan        = i40e_ndo_set_vf_port_vlan,
9254         .ndo_set_vf_rate        = i40e_ndo_set_vf_bw,
9255         .ndo_get_vf_config      = i40e_ndo_get_vf_config,
9256         .ndo_set_vf_link_state  = i40e_ndo_set_vf_link_state,
9257         .ndo_set_vf_spoofchk    = i40e_ndo_set_vf_spoofchk,
9258         .ndo_set_vf_trust       = i40e_ndo_set_vf_trust,
9259         .ndo_udp_tunnel_add     = i40e_udp_tunnel_add,
9260         .ndo_udp_tunnel_del     = i40e_udp_tunnel_del,
9261         .ndo_get_phys_port_id   = i40e_get_phys_port_id,
9262         .ndo_fdb_add            = i40e_ndo_fdb_add,
9263         .ndo_features_check     = i40e_features_check,
9264         .ndo_bridge_getlink     = i40e_ndo_bridge_getlink,
9265         .ndo_bridge_setlink     = i40e_ndo_bridge_setlink,
9266 };
9267
9268 /**
9269  * i40e_config_netdev - Setup the netdev flags
9270  * @vsi: the VSI being configured
9271  *
9272  * Returns 0 on success, negative value on failure
9273  **/
9274 static int i40e_config_netdev(struct i40e_vsi *vsi)
9275 {
9276         struct i40e_pf *pf = vsi->back;
9277         struct i40e_hw *hw = &pf->hw;
9278         struct i40e_netdev_priv *np;
9279         struct net_device *netdev;
9280         u8 broadcast[ETH_ALEN];
9281         u8 mac_addr[ETH_ALEN];
9282         int etherdev_size;
9283
9284         etherdev_size = sizeof(struct i40e_netdev_priv);
9285         netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
9286         if (!netdev)
9287                 return -ENOMEM;
9288
9289         vsi->netdev = netdev;
9290         np = netdev_priv(netdev);
9291         np->vsi = vsi;
9292
9293         netdev->hw_enc_features |= NETIF_F_SG                   |
9294                                    NETIF_F_IP_CSUM              |
9295                                    NETIF_F_IPV6_CSUM            |
9296                                    NETIF_F_HIGHDMA              |
9297                                    NETIF_F_SOFT_FEATURES        |
9298                                    NETIF_F_TSO                  |
9299                                    NETIF_F_TSO_ECN              |
9300                                    NETIF_F_TSO6                 |
9301                                    NETIF_F_GSO_GRE              |
9302                                    NETIF_F_GSO_GRE_CSUM         |
9303                                    NETIF_F_GSO_IPXIP4           |
9304                                    NETIF_F_GSO_IPXIP6           |
9305                                    NETIF_F_GSO_UDP_TUNNEL       |
9306                                    NETIF_F_GSO_UDP_TUNNEL_CSUM  |
9307                                    NETIF_F_GSO_PARTIAL          |
9308                                    NETIF_F_SCTP_CRC             |
9309                                    NETIF_F_RXHASH               |
9310                                    NETIF_F_RXCSUM               |
9311                                    0;
9312
9313         if (!(pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE))
9314                 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
9315
9316         netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
9317
9318         /* record features VLANs can make use of */
9319         netdev->vlan_features |= netdev->hw_enc_features |
9320                                  NETIF_F_TSO_MANGLEID;
9321
9322         if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
9323                 netdev->hw_features |= NETIF_F_NTUPLE;
9324
9325         netdev->hw_features |= netdev->hw_enc_features  |
9326                                NETIF_F_HW_VLAN_CTAG_TX  |
9327                                NETIF_F_HW_VLAN_CTAG_RX;
9328
9329         netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
9330         netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
9331
9332         if (vsi->type == I40E_VSI_MAIN) {
9333                 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
9334                 ether_addr_copy(mac_addr, hw->mac.perm_addr);
9335                 /* The following steps are necessary to prevent reception
9336                  * of tagged packets - some older NVM configurations load a
9337                  * default a MAC-VLAN filter that accepts any tagged packet
9338                  * which must be replaced by a normal filter.
9339                  */
9340                 i40e_rm_default_mac_filter(vsi, mac_addr);
9341                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9342                 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY);
9343                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9344         } else {
9345                 /* relate the VSI_VMDQ name to the VSI_MAIN name */
9346                 snprintf(netdev->name, IFNAMSIZ, "%sv%%d",
9347                          pf->vsi[pf->lan_vsi]->netdev->name);
9348                 random_ether_addr(mac_addr);
9349
9350                 spin_lock_bh(&vsi->mac_filter_hash_lock);
9351                 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY);
9352                 spin_unlock_bh(&vsi->mac_filter_hash_lock);
9353         }
9354
9355         /* Add the broadcast filter so that we initially will receive
9356          * broadcast packets. Note that when a new VLAN is first added the
9357          * driver will convert all filters marked I40E_VLAN_ANY into VLAN
9358          * specific filters as part of transitioning into "vlan" operation.
9359          * When more VLANs are added, the driver will copy each existing MAC
9360          * filter and add it for the new VLAN.
9361          *
9362          * Broadcast filters are handled specially by
9363          * i40e_sync_filters_subtask, as the driver must to set the broadcast
9364          * promiscuous bit instead of adding this directly as a MAC/VLAN
9365          * filter. The subtask will update the correct broadcast promiscuous
9366          * bits as VLANs become active or inactive.
9367          */
9368         eth_broadcast_addr(broadcast);
9369         spin_lock_bh(&vsi->mac_filter_hash_lock);
9370         i40e_add_filter(vsi, broadcast, I40E_VLAN_ANY);
9371         spin_unlock_bh(&vsi->mac_filter_hash_lock);
9372
9373         ether_addr_copy(netdev->dev_addr, mac_addr);
9374         ether_addr_copy(netdev->perm_addr, mac_addr);
9375
9376         netdev->priv_flags |= IFF_UNICAST_FLT;
9377         netdev->priv_flags |= IFF_SUPP_NOFCS;
9378         /* Setup netdev TC information */
9379         i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
9380
9381         netdev->netdev_ops = &i40e_netdev_ops;
9382         netdev->watchdog_timeo = 5 * HZ;
9383         i40e_set_ethtool_ops(netdev);
9384 #ifdef I40E_FCOE
9385         i40e_fcoe_config_netdev(netdev, vsi);
9386 #endif
9387
9388         /* MTU range: 68 - 9706 */
9389         netdev->min_mtu = ETH_MIN_MTU;
9390         netdev->max_mtu = I40E_MAX_RXBUFFER -
9391                           (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN);
9392
9393         return 0;
9394 }
9395
9396 /**
9397  * i40e_vsi_delete - Delete a VSI from the switch
9398  * @vsi: the VSI being removed
9399  *
9400  * Returns 0 on success, negative value on failure
9401  **/
9402 static void i40e_vsi_delete(struct i40e_vsi *vsi)
9403 {
9404         /* remove default VSI is not allowed */
9405         if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
9406                 return;
9407
9408         i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
9409 }
9410
9411 /**
9412  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
9413  * @vsi: the VSI being queried
9414  *
9415  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
9416  **/
9417 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
9418 {
9419         struct i40e_veb *veb;
9420         struct i40e_pf *pf = vsi->back;
9421
9422         /* Uplink is not a bridge so default to VEB */
9423         if (vsi->veb_idx == I40E_NO_VEB)
9424                 return 1;
9425
9426         veb = pf->veb[vsi->veb_idx];
9427         if (!veb) {
9428                 dev_info(&pf->pdev->dev,
9429                          "There is no veb associated with the bridge\n");
9430                 return -ENOENT;
9431         }
9432
9433         /* Uplink is a bridge in VEPA mode */
9434         if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
9435                 return 0;
9436         } else {
9437                 /* Uplink is a bridge in VEB mode */
9438                 return 1;
9439         }
9440
9441         /* VEPA is now default bridge, so return 0 */
9442         return 0;
9443 }
9444
9445 /**
9446  * i40e_add_vsi - Add a VSI to the switch
9447  * @vsi: the VSI being configured
9448  *
9449  * This initializes a VSI context depending on the VSI type to be added and
9450  * passes it down to the add_vsi aq command.
9451  **/
9452 static int i40e_add_vsi(struct i40e_vsi *vsi)
9453 {
9454         int ret = -ENODEV;
9455         struct i40e_pf *pf = vsi->back;
9456         struct i40e_hw *hw = &pf->hw;
9457         struct i40e_vsi_context ctxt;
9458         struct i40e_mac_filter *f;
9459         struct hlist_node *h;
9460         int bkt;
9461
9462         u8 enabled_tc = 0x1; /* TC0 enabled */
9463         int f_count = 0;
9464
9465         memset(&ctxt, 0, sizeof(ctxt));
9466         switch (vsi->type) {
9467         case I40E_VSI_MAIN:
9468                 /* The PF's main VSI is already setup as part of the
9469                  * device initialization, so we'll not bother with
9470                  * the add_vsi call, but we will retrieve the current
9471                  * VSI context.
9472                  */
9473                 ctxt.seid = pf->main_vsi_seid;
9474                 ctxt.pf_num = pf->hw.pf_id;
9475                 ctxt.vf_num = 0;
9476                 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9477                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9478                 if (ret) {
9479                         dev_info(&pf->pdev->dev,
9480                                  "couldn't get PF vsi config, err %s aq_err %s\n",
9481                                  i40e_stat_str(&pf->hw, ret),
9482                                  i40e_aq_str(&pf->hw,
9483                                              pf->hw.aq.asq_last_status));
9484                         return -ENOENT;
9485                 }
9486                 vsi->info = ctxt.info;
9487                 vsi->info.valid_sections = 0;
9488
9489                 vsi->seid = ctxt.seid;
9490                 vsi->id = ctxt.vsi_number;
9491
9492                 enabled_tc = i40e_pf_get_tc_map(pf);
9493
9494                 /* MFP mode setup queue map and update VSI */
9495                 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
9496                     !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
9497                         memset(&ctxt, 0, sizeof(ctxt));
9498                         ctxt.seid = pf->main_vsi_seid;
9499                         ctxt.pf_num = pf->hw.pf_id;
9500                         ctxt.vf_num = 0;
9501                         i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
9502                         ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
9503                         if (ret) {
9504                                 dev_info(&pf->pdev->dev,
9505                                          "update vsi failed, err %s aq_err %s\n",
9506                                          i40e_stat_str(&pf->hw, ret),
9507                                          i40e_aq_str(&pf->hw,
9508                                                     pf->hw.aq.asq_last_status));
9509                                 ret = -ENOENT;
9510                                 goto err;
9511                         }
9512                         /* update the local VSI info queue map */
9513                         i40e_vsi_update_queue_map(vsi, &ctxt);
9514                         vsi->info.valid_sections = 0;
9515                 } else {
9516                         /* Default/Main VSI is only enabled for TC0
9517                          * reconfigure it to enable all TCs that are
9518                          * available on the port in SFP mode.
9519                          * For MFP case the iSCSI PF would use this
9520                          * flow to enable LAN+iSCSI TC.
9521                          */
9522                         ret = i40e_vsi_config_tc(vsi, enabled_tc);
9523                         if (ret) {
9524                                 dev_info(&pf->pdev->dev,
9525                                          "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
9526                                          enabled_tc,
9527                                          i40e_stat_str(&pf->hw, ret),
9528                                          i40e_aq_str(&pf->hw,
9529                                                     pf->hw.aq.asq_last_status));
9530                                 ret = -ENOENT;
9531                         }
9532                 }
9533                 break;
9534
9535         case I40E_VSI_FDIR:
9536                 ctxt.pf_num = hw->pf_id;
9537                 ctxt.vf_num = 0;
9538                 ctxt.uplink_seid = vsi->uplink_seid;
9539                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9540                 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9541                 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
9542                     (i40e_is_vsi_uplink_mode_veb(vsi))) {
9543                         ctxt.info.valid_sections |=
9544                              cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9545                         ctxt.info.switch_id =
9546                            cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9547                 }
9548                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9549                 break;
9550
9551         case I40E_VSI_VMDQ2:
9552                 ctxt.pf_num = hw->pf_id;
9553                 ctxt.vf_num = 0;
9554                 ctxt.uplink_seid = vsi->uplink_seid;
9555                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9556                 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
9557
9558                 /* This VSI is connected to VEB so the switch_id
9559                  * should be set to zero by default.
9560                  */
9561                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9562                         ctxt.info.valid_sections |=
9563                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9564                         ctxt.info.switch_id =
9565                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9566                 }
9567
9568                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9569                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9570                 break;
9571
9572         case I40E_VSI_SRIOV:
9573                 ctxt.pf_num = hw->pf_id;
9574                 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
9575                 ctxt.uplink_seid = vsi->uplink_seid;
9576                 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
9577                 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
9578
9579                 /* This VSI is connected to VEB so the switch_id
9580                  * should be set to zero by default.
9581                  */
9582                 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
9583                         ctxt.info.valid_sections |=
9584                                 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9585                         ctxt.info.switch_id =
9586                                 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9587                 }
9588
9589                 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
9590                         ctxt.info.valid_sections |=
9591                                 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
9592                         ctxt.info.queueing_opt_flags |=
9593                                 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
9594                                  I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
9595                 }
9596
9597                 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
9598                 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
9599                 if (pf->vf[vsi->vf_id].spoofchk) {
9600                         ctxt.info.valid_sections |=
9601                                 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
9602                         ctxt.info.sec_flags |=
9603                                 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
9604                                  I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
9605                 }
9606                 /* Setup the VSI tx/rx queue map for TC0 only for now */
9607                 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
9608                 break;
9609
9610 #ifdef I40E_FCOE
9611         case I40E_VSI_FCOE:
9612                 ret = i40e_fcoe_vsi_init(vsi, &ctxt);
9613                 if (ret) {
9614                         dev_info(&pf->pdev->dev, "failed to initialize FCoE VSI\n");
9615                         return ret;
9616                 }
9617                 break;
9618
9619 #endif /* I40E_FCOE */
9620         case I40E_VSI_IWARP:
9621                 /* send down message to iWARP */
9622                 break;
9623
9624         default:
9625                 return -ENODEV;
9626         }
9627
9628         if (vsi->type != I40E_VSI_MAIN) {
9629                 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
9630                 if (ret) {
9631                         dev_info(&vsi->back->pdev->dev,
9632                                  "add vsi failed, err %s aq_err %s\n",
9633                                  i40e_stat_str(&pf->hw, ret),
9634                                  i40e_aq_str(&pf->hw,
9635                                              pf->hw.aq.asq_last_status));
9636                         ret = -ENOENT;
9637                         goto err;
9638                 }
9639                 vsi->info = ctxt.info;
9640                 vsi->info.valid_sections = 0;
9641                 vsi->seid = ctxt.seid;
9642                 vsi->id = ctxt.vsi_number;
9643         }
9644
9645         vsi->active_filters = 0;
9646         clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state);
9647         spin_lock_bh(&vsi->mac_filter_hash_lock);
9648         /* If macvlan filters already exist, force them to get loaded */
9649         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
9650                 f->state = I40E_FILTER_NEW;
9651                 f_count++;
9652         }
9653         spin_unlock_bh(&vsi->mac_filter_hash_lock);
9654
9655         if (f_count) {
9656                 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
9657                 pf->flags |= I40E_FLAG_FILTER_SYNC;
9658         }
9659
9660         /* Update VSI BW information */
9661         ret = i40e_vsi_get_bw_info(vsi);
9662         if (ret) {
9663                 dev_info(&pf->pdev->dev,
9664                          "couldn't get vsi bw info, err %s aq_err %s\n",
9665                          i40e_stat_str(&pf->hw, ret),
9666                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9667                 /* VSI is already added so not tearing that up */
9668                 ret = 0;
9669         }
9670
9671 err:
9672         return ret;
9673 }
9674
9675 /**
9676  * i40e_vsi_release - Delete a VSI and free its resources
9677  * @vsi: the VSI being removed
9678  *
9679  * Returns 0 on success or < 0 on error
9680  **/
9681 int i40e_vsi_release(struct i40e_vsi *vsi)
9682 {
9683         struct i40e_mac_filter *f;
9684         struct hlist_node *h;
9685         struct i40e_veb *veb = NULL;
9686         struct i40e_pf *pf;
9687         u16 uplink_seid;
9688         int i, n, bkt;
9689
9690         pf = vsi->back;
9691
9692         /* release of a VEB-owner or last VSI is not allowed */
9693         if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
9694                 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
9695                          vsi->seid, vsi->uplink_seid);
9696                 return -ENODEV;
9697         }
9698         if (vsi == pf->vsi[pf->lan_vsi] &&
9699             !test_bit(__I40E_DOWN, &pf->state)) {
9700                 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
9701                 return -ENODEV;
9702         }
9703
9704         uplink_seid = vsi->uplink_seid;
9705         if (vsi->type != I40E_VSI_SRIOV) {
9706                 if (vsi->netdev_registered) {
9707                         vsi->netdev_registered = false;
9708                         if (vsi->netdev) {
9709                                 /* results in a call to i40e_close() */
9710                                 unregister_netdev(vsi->netdev);
9711                         }
9712                 } else {
9713                         i40e_vsi_close(vsi);
9714                 }
9715                 i40e_vsi_disable_irq(vsi);
9716         }
9717
9718         spin_lock_bh(&vsi->mac_filter_hash_lock);
9719
9720         /* clear the sync flag on all filters */
9721         if (vsi->netdev) {
9722                 __dev_uc_unsync(vsi->netdev, NULL);
9723                 __dev_mc_unsync(vsi->netdev, NULL);
9724         }
9725
9726         /* make sure any remaining filters are marked for deletion */
9727         hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
9728                 __i40e_del_filter(vsi, f);
9729
9730         spin_unlock_bh(&vsi->mac_filter_hash_lock);
9731
9732         i40e_sync_vsi_filters(vsi);
9733
9734         i40e_vsi_delete(vsi);
9735         i40e_vsi_free_q_vectors(vsi);
9736         if (vsi->netdev) {
9737                 free_netdev(vsi->netdev);
9738                 vsi->netdev = NULL;
9739         }
9740         i40e_vsi_clear_rings(vsi);
9741         i40e_vsi_clear(vsi);
9742
9743         /* If this was the last thing on the VEB, except for the
9744          * controlling VSI, remove the VEB, which puts the controlling
9745          * VSI onto the next level down in the switch.
9746          *
9747          * Well, okay, there's one more exception here: don't remove
9748          * the orphan VEBs yet.  We'll wait for an explicit remove request
9749          * from up the network stack.
9750          */
9751         for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
9752                 if (pf->vsi[i] &&
9753                     pf->vsi[i]->uplink_seid == uplink_seid &&
9754                     (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
9755                         n++;      /* count the VSIs */
9756                 }
9757         }
9758         for (i = 0; i < I40E_MAX_VEB; i++) {
9759                 if (!pf->veb[i])
9760                         continue;
9761                 if (pf->veb[i]->uplink_seid == uplink_seid)
9762                         n++;     /* count the VEBs */
9763                 if (pf->veb[i]->seid == uplink_seid)
9764                         veb = pf->veb[i];
9765         }
9766         if (n == 0 && veb && veb->uplink_seid != 0)
9767                 i40e_veb_release(veb);
9768
9769         return 0;
9770 }
9771
9772 /**
9773  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
9774  * @vsi: ptr to the VSI
9775  *
9776  * This should only be called after i40e_vsi_mem_alloc() which allocates the
9777  * corresponding SW VSI structure and initializes num_queue_pairs for the
9778  * newly allocated VSI.
9779  *
9780  * Returns 0 on success or negative on failure
9781  **/
9782 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
9783 {
9784         int ret = -ENOENT;
9785         struct i40e_pf *pf = vsi->back;
9786
9787         if (vsi->q_vectors[0]) {
9788                 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
9789                          vsi->seid);
9790                 return -EEXIST;
9791         }
9792
9793         if (vsi->base_vector) {
9794                 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
9795                          vsi->seid, vsi->base_vector);
9796                 return -EEXIST;
9797         }
9798
9799         ret = i40e_vsi_alloc_q_vectors(vsi);
9800         if (ret) {
9801                 dev_info(&pf->pdev->dev,
9802                          "failed to allocate %d q_vector for VSI %d, ret=%d\n",
9803                          vsi->num_q_vectors, vsi->seid, ret);
9804                 vsi->num_q_vectors = 0;
9805                 goto vector_setup_out;
9806         }
9807
9808         /* In Legacy mode, we do not have to get any other vector since we
9809          * piggyback on the misc/ICR0 for queue interrupts.
9810         */
9811         if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
9812                 return ret;
9813         if (vsi->num_q_vectors)
9814                 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
9815                                                  vsi->num_q_vectors, vsi->idx);
9816         if (vsi->base_vector < 0) {
9817                 dev_info(&pf->pdev->dev,
9818                          "failed to get tracking for %d vectors for VSI %d, err=%d\n",
9819                          vsi->num_q_vectors, vsi->seid, vsi->base_vector);
9820                 i40e_vsi_free_q_vectors(vsi);
9821                 ret = -ENOENT;
9822                 goto vector_setup_out;
9823         }
9824
9825 vector_setup_out:
9826         return ret;
9827 }
9828
9829 /**
9830  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
9831  * @vsi: pointer to the vsi.
9832  *
9833  * This re-allocates a vsi's queue resources.
9834  *
9835  * Returns pointer to the successfully allocated and configured VSI sw struct
9836  * on success, otherwise returns NULL on failure.
9837  **/
9838 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
9839 {
9840         struct i40e_pf *pf;
9841         u8 enabled_tc;
9842         int ret;
9843
9844         if (!vsi)
9845                 return NULL;
9846
9847         pf = vsi->back;
9848
9849         i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
9850         i40e_vsi_clear_rings(vsi);
9851
9852         i40e_vsi_free_arrays(vsi, false);
9853         i40e_set_num_rings_in_vsi(vsi);
9854         ret = i40e_vsi_alloc_arrays(vsi, false);
9855         if (ret)
9856                 goto err_vsi;
9857
9858         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx);
9859         if (ret < 0) {
9860                 dev_info(&pf->pdev->dev,
9861                          "failed to get tracking for %d queues for VSI %d err %d\n",
9862                          vsi->alloc_queue_pairs, vsi->seid, ret);
9863                 goto err_vsi;
9864         }
9865         vsi->base_queue = ret;
9866
9867         /* Update the FW view of the VSI. Force a reset of TC and queue
9868          * layout configurations.
9869          */
9870         enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
9871         pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
9872         pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
9873         i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
9874         if (vsi->type == I40E_VSI_MAIN)
9875                 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
9876
9877         /* assign it some queues */
9878         ret = i40e_alloc_rings(vsi);
9879         if (ret)
9880                 goto err_rings;
9881
9882         /* map all of the rings to the q_vectors */
9883         i40e_vsi_map_rings_to_vectors(vsi);
9884         return vsi;
9885
9886 err_rings:
9887         i40e_vsi_free_q_vectors(vsi);
9888         if (vsi->netdev_registered) {
9889                 vsi->netdev_registered = false;
9890                 unregister_netdev(vsi->netdev);
9891                 free_netdev(vsi->netdev);
9892                 vsi->netdev = NULL;
9893         }
9894         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
9895 err_vsi:
9896         i40e_vsi_clear(vsi);
9897         return NULL;
9898 }
9899
9900 /**
9901  * i40e_vsi_setup - Set up a VSI by a given type
9902  * @pf: board private structure
9903  * @type: VSI type
9904  * @uplink_seid: the switch element to link to
9905  * @param1: usage depends upon VSI type. For VF types, indicates VF id
9906  *
9907  * This allocates the sw VSI structure and its queue resources, then add a VSI
9908  * to the identified VEB.
9909  *
9910  * Returns pointer to the successfully allocated and configure VSI sw struct on
9911  * success, otherwise returns NULL on failure.
9912  **/
9913 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
9914                                 u16 uplink_seid, u32 param1)
9915 {
9916         struct i40e_vsi *vsi = NULL;
9917         struct i40e_veb *veb = NULL;
9918         int ret, i;
9919         int v_idx;
9920
9921         /* The requested uplink_seid must be either
9922          *     - the PF's port seid
9923          *              no VEB is needed because this is the PF
9924          *              or this is a Flow Director special case VSI
9925          *     - seid of an existing VEB
9926          *     - seid of a VSI that owns an existing VEB
9927          *     - seid of a VSI that doesn't own a VEB
9928          *              a new VEB is created and the VSI becomes the owner
9929          *     - seid of the PF VSI, which is what creates the first VEB
9930          *              this is a special case of the previous
9931          *
9932          * Find which uplink_seid we were given and create a new VEB if needed
9933          */
9934         for (i = 0; i < I40E_MAX_VEB; i++) {
9935                 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
9936                         veb = pf->veb[i];
9937                         break;
9938                 }
9939         }
9940
9941         if (!veb && uplink_seid != pf->mac_seid) {
9942
9943                 for (i = 0; i < pf->num_alloc_vsi; i++) {
9944                         if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
9945                                 vsi = pf->vsi[i];
9946                                 break;
9947                         }
9948                 }
9949                 if (!vsi) {
9950                         dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
9951                                  uplink_seid);
9952                         return NULL;
9953                 }
9954
9955                 if (vsi->uplink_seid == pf->mac_seid)
9956                         veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
9957                                              vsi->tc_config.enabled_tc);
9958                 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
9959                         veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
9960                                              vsi->tc_config.enabled_tc);
9961                 if (veb) {
9962                         if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
9963                                 dev_info(&vsi->back->pdev->dev,
9964                                          "New VSI creation error, uplink seid of LAN VSI expected.\n");
9965                                 return NULL;
9966                         }
9967                         /* We come up by default in VEPA mode if SRIOV is not
9968                          * already enabled, in which case we can't force VEPA
9969                          * mode.
9970                          */
9971                         if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
9972                                 veb->bridge_mode = BRIDGE_MODE_VEPA;
9973                                 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
9974                         }
9975                         i40e_config_bridge_mode(veb);
9976                 }
9977                 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
9978                         if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
9979                                 veb = pf->veb[i];
9980                 }
9981                 if (!veb) {
9982                         dev_info(&pf->pdev->dev, "couldn't add VEB\n");
9983                         return NULL;
9984                 }
9985
9986                 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
9987                 uplink_seid = veb->seid;
9988         }
9989
9990         /* get vsi sw struct */
9991         v_idx = i40e_vsi_mem_alloc(pf, type);
9992         if (v_idx < 0)
9993                 goto err_alloc;
9994         vsi = pf->vsi[v_idx];
9995         if (!vsi)
9996                 goto err_alloc;
9997         vsi->type = type;
9998         vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
9999
10000         if (type == I40E_VSI_MAIN)
10001                 pf->lan_vsi = v_idx;
10002         else if (type == I40E_VSI_SRIOV)
10003                 vsi->vf_id = param1;
10004         /* assign it some queues */
10005         ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs,
10006                                 vsi->idx);
10007         if (ret < 0) {
10008                 dev_info(&pf->pdev->dev,
10009                          "failed to get tracking for %d queues for VSI %d err=%d\n",
10010                          vsi->alloc_queue_pairs, vsi->seid, ret);
10011                 goto err_vsi;
10012         }
10013         vsi->base_queue = ret;
10014
10015         /* get a VSI from the hardware */
10016         vsi->uplink_seid = uplink_seid;
10017         ret = i40e_add_vsi(vsi);
10018         if (ret)
10019                 goto err_vsi;
10020
10021         switch (vsi->type) {
10022         /* setup the netdev if needed */
10023         case I40E_VSI_MAIN:
10024                 /* Apply relevant filters if a platform-specific mac
10025                  * address was selected.
10026                  */
10027                 if (!!(pf->flags & I40E_FLAG_PF_MAC)) {
10028                         ret = i40e_macaddr_init(vsi, pf->hw.mac.addr);
10029                         if (ret) {
10030                                 dev_warn(&pf->pdev->dev,
10031                                          "could not set up macaddr; err %d\n",
10032                                          ret);
10033                         }
10034                 }
10035         case I40E_VSI_VMDQ2:
10036         case I40E_VSI_FCOE:
10037                 ret = i40e_config_netdev(vsi);
10038                 if (ret)
10039                         goto err_netdev;
10040                 ret = register_netdev(vsi->netdev);
10041                 if (ret)
10042                         goto err_netdev;
10043                 vsi->netdev_registered = true;
10044                 netif_carrier_off(vsi->netdev);
10045 #ifdef CONFIG_I40E_DCB
10046                 /* Setup DCB netlink interface */
10047                 i40e_dcbnl_setup(vsi);
10048 #endif /* CONFIG_I40E_DCB */
10049                 /* fall through */
10050
10051         case I40E_VSI_FDIR:
10052                 /* set up vectors and rings if needed */
10053                 ret = i40e_vsi_setup_vectors(vsi);
10054                 if (ret)
10055                         goto err_msix;
10056
10057                 ret = i40e_alloc_rings(vsi);
10058                 if (ret)
10059                         goto err_rings;
10060
10061                 /* map all of the rings to the q_vectors */
10062                 i40e_vsi_map_rings_to_vectors(vsi);
10063
10064                 i40e_vsi_reset_stats(vsi);
10065                 break;
10066
10067         default:
10068                 /* no netdev or rings for the other VSI types */
10069                 break;
10070         }
10071
10072         if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) &&
10073             (vsi->type == I40E_VSI_VMDQ2)) {
10074                 ret = i40e_vsi_config_rss(vsi);
10075         }
10076         return vsi;
10077
10078 err_rings:
10079         i40e_vsi_free_q_vectors(vsi);
10080 err_msix:
10081         if (vsi->netdev_registered) {
10082                 vsi->netdev_registered = false;
10083                 unregister_netdev(vsi->netdev);
10084                 free_netdev(vsi->netdev);
10085                 vsi->netdev = NULL;
10086         }
10087 err_netdev:
10088         i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
10089 err_vsi:
10090         i40e_vsi_clear(vsi);
10091 err_alloc:
10092         return NULL;
10093 }
10094
10095 /**
10096  * i40e_veb_get_bw_info - Query VEB BW information
10097  * @veb: the veb to query
10098  *
10099  * Query the Tx scheduler BW configuration data for given VEB
10100  **/
10101 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
10102 {
10103         struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
10104         struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
10105         struct i40e_pf *pf = veb->pf;
10106         struct i40e_hw *hw = &pf->hw;
10107         u32 tc_bw_max;
10108         int ret = 0;
10109         int i;
10110
10111         ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
10112                                                   &bw_data, NULL);
10113         if (ret) {
10114                 dev_info(&pf->pdev->dev,
10115                          "query veb bw config failed, err %s aq_err %s\n",
10116                          i40e_stat_str(&pf->hw, ret),
10117                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10118                 goto out;
10119         }
10120
10121         ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
10122                                                    &ets_data, NULL);
10123         if (ret) {
10124                 dev_info(&pf->pdev->dev,
10125                          "query veb bw ets config failed, err %s aq_err %s\n",
10126                          i40e_stat_str(&pf->hw, ret),
10127                          i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
10128                 goto out;
10129         }
10130
10131         veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
10132         veb->bw_max_quanta = ets_data.tc_bw_max;
10133         veb->is_abs_credits = bw_data.absolute_credits_enable;
10134         veb->enabled_tc = ets_data.tc_valid_bits;
10135         tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
10136                     (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
10137         for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
10138                 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
10139                 veb->bw_tc_limit_credits[i] =
10140                                         le16_to_cpu(bw_data.tc_bw_limits[i]);
10141                 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
10142         }
10143
10144 out:
10145         return ret;
10146 }
10147
10148 /**
10149  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
10150  * @pf: board private structure
10151  *
10152  * On error: returns error code (negative)
10153  * On success: returns vsi index in PF (positive)
10154  **/
10155 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
10156 {
10157         int ret = -ENOENT;
10158         struct i40e_veb *veb;
10159         int i;
10160
10161         /* Need to protect the allocation of switch elements at the PF level */
10162         mutex_lock(&pf->switch_mutex);
10163
10164         /* VEB list may be fragmented if VEB creation/destruction has
10165          * been happening.  We can afford to do a quick scan to look
10166          * for any free slots in the list.
10167          *
10168          * find next empty veb slot, looping back around if necessary
10169          */
10170         i = 0;
10171         while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
10172                 i++;
10173         if (i >= I40E_MAX_VEB) {
10174                 ret = -ENOMEM;
10175                 goto err_alloc_veb;  /* out of VEB slots! */
10176         }
10177
10178         veb = kzalloc(sizeof(*veb), GFP_KERNEL);
10179         if (!veb) {
10180                 ret = -ENOMEM;
10181                 goto err_alloc_veb;
10182         }
10183         veb->pf = pf;
10184         veb->idx = i;
10185         veb->enabled_tc = 1;
10186
10187         pf->veb[i] = veb;
10188         ret = i;
10189 err_alloc_veb:
10190         mutex_unlock(&pf->switch_mutex);
10191         return ret;
10192 }
10193
10194 /**
10195  * i40e_switch_branch_release - Delete a branch of the switch tree
10196  * @branch: where to start deleting
10197  *
10198  * This uses recursion to find the tips of the branch to be
10199  * removed, deleting until we get back to and can delete this VEB.
10200  **/
10201 static void i40e_switch_branch_release(struct i40e_veb *branch)
10202 {
10203         struct i40e_pf *pf = branch->pf;
10204         u16 branch_seid = branch->seid;
10205         u16 veb_idx = branch->idx;
10206         int i;
10207
10208         /* release any VEBs on this VEB - RECURSION */
10209         for (i = 0; i < I40E_MAX_VEB; i++) {
10210                 if (!pf->veb[i])
10211                         continue;
10212                 if (pf->veb[i]->uplink_seid == branch->seid)
10213                         i40e_switch_branch_release(pf->veb[i]);
10214         }
10215
10216         /* Release the VSIs on this VEB, but not the owner VSI.
10217          *
10218          * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
10219          *       the VEB itself, so don't use (*branch) after this loop.
10220          */
10221         for (i = 0; i < pf->num_alloc_vsi; i++) {
10222                 if (!pf->vsi[i])
10223                         continue;
10224                 if (pf->vsi[i]->uplink_seid == branch_seid &&
10225                    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
10226                         i40e_vsi_release(pf->vsi[i]);
10227                 }
10228         }
10229
10230         /* There's one corner case where the VEB might not have been
10231          * removed, so double check it here and remove it if needed.
10232          * This case happens if the veb was created from the debugfs
10233          * commands and no VSIs were added to it.
10234          */
10235         if (pf->veb[veb_idx])
10236                 i40e_veb_release(pf->veb[veb_idx]);
10237 }
10238
10239 /**
10240  * i40e_veb_clear - remove veb struct
10241  * @veb: the veb to remove
10242  **/
10243 static void i40e_veb_clear(struct i40e_veb *veb)
10244 {
10245         if (!veb)
10246                 return;
10247
10248         if (veb->pf) {
10249                 struct i40e_pf *pf = veb->pf;
10250
10251                 mutex_lock(&pf->switch_mutex);
10252                 if (pf->veb[veb->idx] == veb)
10253                         pf->veb[veb->idx] = NULL;
10254                 mutex_unlock(&pf->switch_mutex);
10255         }
10256
10257         kfree(veb);
10258 }
10259
10260 /**
10261  * i40e_veb_release - Delete a VEB and free its resources
10262  * @veb: the VEB being removed
10263  **/
10264 void i40e_veb_release(struct i40e_veb *veb)
10265 {
10266         struct i40e_vsi *vsi = NULL;
10267         struct i40e_pf *pf;
10268         int i, n = 0;
10269
10270         pf = veb->pf;
10271
10272         /* find the remaining VSI and check for extras */
10273         for (i = 0; i < pf->num_alloc_vsi; i++) {
10274                 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
10275                         n++;
10276                         vsi = pf->vsi[i];
10277                 }
10278         }
10279         if (n != 1) {
10280                 dev_info(&pf->pdev->dev,
10281                          "can't remove VEB %d with %d VSIs left\n",
10282                          veb->seid, n);
10283                 return;
10284         }
10285
10286         /* move the remaining VSI to uplink veb */
10287         vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
10288         if (veb->uplink_seid) {
10289                 vsi->uplink_seid = veb->uplink_seid;
10290                 if (veb->uplink_seid == pf->mac_seid)
10291                         vsi->veb_idx = I40E_NO_VEB;
10292                 else
10293                         vsi->veb_idx = veb->veb_idx;
10294         } else {
10295                 /* floating VEB */
10296                 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10297                 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
10298         }
10299
10300         i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10301         i40e_veb_clear(veb);
10302 }
10303
10304 /**
10305  * i40e_add_veb - create the VEB in the switch
10306  * @veb: the VEB to be instantiated
10307  * @vsi: the controlling VSI
10308  **/
10309 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
10310 {
10311         struct i40e_pf *pf = veb->pf;
10312         bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
10313         int ret;
10314
10315         ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
10316                               veb->enabled_tc, false,
10317                               &veb->seid, enable_stats, NULL);
10318
10319         /* get a VEB from the hardware */
10320         if (ret) {
10321                 dev_info(&pf->pdev->dev,
10322                          "couldn't add VEB, err %s aq_err %s\n",
10323                          i40e_stat_str(&pf->hw, ret),
10324                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10325                 return -EPERM;
10326         }
10327
10328         /* get statistics counter */
10329         ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
10330                                          &veb->stats_idx, NULL, NULL, NULL);
10331         if (ret) {
10332                 dev_info(&pf->pdev->dev,
10333                          "couldn't get VEB statistics idx, err %s aq_err %s\n",
10334                          i40e_stat_str(&pf->hw, ret),
10335                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10336                 return -EPERM;
10337         }
10338         ret = i40e_veb_get_bw_info(veb);
10339         if (ret) {
10340                 dev_info(&pf->pdev->dev,
10341                          "couldn't get VEB bw info, err %s aq_err %s\n",
10342                          i40e_stat_str(&pf->hw, ret),
10343                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10344                 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
10345                 return -ENOENT;
10346         }
10347
10348         vsi->uplink_seid = veb->seid;
10349         vsi->veb_idx = veb->idx;
10350         vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
10351
10352         return 0;
10353 }
10354
10355 /**
10356  * i40e_veb_setup - Set up a VEB
10357  * @pf: board private structure
10358  * @flags: VEB setup flags
10359  * @uplink_seid: the switch element to link to
10360  * @vsi_seid: the initial VSI seid
10361  * @enabled_tc: Enabled TC bit-map
10362  *
10363  * This allocates the sw VEB structure and links it into the switch
10364  * It is possible and legal for this to be a duplicate of an already
10365  * existing VEB.  It is also possible for both uplink and vsi seids
10366  * to be zero, in order to create a floating VEB.
10367  *
10368  * Returns pointer to the successfully allocated VEB sw struct on
10369  * success, otherwise returns NULL on failure.
10370  **/
10371 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
10372                                 u16 uplink_seid, u16 vsi_seid,
10373                                 u8 enabled_tc)
10374 {
10375         struct i40e_veb *veb, *uplink_veb = NULL;
10376         int vsi_idx, veb_idx;
10377         int ret;
10378
10379         /* if one seid is 0, the other must be 0 to create a floating relay */
10380         if ((uplink_seid == 0 || vsi_seid == 0) &&
10381             (uplink_seid + vsi_seid != 0)) {
10382                 dev_info(&pf->pdev->dev,
10383                          "one, not both seid's are 0: uplink=%d vsi=%d\n",
10384                          uplink_seid, vsi_seid);
10385                 return NULL;
10386         }
10387
10388         /* make sure there is such a vsi and uplink */
10389         for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
10390                 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
10391                         break;
10392         if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) {
10393                 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
10394                          vsi_seid);
10395                 return NULL;
10396         }
10397
10398         if (uplink_seid && uplink_seid != pf->mac_seid) {
10399                 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10400                         if (pf->veb[veb_idx] &&
10401                             pf->veb[veb_idx]->seid == uplink_seid) {
10402                                 uplink_veb = pf->veb[veb_idx];
10403                                 break;
10404                         }
10405                 }
10406                 if (!uplink_veb) {
10407                         dev_info(&pf->pdev->dev,
10408                                  "uplink seid %d not found\n", uplink_seid);
10409                         return NULL;
10410                 }
10411         }
10412
10413         /* get veb sw struct */
10414         veb_idx = i40e_veb_mem_alloc(pf);
10415         if (veb_idx < 0)
10416                 goto err_alloc;
10417         veb = pf->veb[veb_idx];
10418         veb->flags = flags;
10419         veb->uplink_seid = uplink_seid;
10420         veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
10421         veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
10422
10423         /* create the VEB in the switch */
10424         ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
10425         if (ret)
10426                 goto err_veb;
10427         if (vsi_idx == pf->lan_vsi)
10428                 pf->lan_veb = veb->idx;
10429
10430         return veb;
10431
10432 err_veb:
10433         i40e_veb_clear(veb);
10434 err_alloc:
10435         return NULL;
10436 }
10437
10438 /**
10439  * i40e_setup_pf_switch_element - set PF vars based on switch type
10440  * @pf: board private structure
10441  * @ele: element we are building info from
10442  * @num_reported: total number of elements
10443  * @printconfig: should we print the contents
10444  *
10445  * helper function to assist in extracting a few useful SEID values.
10446  **/
10447 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
10448                                 struct i40e_aqc_switch_config_element_resp *ele,
10449                                 u16 num_reported, bool printconfig)
10450 {
10451         u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
10452         u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
10453         u8 element_type = ele->element_type;
10454         u16 seid = le16_to_cpu(ele->seid);
10455
10456         if (printconfig)
10457                 dev_info(&pf->pdev->dev,
10458                          "type=%d seid=%d uplink=%d downlink=%d\n",
10459                          element_type, seid, uplink_seid, downlink_seid);
10460
10461         switch (element_type) {
10462         case I40E_SWITCH_ELEMENT_TYPE_MAC:
10463                 pf->mac_seid = seid;
10464                 break;
10465         case I40E_SWITCH_ELEMENT_TYPE_VEB:
10466                 /* Main VEB? */
10467                 if (uplink_seid != pf->mac_seid)
10468                         break;
10469                 if (pf->lan_veb == I40E_NO_VEB) {
10470                         int v;
10471
10472                         /* find existing or else empty VEB */
10473                         for (v = 0; v < I40E_MAX_VEB; v++) {
10474                                 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
10475                                         pf->lan_veb = v;
10476                                         break;
10477                                 }
10478                         }
10479                         if (pf->lan_veb == I40E_NO_VEB) {
10480                                 v = i40e_veb_mem_alloc(pf);
10481                                 if (v < 0)
10482                                         break;
10483                                 pf->lan_veb = v;
10484                         }
10485                 }
10486
10487                 pf->veb[pf->lan_veb]->seid = seid;
10488                 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
10489                 pf->veb[pf->lan_veb]->pf = pf;
10490                 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
10491                 break;
10492         case I40E_SWITCH_ELEMENT_TYPE_VSI:
10493                 if (num_reported != 1)
10494                         break;
10495                 /* This is immediately after a reset so we can assume this is
10496                  * the PF's VSI
10497                  */
10498                 pf->mac_seid = uplink_seid;
10499                 pf->pf_seid = downlink_seid;
10500                 pf->main_vsi_seid = seid;
10501                 if (printconfig)
10502                         dev_info(&pf->pdev->dev,
10503                                  "pf_seid=%d main_vsi_seid=%d\n",
10504                                  pf->pf_seid, pf->main_vsi_seid);
10505                 break;
10506         case I40E_SWITCH_ELEMENT_TYPE_PF:
10507         case I40E_SWITCH_ELEMENT_TYPE_VF:
10508         case I40E_SWITCH_ELEMENT_TYPE_EMP:
10509         case I40E_SWITCH_ELEMENT_TYPE_BMC:
10510         case I40E_SWITCH_ELEMENT_TYPE_PE:
10511         case I40E_SWITCH_ELEMENT_TYPE_PA:
10512                 /* ignore these for now */
10513                 break;
10514         default:
10515                 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
10516                          element_type, seid);
10517                 break;
10518         }
10519 }
10520
10521 /**
10522  * i40e_fetch_switch_configuration - Get switch config from firmware
10523  * @pf: board private structure
10524  * @printconfig: should we print the contents
10525  *
10526  * Get the current switch configuration from the device and
10527  * extract a few useful SEID values.
10528  **/
10529 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
10530 {
10531         struct i40e_aqc_get_switch_config_resp *sw_config;
10532         u16 next_seid = 0;
10533         int ret = 0;
10534         u8 *aq_buf;
10535         int i;
10536
10537         aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
10538         if (!aq_buf)
10539                 return -ENOMEM;
10540
10541         sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
10542         do {
10543                 u16 num_reported, num_total;
10544
10545                 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
10546                                                 I40E_AQ_LARGE_BUF,
10547                                                 &next_seid, NULL);
10548                 if (ret) {
10549                         dev_info(&pf->pdev->dev,
10550                                  "get switch config failed err %s aq_err %s\n",
10551                                  i40e_stat_str(&pf->hw, ret),
10552                                  i40e_aq_str(&pf->hw,
10553                                              pf->hw.aq.asq_last_status));
10554                         kfree(aq_buf);
10555                         return -ENOENT;
10556                 }
10557
10558                 num_reported = le16_to_cpu(sw_config->header.num_reported);
10559                 num_total = le16_to_cpu(sw_config->header.num_total);
10560
10561                 if (printconfig)
10562                         dev_info(&pf->pdev->dev,
10563                                  "header: %d reported %d total\n",
10564                                  num_reported, num_total);
10565
10566                 for (i = 0; i < num_reported; i++) {
10567                         struct i40e_aqc_switch_config_element_resp *ele =
10568                                 &sw_config->element[i];
10569
10570                         i40e_setup_pf_switch_element(pf, ele, num_reported,
10571                                                      printconfig);
10572                 }
10573         } while (next_seid != 0);
10574
10575         kfree(aq_buf);
10576         return ret;
10577 }
10578
10579 /**
10580  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
10581  * @pf: board private structure
10582  * @reinit: if the Main VSI needs to re-initialized.
10583  *
10584  * Returns 0 on success, negative value on failure
10585  **/
10586 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
10587 {
10588         u16 flags = 0;
10589         int ret;
10590
10591         /* find out what's out there already */
10592         ret = i40e_fetch_switch_configuration(pf, false);
10593         if (ret) {
10594                 dev_info(&pf->pdev->dev,
10595                          "couldn't fetch switch config, err %s aq_err %s\n",
10596                          i40e_stat_str(&pf->hw, ret),
10597                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10598                 return ret;
10599         }
10600         i40e_pf_reset_stats(pf);
10601
10602         /* set the switch config bit for the whole device to
10603          * support limited promisc or true promisc
10604          * when user requests promisc. The default is limited
10605          * promisc.
10606         */
10607
10608         if ((pf->hw.pf_id == 0) &&
10609             !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT))
10610                 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10611
10612         if (pf->hw.pf_id == 0) {
10613                 u16 valid_flags;
10614
10615                 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
10616                 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags,
10617                                                 NULL);
10618                 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
10619                         dev_info(&pf->pdev->dev,
10620                                  "couldn't set switch config bits, err %s aq_err %s\n",
10621                                  i40e_stat_str(&pf->hw, ret),
10622                                  i40e_aq_str(&pf->hw,
10623                                              pf->hw.aq.asq_last_status));
10624                         /* not a fatal problem, just keep going */
10625                 }
10626         }
10627
10628         /* first time setup */
10629         if (pf->lan_vsi == I40E_NO_VSI || reinit) {
10630                 struct i40e_vsi *vsi = NULL;
10631                 u16 uplink_seid;
10632
10633                 /* Set up the PF VSI associated with the PF's main VSI
10634                  * that is already in the HW switch
10635                  */
10636                 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb])
10637                         uplink_seid = pf->veb[pf->lan_veb]->seid;
10638                 else
10639                         uplink_seid = pf->mac_seid;
10640                 if (pf->lan_vsi == I40E_NO_VSI)
10641                         vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
10642                 else if (reinit)
10643                         vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
10644                 if (!vsi) {
10645                         dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
10646                         i40e_fdir_teardown(pf);
10647                         return -EAGAIN;
10648                 }
10649         } else {
10650                 /* force a reset of TC and queue layout configurations */
10651                 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
10652
10653                 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
10654                 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
10655                 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
10656         }
10657         i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
10658
10659         i40e_fdir_sb_setup(pf);
10660
10661         /* Setup static PF queue filter control settings */
10662         ret = i40e_setup_pf_filter_control(pf);
10663         if (ret) {
10664                 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
10665                          ret);
10666                 /* Failure here should not stop continuing other steps */
10667         }
10668
10669         /* enable RSS in the HW, even for only one queue, as the stack can use
10670          * the hash
10671          */
10672         if ((pf->flags & I40E_FLAG_RSS_ENABLED))
10673                 i40e_pf_config_rss(pf);
10674
10675         /* fill in link information and enable LSE reporting */
10676         i40e_update_link_info(&pf->hw);
10677         i40e_link_event(pf);
10678
10679         /* Initialize user-specific link properties */
10680         pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
10681                                   I40E_AQ_AN_COMPLETED) ? true : false);
10682
10683         i40e_ptp_init(pf);
10684
10685         return ret;
10686 }
10687
10688 /**
10689  * i40e_determine_queue_usage - Work out queue distribution
10690  * @pf: board private structure
10691  **/
10692 static void i40e_determine_queue_usage(struct i40e_pf *pf)
10693 {
10694         int queues_left;
10695
10696         pf->num_lan_qps = 0;
10697 #ifdef I40E_FCOE
10698         pf->num_fcoe_qps = 0;
10699 #endif
10700
10701         /* Find the max queues to be put into basic use.  We'll always be
10702          * using TC0, whether or not DCB is running, and TC0 will get the
10703          * big RSS set.
10704          */
10705         queues_left = pf->hw.func_caps.num_tx_qp;
10706
10707         if ((queues_left == 1) ||
10708             !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
10709                 /* one qp for PF, no queues for anything else */
10710                 queues_left = 0;
10711                 pf->alloc_rss_size = pf->num_lan_qps = 1;
10712
10713                 /* make sure all the fancies are disabled */
10714                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
10715                                I40E_FLAG_IWARP_ENABLED  |
10716 #ifdef I40E_FCOE
10717                                I40E_FLAG_FCOE_ENABLED   |
10718 #endif
10719                                I40E_FLAG_FD_SB_ENABLED  |
10720                                I40E_FLAG_FD_ATR_ENABLED |
10721                                I40E_FLAG_DCB_CAPABLE    |
10722                                I40E_FLAG_DCB_ENABLED    |
10723                                I40E_FLAG_SRIOV_ENABLED  |
10724                                I40E_FLAG_VMDQ_ENABLED);
10725         } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
10726                                   I40E_FLAG_FD_SB_ENABLED |
10727                                   I40E_FLAG_FD_ATR_ENABLED |
10728                                   I40E_FLAG_DCB_CAPABLE))) {
10729                 /* one qp for PF */
10730                 pf->alloc_rss_size = pf->num_lan_qps = 1;
10731                 queues_left -= pf->num_lan_qps;
10732
10733                 pf->flags &= ~(I40E_FLAG_RSS_ENABLED    |
10734                                I40E_FLAG_IWARP_ENABLED  |
10735 #ifdef I40E_FCOE
10736                                I40E_FLAG_FCOE_ENABLED   |
10737 #endif
10738                                I40E_FLAG_FD_SB_ENABLED  |
10739                                I40E_FLAG_FD_ATR_ENABLED |
10740                                I40E_FLAG_DCB_ENABLED    |
10741                                I40E_FLAG_VMDQ_ENABLED);
10742         } else {
10743                 /* Not enough queues for all TCs */
10744                 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
10745                     (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
10746                         pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
10747                                         I40E_FLAG_DCB_ENABLED);
10748                         dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
10749                 }
10750                 pf->num_lan_qps = max_t(int, pf->rss_size_max,
10751                                         num_online_cpus());
10752                 pf->num_lan_qps = min_t(int, pf->num_lan_qps,
10753                                         pf->hw.func_caps.num_tx_qp);
10754
10755                 queues_left -= pf->num_lan_qps;
10756         }
10757
10758 #ifdef I40E_FCOE
10759         if (pf->flags & I40E_FLAG_FCOE_ENABLED) {
10760                 if (I40E_DEFAULT_FCOE <= queues_left) {
10761                         pf->num_fcoe_qps = I40E_DEFAULT_FCOE;
10762                 } else if (I40E_MINIMUM_FCOE <= queues_left) {
10763                         pf->num_fcoe_qps = I40E_MINIMUM_FCOE;
10764                 } else {
10765                         pf->num_fcoe_qps = 0;
10766                         pf->flags &= ~I40E_FLAG_FCOE_ENABLED;
10767                         dev_info(&pf->pdev->dev, "not enough queues for FCoE. FCoE feature will be disabled\n");
10768                 }
10769
10770                 queues_left -= pf->num_fcoe_qps;
10771         }
10772
10773 #endif
10774         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10775                 if (queues_left > 1) {
10776                         queues_left -= 1; /* save 1 queue for FD */
10777                 } else {
10778                         pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10779                         dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
10780                 }
10781         }
10782
10783         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
10784             pf->num_vf_qps && pf->num_req_vfs && queues_left) {
10785                 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
10786                                         (queues_left / pf->num_vf_qps));
10787                 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
10788         }
10789
10790         if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
10791             pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
10792                 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
10793                                           (queues_left / pf->num_vmdq_qps));
10794                 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
10795         }
10796
10797         pf->queues_left = queues_left;
10798         dev_dbg(&pf->pdev->dev,
10799                 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
10800                 pf->hw.func_caps.num_tx_qp,
10801                 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
10802                 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
10803                 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
10804                 queues_left);
10805 #ifdef I40E_FCOE
10806         dev_dbg(&pf->pdev->dev, "fcoe queues = %d\n", pf->num_fcoe_qps);
10807 #endif
10808 }
10809
10810 /**
10811  * i40e_setup_pf_filter_control - Setup PF static filter control
10812  * @pf: PF to be setup
10813  *
10814  * i40e_setup_pf_filter_control sets up a PF's initial filter control
10815  * settings. If PE/FCoE are enabled then it will also set the per PF
10816  * based filter sizes required for them. It also enables Flow director,
10817  * ethertype and macvlan type filter settings for the pf.
10818  *
10819  * Returns 0 on success, negative on failure
10820  **/
10821 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
10822 {
10823         struct i40e_filter_control_settings *settings = &pf->filter_settings;
10824
10825         settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
10826
10827         /* Flow Director is enabled */
10828         if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
10829                 settings->enable_fdir = true;
10830
10831         /* Ethtype and MACVLAN filters enabled for PF */
10832         settings->enable_ethtype = true;
10833         settings->enable_macvlan = true;
10834
10835         if (i40e_set_filter_control(&pf->hw, settings))
10836                 return -ENOENT;
10837
10838         return 0;
10839 }
10840
10841 #define INFO_STRING_LEN 255
10842 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
10843 static void i40e_print_features(struct i40e_pf *pf)
10844 {
10845         struct i40e_hw *hw = &pf->hw;
10846         char *buf;
10847         int i;
10848
10849         buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
10850         if (!buf)
10851                 return;
10852
10853         i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
10854 #ifdef CONFIG_PCI_IOV
10855         i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
10856 #endif
10857         i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
10858                       pf->hw.func_caps.num_vsis,
10859                       pf->vsi[pf->lan_vsi]->num_queue_pairs);
10860         if (pf->flags & I40E_FLAG_RSS_ENABLED)
10861                 i += snprintf(&buf[i], REMAIN(i), " RSS");
10862         if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
10863                 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
10864         if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10865                 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
10866                 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
10867         }
10868         if (pf->flags & I40E_FLAG_DCB_CAPABLE)
10869                 i += snprintf(&buf[i], REMAIN(i), " DCB");
10870         i += snprintf(&buf[i], REMAIN(i), " VxLAN");
10871         i += snprintf(&buf[i], REMAIN(i), " Geneve");
10872         if (pf->flags & I40E_FLAG_PTP)
10873                 i += snprintf(&buf[i], REMAIN(i), " PTP");
10874 #ifdef I40E_FCOE
10875         if (pf->flags & I40E_FLAG_FCOE_ENABLED)
10876                 i += snprintf(&buf[i], REMAIN(i), " FCOE");
10877 #endif
10878         if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
10879                 i += snprintf(&buf[i], REMAIN(i), " VEB");
10880         else
10881                 i += snprintf(&buf[i], REMAIN(i), " VEPA");
10882
10883         dev_info(&pf->pdev->dev, "%s\n", buf);
10884         kfree(buf);
10885         WARN_ON(i > INFO_STRING_LEN);
10886 }
10887
10888 /**
10889  * i40e_get_platform_mac_addr - get platform-specific MAC address
10890  *
10891  * @pdev: PCI device information struct
10892  * @pf: board private structure
10893  *
10894  * Look up the MAC address in Open Firmware  on systems that support it,
10895  * and use IDPROM on SPARC if no OF address is found. On return, the
10896  * I40E_FLAG_PF_MAC will be wset in pf->flags if a platform-specific value
10897  * has been selected.
10898  **/
10899 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
10900 {
10901         pf->flags &= ~I40E_FLAG_PF_MAC;
10902         if (!eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
10903                 pf->flags |= I40E_FLAG_PF_MAC;
10904 }
10905
10906 /**
10907  * i40e_probe - Device initialization routine
10908  * @pdev: PCI device information struct
10909  * @ent: entry in i40e_pci_tbl
10910  *
10911  * i40e_probe initializes a PF identified by a pci_dev structure.
10912  * The OS initialization, configuring of the PF private structure,
10913  * and a hardware reset occur.
10914  *
10915  * Returns 0 on success, negative on failure
10916  **/
10917 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
10918 {
10919         struct i40e_aq_get_phy_abilities_resp abilities;
10920         struct i40e_pf *pf;
10921         struct i40e_hw *hw;
10922         static u16 pfs_found;
10923         u16 wol_nvm_bits;
10924         u16 link_status;
10925         int err;
10926         u32 val;
10927         u32 i;
10928         u8 set_fc_aq_fail;
10929
10930         err = pci_enable_device_mem(pdev);
10931         if (err)
10932                 return err;
10933
10934         /* set up for high or low dma */
10935         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
10936         if (err) {
10937                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
10938                 if (err) {
10939                         dev_err(&pdev->dev,
10940                                 "DMA configuration failed: 0x%x\n", err);
10941                         goto err_dma;
10942                 }
10943         }
10944
10945         /* set up pci connections */
10946         err = pci_request_mem_regions(pdev, i40e_driver_name);
10947         if (err) {
10948                 dev_info(&pdev->dev,
10949                          "pci_request_selected_regions failed %d\n", err);
10950                 goto err_pci_reg;
10951         }
10952
10953         pci_enable_pcie_error_reporting(pdev);
10954         pci_set_master(pdev);
10955
10956         /* Now that we have a PCI connection, we need to do the
10957          * low level device setup.  This is primarily setting up
10958          * the Admin Queue structures and then querying for the
10959          * device's current profile information.
10960          */
10961         pf = kzalloc(sizeof(*pf), GFP_KERNEL);
10962         if (!pf) {
10963                 err = -ENOMEM;
10964                 goto err_pf_alloc;
10965         }
10966         pf->next_vsi = 0;
10967         pf->pdev = pdev;
10968         set_bit(__I40E_DOWN, &pf->state);
10969
10970         hw = &pf->hw;
10971         hw->back = pf;
10972
10973         pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
10974                                 I40E_MAX_CSR_SPACE);
10975
10976         hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
10977         if (!hw->hw_addr) {
10978                 err = -EIO;
10979                 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
10980                          (unsigned int)pci_resource_start(pdev, 0),
10981                          pf->ioremap_len, err);
10982                 goto err_ioremap;
10983         }
10984         hw->vendor_id = pdev->vendor;
10985         hw->device_id = pdev->device;
10986         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
10987         hw->subsystem_vendor_id = pdev->subsystem_vendor;
10988         hw->subsystem_device_id = pdev->subsystem_device;
10989         hw->bus.device = PCI_SLOT(pdev->devfn);
10990         hw->bus.func = PCI_FUNC(pdev->devfn);
10991         pf->instance = pfs_found;
10992
10993         /* set up the locks for the AQ, do this only once in probe
10994          * and destroy them only once in remove
10995          */
10996         mutex_init(&hw->aq.asq_mutex);
10997         mutex_init(&hw->aq.arq_mutex);
10998
10999         pf->msg_enable = netif_msg_init(debug,
11000                                         NETIF_MSG_DRV |
11001                                         NETIF_MSG_PROBE |
11002                                         NETIF_MSG_LINK);
11003         if (debug < -1)
11004                 pf->hw.debug_mask = debug;
11005
11006         /* do a special CORER for clearing PXE mode once at init */
11007         if (hw->revision_id == 0 &&
11008             (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
11009                 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
11010                 i40e_flush(hw);
11011                 msleep(200);
11012                 pf->corer_count++;
11013
11014                 i40e_clear_pxe_mode(hw);
11015         }
11016
11017         /* Reset here to make sure all is clean and to define PF 'n' */
11018         i40e_clear_hw(hw);
11019         err = i40e_pf_reset(hw);
11020         if (err) {
11021                 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
11022                 goto err_pf_reset;
11023         }
11024         pf->pfr_count++;
11025
11026         hw->aq.num_arq_entries = I40E_AQ_LEN;
11027         hw->aq.num_asq_entries = I40E_AQ_LEN;
11028         hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11029         hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
11030         pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
11031
11032         snprintf(pf->int_name, sizeof(pf->int_name) - 1,
11033                  "%s-%s:misc",
11034                  dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
11035
11036         err = i40e_init_shared_code(hw);
11037         if (err) {
11038                 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
11039                          err);
11040                 goto err_pf_reset;
11041         }
11042
11043         /* set up a default setting for link flow control */
11044         pf->hw.fc.requested_mode = I40E_FC_NONE;
11045
11046         err = i40e_init_adminq(hw);
11047         if (err) {
11048                 if (err == I40E_ERR_FIRMWARE_API_VERSION)
11049                         dev_info(&pdev->dev,
11050                                  "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n");
11051                 else
11052                         dev_info(&pdev->dev,
11053                                  "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
11054
11055                 goto err_pf_reset;
11056         }
11057
11058         /* provide nvm, fw, api versions */
11059         dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n",
11060                  hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
11061                  hw->aq.api_maj_ver, hw->aq.api_min_ver,
11062                  i40e_nvm_version_str(hw));
11063
11064         if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
11065             hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR)
11066                 dev_info(&pdev->dev,
11067                          "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n");
11068         else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR ||
11069                  hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1))
11070                 dev_info(&pdev->dev,
11071                          "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n");
11072
11073         i40e_verify_eeprom(pf);
11074
11075         /* Rev 0 hardware was never productized */
11076         if (hw->revision_id < 1)
11077                 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
11078
11079         i40e_clear_pxe_mode(hw);
11080         err = i40e_get_capabilities(pf);
11081         if (err)
11082                 goto err_adminq_setup;
11083
11084         err = i40e_sw_init(pf);
11085         if (err) {
11086                 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
11087                 goto err_sw_init;
11088         }
11089
11090         err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
11091                                 hw->func_caps.num_rx_qp,
11092                                 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num);
11093         if (err) {
11094                 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
11095                 goto err_init_lan_hmc;
11096         }
11097
11098         err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
11099         if (err) {
11100                 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
11101                 err = -ENOENT;
11102                 goto err_configure_lan_hmc;
11103         }
11104
11105         /* Disable LLDP for NICs that have firmware versions lower than v4.3.
11106          * Ignore error return codes because if it was already disabled via
11107          * hardware settings this will fail
11108          */
11109         if (pf->flags & I40E_FLAG_STOP_FW_LLDP) {
11110                 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
11111                 i40e_aq_stop_lldp(hw, true, NULL);
11112         }
11113
11114         i40e_get_mac_addr(hw, hw->mac.addr);
11115         /* allow a platform config to override the HW addr */
11116         i40e_get_platform_mac_addr(pdev, pf);
11117         if (!is_valid_ether_addr(hw->mac.addr)) {
11118                 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
11119                 err = -EIO;
11120                 goto err_mac_addr;
11121         }
11122         dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
11123         ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
11124         i40e_get_port_mac_addr(hw, hw->mac.port_addr);
11125         if (is_valid_ether_addr(hw->mac.port_addr))
11126                 pf->flags |= I40E_FLAG_PORT_ID_VALID;
11127 #ifdef I40E_FCOE
11128         err = i40e_get_san_mac_addr(hw, hw->mac.san_addr);
11129         if (err)
11130                 dev_info(&pdev->dev,
11131                          "(non-fatal) SAN MAC retrieval failed: %d\n", err);
11132         if (!is_valid_ether_addr(hw->mac.san_addr)) {
11133                 dev_warn(&pdev->dev, "invalid SAN MAC address %pM, falling back to LAN MAC\n",
11134                          hw->mac.san_addr);
11135                 ether_addr_copy(hw->mac.san_addr, hw->mac.addr);
11136         }
11137         dev_info(&pf->pdev->dev, "SAN MAC: %pM\n", hw->mac.san_addr);
11138 #endif /* I40E_FCOE */
11139
11140         pci_set_drvdata(pdev, pf);
11141         pci_save_state(pdev);
11142 #ifdef CONFIG_I40E_DCB
11143         err = i40e_init_pf_dcb(pf);
11144         if (err) {
11145                 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
11146                 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
11147                 /* Continue without DCB enabled */
11148         }
11149 #endif /* CONFIG_I40E_DCB */
11150
11151         /* set up periodic task facility */
11152         setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf);
11153         pf->service_timer_period = HZ;
11154
11155         INIT_WORK(&pf->service_task, i40e_service_task);
11156         clear_bit(__I40E_SERVICE_SCHED, &pf->state);
11157         pf->flags |= I40E_FLAG_NEED_LINK_UPDATE;
11158
11159         /* NVM bit on means WoL disabled for the port */
11160         i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
11161         if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
11162                 pf->wol_en = false;
11163         else
11164                 pf->wol_en = true;
11165         device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
11166
11167         /* set up the main switch operations */
11168         i40e_determine_queue_usage(pf);
11169         err = i40e_init_interrupt_scheme(pf);
11170         if (err)
11171                 goto err_switch_setup;
11172
11173         /* The number of VSIs reported by the FW is the minimum guaranteed
11174          * to us; HW supports far more and we share the remaining pool with
11175          * the other PFs. We allocate space for more than the guarantee with
11176          * the understanding that we might not get them all later.
11177          */
11178         if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
11179                 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
11180         else
11181                 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
11182
11183         /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
11184         pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
11185                           GFP_KERNEL);
11186         if (!pf->vsi) {
11187                 err = -ENOMEM;
11188                 goto err_switch_setup;
11189         }
11190
11191 #ifdef CONFIG_PCI_IOV
11192         /* prep for VF support */
11193         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11194             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11195             !test_bit(__I40E_BAD_EEPROM, &pf->state)) {
11196                 if (pci_num_vf(pdev))
11197                         pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
11198         }
11199 #endif
11200         err = i40e_setup_pf_switch(pf, false);
11201         if (err) {
11202                 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
11203                 goto err_vsis;
11204         }
11205
11206         /* Make sure flow control is set according to current settings */
11207         err = i40e_set_fc(hw, &set_fc_aq_fail, true);
11208         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
11209                 dev_dbg(&pf->pdev->dev,
11210                         "Set fc with err %s aq_err %s on get_phy_cap\n",
11211                         i40e_stat_str(hw, err),
11212                         i40e_aq_str(hw, hw->aq.asq_last_status));
11213         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
11214                 dev_dbg(&pf->pdev->dev,
11215                         "Set fc with err %s aq_err %s on set_phy_config\n",
11216                         i40e_stat_str(hw, err),
11217                         i40e_aq_str(hw, hw->aq.asq_last_status));
11218         if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
11219                 dev_dbg(&pf->pdev->dev,
11220                         "Set fc with err %s aq_err %s on get_link_info\n",
11221                         i40e_stat_str(hw, err),
11222                         i40e_aq_str(hw, hw->aq.asq_last_status));
11223
11224         /* if FDIR VSI was set up, start it now */
11225         for (i = 0; i < pf->num_alloc_vsi; i++) {
11226                 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
11227                         i40e_vsi_open(pf->vsi[i]);
11228                         break;
11229                 }
11230         }
11231
11232         /* The driver only wants link up/down and module qualification
11233          * reports from firmware.  Note the negative logic.
11234          */
11235         err = i40e_aq_set_phy_int_mask(&pf->hw,
11236                                        ~(I40E_AQ_EVENT_LINK_UPDOWN |
11237                                          I40E_AQ_EVENT_MEDIA_NA |
11238                                          I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
11239         if (err)
11240                 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
11241                          i40e_stat_str(&pf->hw, err),
11242                          i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11243
11244         /* Reconfigure hardware for allowing smaller MSS in the case
11245          * of TSO, so that we avoid the MDD being fired and causing
11246          * a reset in the case of small MSS+TSO.
11247          */
11248         val = rd32(hw, I40E_REG_MSS);
11249         if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11250                 val &= ~I40E_REG_MSS_MIN_MASK;
11251                 val |= I40E_64BYTE_MSS;
11252                 wr32(hw, I40E_REG_MSS, val);
11253         }
11254
11255         if (pf->flags & I40E_FLAG_RESTART_AUTONEG) {
11256                 msleep(75);
11257                 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11258                 if (err)
11259                         dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
11260                                  i40e_stat_str(&pf->hw, err),
11261                                  i40e_aq_str(&pf->hw,
11262                                              pf->hw.aq.asq_last_status));
11263         }
11264         /* The main driver is (mostly) up and happy. We need to set this state
11265          * before setting up the misc vector or we get a race and the vector
11266          * ends up disabled forever.
11267          */
11268         clear_bit(__I40E_DOWN, &pf->state);
11269
11270         /* In case of MSIX we are going to setup the misc vector right here
11271          * to handle admin queue events etc. In case of legacy and MSI
11272          * the misc functionality and queue processing is combined in
11273          * the same vector and that gets setup at open.
11274          */
11275         if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11276                 err = i40e_setup_misc_vector(pf);
11277                 if (err) {
11278                         dev_info(&pdev->dev,
11279                                  "setup of misc vector failed: %d\n", err);
11280                         goto err_vsis;
11281                 }
11282         }
11283
11284 #ifdef CONFIG_PCI_IOV
11285         /* prep for VF support */
11286         if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
11287             (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11288             !test_bit(__I40E_BAD_EEPROM, &pf->state)) {
11289                 /* disable link interrupts for VFs */
11290                 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
11291                 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
11292                 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
11293                 i40e_flush(hw);
11294
11295                 if (pci_num_vf(pdev)) {
11296                         dev_info(&pdev->dev,
11297                                  "Active VFs found, allocating resources.\n");
11298                         err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
11299                         if (err)
11300                                 dev_info(&pdev->dev,
11301                                          "Error %d allocating resources for existing VFs\n",
11302                                          err);
11303                 }
11304         }
11305 #endif /* CONFIG_PCI_IOV */
11306
11307         if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11308                 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
11309                                                       pf->num_iwarp_msix,
11310                                                       I40E_IWARP_IRQ_PILE_ID);
11311                 if (pf->iwarp_base_vector < 0) {
11312                         dev_info(&pdev->dev,
11313                                  "failed to get tracking for %d vectors for IWARP err=%d\n",
11314                                  pf->num_iwarp_msix, pf->iwarp_base_vector);
11315                         pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11316                 }
11317         }
11318
11319         i40e_dbg_pf_init(pf);
11320
11321         /* tell the firmware that we're starting */
11322         i40e_send_version(pf);
11323
11324         /* since everything's happy, start the service_task timer */
11325         mod_timer(&pf->service_timer,
11326                   round_jiffies(jiffies + pf->service_timer_period));
11327
11328         /* add this PF to client device list and launch a client service task */
11329         err = i40e_lan_add_device(pf);
11330         if (err)
11331                 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
11332                          err);
11333
11334 #ifdef I40E_FCOE
11335         /* create FCoE interface */
11336         i40e_fcoe_vsi_setup(pf);
11337
11338 #endif
11339 #define PCI_SPEED_SIZE 8
11340 #define PCI_WIDTH_SIZE 8
11341         /* Devices on the IOSF bus do not have this information
11342          * and will report PCI Gen 1 x 1 by default so don't bother
11343          * checking them.
11344          */
11345         if (!(pf->flags & I40E_FLAG_NO_PCI_LINK_CHECK)) {
11346                 char speed[PCI_SPEED_SIZE] = "Unknown";
11347                 char width[PCI_WIDTH_SIZE] = "Unknown";
11348
11349                 /* Get the negotiated link width and speed from PCI config
11350                  * space
11351                  */
11352                 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
11353                                           &link_status);
11354
11355                 i40e_set_pci_config_data(hw, link_status);
11356
11357                 switch (hw->bus.speed) {
11358                 case i40e_bus_speed_8000:
11359                         strncpy(speed, "8.0", PCI_SPEED_SIZE); break;
11360                 case i40e_bus_speed_5000:
11361                         strncpy(speed, "5.0", PCI_SPEED_SIZE); break;
11362                 case i40e_bus_speed_2500:
11363                         strncpy(speed, "2.5", PCI_SPEED_SIZE); break;
11364                 default:
11365                         break;
11366                 }
11367                 switch (hw->bus.width) {
11368                 case i40e_bus_width_pcie_x8:
11369                         strncpy(width, "8", PCI_WIDTH_SIZE); break;
11370                 case i40e_bus_width_pcie_x4:
11371                         strncpy(width, "4", PCI_WIDTH_SIZE); break;
11372                 case i40e_bus_width_pcie_x2:
11373                         strncpy(width, "2", PCI_WIDTH_SIZE); break;
11374                 case i40e_bus_width_pcie_x1:
11375                         strncpy(width, "1", PCI_WIDTH_SIZE); break;
11376                 default:
11377                         break;
11378                 }
11379
11380                 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
11381                          speed, width);
11382
11383                 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
11384                     hw->bus.speed < i40e_bus_speed_8000) {
11385                         dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
11386                         dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
11387                 }
11388         }
11389
11390         /* get the requested speeds from the fw */
11391         err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
11392         if (err)
11393                 dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %s last_status =  %s\n",
11394                         i40e_stat_str(&pf->hw, err),
11395                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11396         pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
11397
11398         /* get the supported phy types from the fw */
11399         err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
11400         if (err)
11401                 dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %s last_status =  %s\n",
11402                         i40e_stat_str(&pf->hw, err),
11403                         i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11404
11405         /* Add a filter to drop all Flow control frames from any VSI from being
11406          * transmitted. By doing so we stop a malicious VF from sending out
11407          * PAUSE or PFC frames and potentially controlling traffic for other
11408          * PF/VF VSIs.
11409          * The FW can still send Flow control frames if enabled.
11410          */
11411         i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11412                                                        pf->main_vsi_seid);
11413
11414         if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
11415                 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
11416                 pf->flags |= I40E_FLAG_PHY_CONTROLS_LEDS;
11417         if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
11418                 pf->flags |= I40E_FLAG_HAVE_CRT_RETIMER;
11419         /* print a string summarizing features */
11420         i40e_print_features(pf);
11421
11422         return 0;
11423
11424         /* Unwind what we've done if something failed in the setup */
11425 err_vsis:
11426         set_bit(__I40E_DOWN, &pf->state);
11427         i40e_clear_interrupt_scheme(pf);
11428         kfree(pf->vsi);
11429 err_switch_setup:
11430         i40e_reset_interrupt_capability(pf);
11431         del_timer_sync(&pf->service_timer);
11432 err_mac_addr:
11433 err_configure_lan_hmc:
11434         (void)i40e_shutdown_lan_hmc(hw);
11435 err_init_lan_hmc:
11436         kfree(pf->qp_pile);
11437 err_sw_init:
11438 err_adminq_setup:
11439 err_pf_reset:
11440         iounmap(hw->hw_addr);
11441 err_ioremap:
11442         kfree(pf);
11443 err_pf_alloc:
11444         pci_disable_pcie_error_reporting(pdev);
11445         pci_release_mem_regions(pdev);
11446 err_pci_reg:
11447 err_dma:
11448         pci_disable_device(pdev);
11449         return err;
11450 }
11451
11452 /**
11453  * i40e_remove - Device removal routine
11454  * @pdev: PCI device information struct
11455  *
11456  * i40e_remove is called by the PCI subsystem to alert the driver
11457  * that is should release a PCI device.  This could be caused by a
11458  * Hot-Plug event, or because the driver is going to be removed from
11459  * memory.
11460  **/
11461 static void i40e_remove(struct pci_dev *pdev)
11462 {
11463         struct i40e_pf *pf = pci_get_drvdata(pdev);
11464         struct i40e_hw *hw = &pf->hw;
11465         i40e_status ret_code;
11466         int i;
11467
11468         i40e_dbg_pf_exit(pf);
11469
11470         i40e_ptp_stop(pf);
11471
11472         /* Disable RSS in hw */
11473         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
11474         i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
11475
11476         /* no more scheduling of any task */
11477         set_bit(__I40E_SUSPENDED, &pf->state);
11478         set_bit(__I40E_DOWN, &pf->state);
11479         if (pf->service_timer.data)
11480                 del_timer_sync(&pf->service_timer);
11481         if (pf->service_task.func)
11482                 cancel_work_sync(&pf->service_task);
11483
11484         if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
11485                 i40e_free_vfs(pf);
11486                 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
11487         }
11488
11489         i40e_fdir_teardown(pf);
11490
11491         /* If there is a switch structure or any orphans, remove them.
11492          * This will leave only the PF's VSI remaining.
11493          */
11494         for (i = 0; i < I40E_MAX_VEB; i++) {
11495                 if (!pf->veb[i])
11496                         continue;
11497
11498                 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
11499                     pf->veb[i]->uplink_seid == 0)
11500                         i40e_switch_branch_release(pf->veb[i]);
11501         }
11502
11503         /* Now we can shutdown the PF's VSI, just before we kill
11504          * adminq and hmc.
11505          */
11506         if (pf->vsi[pf->lan_vsi])
11507                 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
11508
11509         /* remove attached clients */
11510         ret_code = i40e_lan_del_device(pf);
11511         if (ret_code) {
11512                 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
11513                          ret_code);
11514         }
11515
11516         /* shutdown and destroy the HMC */
11517         if (hw->hmc.hmc_obj) {
11518                 ret_code = i40e_shutdown_lan_hmc(hw);
11519                 if (ret_code)
11520                         dev_warn(&pdev->dev,
11521                                  "Failed to destroy the HMC resources: %d\n",
11522                                  ret_code);
11523         }
11524
11525         /* shutdown the adminq */
11526         i40e_shutdown_adminq(hw);
11527
11528         /* destroy the locks only once, here */
11529         mutex_destroy(&hw->aq.arq_mutex);
11530         mutex_destroy(&hw->aq.asq_mutex);
11531
11532         /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
11533         i40e_clear_interrupt_scheme(pf);
11534         for (i = 0; i < pf->num_alloc_vsi; i++) {
11535                 if (pf->vsi[i]) {
11536                         i40e_vsi_clear_rings(pf->vsi[i]);
11537                         i40e_vsi_clear(pf->vsi[i]);
11538                         pf->vsi[i] = NULL;
11539                 }
11540         }
11541
11542         for (i = 0; i < I40E_MAX_VEB; i++) {
11543                 kfree(pf->veb[i]);
11544                 pf->veb[i] = NULL;
11545         }
11546
11547         kfree(pf->qp_pile);
11548         kfree(pf->vsi);
11549
11550         iounmap(hw->hw_addr);
11551         kfree(pf);
11552         pci_release_mem_regions(pdev);
11553
11554         pci_disable_pcie_error_reporting(pdev);
11555         pci_disable_device(pdev);
11556 }
11557
11558 /**
11559  * i40e_pci_error_detected - warning that something funky happened in PCI land
11560  * @pdev: PCI device information struct
11561  *
11562  * Called to warn that something happened and the error handling steps
11563  * are in progress.  Allows the driver to quiesce things, be ready for
11564  * remediation.
11565  **/
11566 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
11567                                                 enum pci_channel_state error)
11568 {
11569         struct i40e_pf *pf = pci_get_drvdata(pdev);
11570
11571         dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
11572
11573         if (!pf) {
11574                 dev_info(&pdev->dev,
11575                          "Cannot recover - error happened during device probe\n");
11576                 return PCI_ERS_RESULT_DISCONNECT;
11577         }
11578
11579         /* shutdown all operations */
11580         if (!test_bit(__I40E_SUSPENDED, &pf->state)) {
11581                 rtnl_lock();
11582                 i40e_prep_for_reset(pf);
11583                 rtnl_unlock();
11584         }
11585
11586         /* Request a slot reset */
11587         return PCI_ERS_RESULT_NEED_RESET;
11588 }
11589
11590 /**
11591  * i40e_pci_error_slot_reset - a PCI slot reset just happened
11592  * @pdev: PCI device information struct
11593  *
11594  * Called to find if the driver can work with the device now that
11595  * the pci slot has been reset.  If a basic connection seems good
11596  * (registers are readable and have sane content) then return a
11597  * happy little PCI_ERS_RESULT_xxx.
11598  **/
11599 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
11600 {
11601         struct i40e_pf *pf = pci_get_drvdata(pdev);
11602         pci_ers_result_t result;
11603         int err;
11604         u32 reg;
11605
11606         dev_dbg(&pdev->dev, "%s\n", __func__);
11607         if (pci_enable_device_mem(pdev)) {
11608                 dev_info(&pdev->dev,
11609                          "Cannot re-enable PCI device after reset.\n");
11610                 result = PCI_ERS_RESULT_DISCONNECT;
11611         } else {
11612                 pci_set_master(pdev);
11613                 pci_restore_state(pdev);
11614                 pci_save_state(pdev);
11615                 pci_wake_from_d3(pdev, false);
11616
11617                 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
11618                 if (reg == 0)
11619                         result = PCI_ERS_RESULT_RECOVERED;
11620                 else
11621                         result = PCI_ERS_RESULT_DISCONNECT;
11622         }
11623
11624         err = pci_cleanup_aer_uncorrect_error_status(pdev);
11625         if (err) {
11626                 dev_info(&pdev->dev,
11627                          "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n",
11628                          err);
11629                 /* non-fatal, continue */
11630         }
11631
11632         return result;
11633 }
11634
11635 /**
11636  * i40e_pci_error_resume - restart operations after PCI error recovery
11637  * @pdev: PCI device information struct
11638  *
11639  * Called to allow the driver to bring things back up after PCI error
11640  * and/or reset recovery has finished.
11641  **/
11642 static void i40e_pci_error_resume(struct pci_dev *pdev)
11643 {
11644         struct i40e_pf *pf = pci_get_drvdata(pdev);
11645
11646         dev_dbg(&pdev->dev, "%s\n", __func__);
11647         if (test_bit(__I40E_SUSPENDED, &pf->state))
11648                 return;
11649
11650         rtnl_lock();
11651         i40e_handle_reset_warning(pf);
11652         rtnl_unlock();
11653 }
11654
11655 /**
11656  * i40e_shutdown - PCI callback for shutting down
11657  * @pdev: PCI device information struct
11658  **/
11659 static void i40e_shutdown(struct pci_dev *pdev)
11660 {
11661         struct i40e_pf *pf = pci_get_drvdata(pdev);
11662         struct i40e_hw *hw = &pf->hw;
11663
11664         set_bit(__I40E_SUSPENDED, &pf->state);
11665         set_bit(__I40E_DOWN, &pf->state);
11666         rtnl_lock();
11667         i40e_prep_for_reset(pf);
11668         rtnl_unlock();
11669
11670         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11671         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11672
11673         del_timer_sync(&pf->service_timer);
11674         cancel_work_sync(&pf->service_task);
11675         i40e_fdir_teardown(pf);
11676
11677         rtnl_lock();
11678         i40e_prep_for_reset(pf);
11679         rtnl_unlock();
11680
11681         wr32(hw, I40E_PFPM_APM,
11682              (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11683         wr32(hw, I40E_PFPM_WUFC,
11684              (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11685
11686         i40e_clear_interrupt_scheme(pf);
11687
11688         if (system_state == SYSTEM_POWER_OFF) {
11689                 pci_wake_from_d3(pdev, pf->wol_en);
11690                 pci_set_power_state(pdev, PCI_D3hot);
11691         }
11692 }
11693
11694 #ifdef CONFIG_PM
11695 /**
11696  * i40e_suspend - PCI callback for moving to D3
11697  * @pdev: PCI device information struct
11698  **/
11699 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state)
11700 {
11701         struct i40e_pf *pf = pci_get_drvdata(pdev);
11702         struct i40e_hw *hw = &pf->hw;
11703         int retval = 0;
11704
11705         set_bit(__I40E_SUSPENDED, &pf->state);
11706         set_bit(__I40E_DOWN, &pf->state);
11707
11708         rtnl_lock();
11709         i40e_prep_for_reset(pf);
11710         rtnl_unlock();
11711
11712         wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
11713         wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
11714
11715         i40e_stop_misc_vector(pf);
11716
11717         retval = pci_save_state(pdev);
11718         if (retval)
11719                 return retval;
11720
11721         pci_wake_from_d3(pdev, pf->wol_en);
11722         pci_set_power_state(pdev, PCI_D3hot);
11723
11724         return retval;
11725 }
11726
11727 /**
11728  * i40e_resume - PCI callback for waking up from D3
11729  * @pdev: PCI device information struct
11730  **/
11731 static int i40e_resume(struct pci_dev *pdev)
11732 {
11733         struct i40e_pf *pf = pci_get_drvdata(pdev);
11734         u32 err;
11735
11736         pci_set_power_state(pdev, PCI_D0);
11737         pci_restore_state(pdev);
11738         /* pci_restore_state() clears dev->state_saves, so
11739          * call pci_save_state() again to restore it.
11740          */
11741         pci_save_state(pdev);
11742
11743         err = pci_enable_device_mem(pdev);
11744         if (err) {
11745                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
11746                 return err;
11747         }
11748         pci_set_master(pdev);
11749
11750         /* no wakeup events while running */
11751         pci_wake_from_d3(pdev, false);
11752
11753         /* handling the reset will rebuild the device state */
11754         if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) {
11755                 clear_bit(__I40E_DOWN, &pf->state);
11756                 rtnl_lock();
11757                 i40e_reset_and_rebuild(pf, false);
11758                 rtnl_unlock();
11759         }
11760
11761         return 0;
11762 }
11763
11764 #endif
11765 static const struct pci_error_handlers i40e_err_handler = {
11766         .error_detected = i40e_pci_error_detected,
11767         .slot_reset = i40e_pci_error_slot_reset,
11768         .resume = i40e_pci_error_resume,
11769 };
11770
11771 static struct pci_driver i40e_driver = {
11772         .name     = i40e_driver_name,
11773         .id_table = i40e_pci_tbl,
11774         .probe    = i40e_probe,
11775         .remove   = i40e_remove,
11776 #ifdef CONFIG_PM
11777         .suspend  = i40e_suspend,
11778         .resume   = i40e_resume,
11779 #endif
11780         .shutdown = i40e_shutdown,
11781         .err_handler = &i40e_err_handler,
11782         .sriov_configure = i40e_pci_sriov_configure,
11783 };
11784
11785 /**
11786  * i40e_init_module - Driver registration routine
11787  *
11788  * i40e_init_module is the first routine called when the driver is
11789  * loaded. All it does is register with the PCI subsystem.
11790  **/
11791 static int __init i40e_init_module(void)
11792 {
11793         pr_info("%s: %s - version %s\n", i40e_driver_name,
11794                 i40e_driver_string, i40e_driver_version_str);
11795         pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
11796
11797         /* we will see if single thread per module is enough for now,
11798          * it can't be any worse than using the system workqueue which
11799          * was already single threaded
11800          */
11801         i40e_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
11802                                   i40e_driver_name);
11803         if (!i40e_wq) {
11804                 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
11805                 return -ENOMEM;
11806         }
11807
11808         i40e_dbg_init();
11809         return pci_register_driver(&i40e_driver);
11810 }
11811 module_init(i40e_init_module);
11812
11813 /**
11814  * i40e_exit_module - Driver exit cleanup routine
11815  *
11816  * i40e_exit_module is called just before the driver is removed
11817  * from memory.
11818  **/
11819 static void __exit i40e_exit_module(void)
11820 {
11821         pci_unregister_driver(&i40e_driver);
11822         destroy_workqueue(i40e_wq);
11823         i40e_dbg_exit();
11824 }
11825 module_exit(i40e_exit_module);