drivers/net/ethernet/intel/iavf/iavf_main.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
   3
   4 #include "iavf.h"
   5 #include "iavf_prototype.h"
   6 #include "iavf_client.h"
   7 /* All iavf tracepoints are defined by the include below, which must
   8  * be included exactly once across the whole kernel with
   9  * CREATE_TRACE_POINTS defined
  10  */
  11 #define CREATE_TRACE_POINTS
  12 #include "iavf_trace.h"
  13
  14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
  15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
  16 static int iavf_close(struct net_device *netdev);
  17 static void iavf_init_get_resources(struct iavf_adapter *adapter);
  18 static int iavf_check_reset_complete(struct iavf_hw *hw);
  19
  20 char iavf_driver_name[] = "iavf";
  21 static const char iavf_driver_string[] =
  22         "Intel(R) Ethernet Adaptive Virtual Function Network Driver";
  23
  24 static const char iavf_copyright[] =
  25         "Copyright (c) 2013 - 2018 Intel Corporation.";
  26
  27 /* iavf_pci_tbl - PCI Device ID Table
  28  *
  29  * Wildcard entries (PCI_ANY_ID) should come last
  30  * Last entry must be all 0s
  31  *
  32  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  33  *   Class, Class Mask, private data (not used) }
  34  */
  35 static const struct pci_device_id iavf_pci_tbl[] = {
  36         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
  37         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
  38         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
  39         {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
  40         /* required last entry */
  41         {0, }
  42 };
  43
  44 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
  45
  46 MODULE_ALIAS("i40evf");
  47 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");
  49 MODULE_LICENSE("GPL v2");
  50
  51 static const struct net_device_ops iavf_netdev_ops;
  52
  53 int iavf_status_to_errno(enum iavf_status status)
  54 {
  55         switch (status) {
  56         case IAVF_SUCCESS:
  57                 return 0;
  58         case IAVF_ERR_PARAM:
  59         case IAVF_ERR_MAC_TYPE:
  60         case IAVF_ERR_INVALID_MAC_ADDR:
  61         case IAVF_ERR_INVALID_LINK_SETTINGS:
  62         case IAVF_ERR_INVALID_PD_ID:
  63         case IAVF_ERR_INVALID_QP_ID:
  64         case IAVF_ERR_INVALID_CQ_ID:
  65         case IAVF_ERR_INVALID_CEQ_ID:
  66         case IAVF_ERR_INVALID_AEQ_ID:
  67         case IAVF_ERR_INVALID_SIZE:
  68         case IAVF_ERR_INVALID_ARP_INDEX:
  69         case IAVF_ERR_INVALID_FPM_FUNC_ID:
  70         case IAVF_ERR_QP_INVALID_MSG_SIZE:
  71         case IAVF_ERR_INVALID_FRAG_COUNT:
  72         case IAVF_ERR_INVALID_ALIGNMENT:
  73         case IAVF_ERR_INVALID_PUSH_PAGE_INDEX:
  74         case IAVF_ERR_INVALID_IMM_DATA_SIZE:
  75         case IAVF_ERR_INVALID_VF_ID:
  76         case IAVF_ERR_INVALID_HMCFN_ID:
  77         case IAVF_ERR_INVALID_PBLE_INDEX:
  78         case IAVF_ERR_INVALID_SD_INDEX:
  79         case IAVF_ERR_INVALID_PAGE_DESC_INDEX:
  80         case IAVF_ERR_INVALID_SD_TYPE:
  81         case IAVF_ERR_INVALID_HMC_OBJ_INDEX:
  82         case IAVF_ERR_INVALID_HMC_OBJ_COUNT:
  83         case IAVF_ERR_INVALID_SRQ_ARM_LIMIT:
  84                 return -EINVAL;
  85         case IAVF_ERR_NVM:
  86         case IAVF_ERR_NVM_CHECKSUM:
  87         case IAVF_ERR_PHY:
  88         case IAVF_ERR_CONFIG:
  89         case IAVF_ERR_UNKNOWN_PHY:
  90         case IAVF_ERR_LINK_SETUP:
  91         case IAVF_ERR_ADAPTER_STOPPED:
  92         case IAVF_ERR_PRIMARY_REQUESTS_PENDING:
  93         case IAVF_ERR_AUTONEG_NOT_COMPLETE:
  94         case IAVF_ERR_RESET_FAILED:
  95         case IAVF_ERR_BAD_PTR:
  96         case IAVF_ERR_SWFW_SYNC:
  97         case IAVF_ERR_QP_TOOMANY_WRS_POSTED:
  98         case IAVF_ERR_QUEUE_EMPTY:
  99         case IAVF_ERR_FLUSHED_QUEUE:
 100         case IAVF_ERR_OPCODE_MISMATCH:
 101         case IAVF_ERR_CQP_COMPL_ERROR:
 102         case IAVF_ERR_BACKING_PAGE_ERROR:
 103         case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE:
 104         case IAVF_ERR_MEMCPY_FAILED:
 105         case IAVF_ERR_SRQ_ENABLED:
 106         case IAVF_ERR_ADMIN_QUEUE_ERROR:
 107         case IAVF_ERR_ADMIN_QUEUE_FULL:
 108         case IAVF_ERR_BAD_IWARP_CQE:
 109         case IAVF_ERR_NVM_BLANK_MODE:
 110         case IAVF_ERR_PE_DOORBELL_NOT_ENABLED:
 111         case IAVF_ERR_DIAG_TEST_FAILED:
 112         case IAVF_ERR_FIRMWARE_API_VERSION:
 113         case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
 114                 return -EIO;
 115         case IAVF_ERR_DEVICE_NOT_SUPPORTED:
 116                 return -ENODEV;
 117         case IAVF_ERR_NO_AVAILABLE_VSI:
 118         case IAVF_ERR_RING_FULL:
 119                 return -ENOSPC;
 120         case IAVF_ERR_NO_MEMORY:
 121                 return -ENOMEM;
 122         case IAVF_ERR_TIMEOUT:
 123         case IAVF_ERR_ADMIN_QUEUE_TIMEOUT:
 124                 return -ETIMEDOUT;
 125         case IAVF_ERR_NOT_IMPLEMENTED:
 126         case IAVF_NOT_SUPPORTED:
 127                 return -EOPNOTSUPP;
 128         case IAVF_ERR_ADMIN_QUEUE_NO_WORK:
 129                 return -EALREADY;
 130         case IAVF_ERR_NOT_READY:
 131                 return -EBUSY;
 132         case IAVF_ERR_BUF_TOO_SHORT:
 133                 return -EMSGSIZE;
 134         }
 135
 136         return -EIO;
 137 }
 138
 139 int virtchnl_status_to_errno(enum virtchnl_status_code v_status)
 140 {
 141         switch (v_status) {
 142         case VIRTCHNL_STATUS_SUCCESS:
 143                 return 0;
 144         case VIRTCHNL_STATUS_ERR_PARAM:
 145         case VIRTCHNL_STATUS_ERR_INVALID_VF_ID:
 146                 return -EINVAL;
 147         case VIRTCHNL_STATUS_ERR_NO_MEMORY:
 148                 return -ENOMEM;
 149         case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH:
 150         case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR:
 151         case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR:
 152                 return -EIO;
 153         case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED:
 154                 return -EOPNOTSUPP;
 155         }
 156
 157         return -EIO;
 158 }
 159
 160 /**
 161  * iavf_pdev_to_adapter - go from pci_dev to adapter
 162  * @pdev: pci_dev pointer
 163  */
 164 static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev)
 165 {
 166         return netdev_priv(pci_get_drvdata(pdev));
 167 }
 168
 169 /**
 170  * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
 171  * @hw:   pointer to the HW structure
 172  * @mem:  ptr to mem struct to fill out
 173  * @size: size of memory requested
 174  * @alignment: what to align the allocation to
 175  **/
 176 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,
 177                                          struct iavf_dma_mem *mem,
 178                                          u64 size, u32 alignment)
 179 {
 180         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
 181
 182         if (!mem)
 183                 return IAVF_ERR_PARAM;
 184
 185         mem->size = ALIGN(size, alignment);
 186         mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
 187                                      (dma_addr_t *)&mem->pa, GFP_KERNEL);
 188         if (mem->va)
 189                 return 0;
 190         else
 191                 return IAVF_ERR_NO_MEMORY;
 192 }
 193
 194 /**
 195  * iavf_free_dma_mem_d - OS specific memory free for shared code
 196  * @hw:   pointer to the HW structure
 197  * @mem:  ptr to mem struct to free
 198  **/
 199 enum iavf_status iavf_free_dma_mem_d(struct iavf_hw *hw,
 200                                      struct iavf_dma_mem *mem)
 201 {
 202         struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
 203
 204         if (!mem || !mem->va)
 205                 return IAVF_ERR_PARAM;
 206         dma_free_coherent(&adapter->pdev->dev, mem->size,
 207                           mem->va, (dma_addr_t)mem->pa);
 208         return 0;
 209 }
 210
 211 /**
 212  * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code
 213  * @hw:   pointer to the HW structure
 214  * @mem:  ptr to mem struct to fill out
 215  * @size: size of memory requested
 216  **/
 217 enum iavf_status iavf_allocate_virt_mem_d(struct iavf_hw *hw,
 218                                           struct iavf_virt_mem *mem, u32 size)
 219 {
 220         if (!mem)
 221                 return IAVF_ERR_PARAM;
 222
 223         mem->size = size;
 224         mem->va = kzalloc(size, GFP_KERNEL);
 225
 226         if (mem->va)
 227                 return 0;
 228         else
 229                 return IAVF_ERR_NO_MEMORY;
 230 }
 231
 232 /**
 233  * iavf_free_virt_mem_d - OS specific memory free for shared code
 234  * @hw:   pointer to the HW structure
 235  * @mem:  ptr to mem struct to free
 236  **/
 237 enum iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw,
 238                                       struct iavf_virt_mem *mem)
 239 {
 240         if (!mem)
 241                 return IAVF_ERR_PARAM;
 242
 243         /* it's ok to kfree a NULL pointer */
 244         kfree(mem->va);
 245
 246         return 0;
 247 }
 248
 249 /**
 250  * iavf_lock_timeout - try to lock mutex but give up after timeout
 251  * @lock: mutex that should be locked
 252  * @msecs: timeout in msecs
 253  *
 254  * Returns 0 on success, negative on failure
 255  **/
 256 int iavf_lock_timeout(struct mutex *lock, unsigned int msecs)
 257 {
 258         unsigned int wait, delay = 10;
 259
 260         for (wait = 0; wait < msecs; wait += delay) {
 261                 if (mutex_trylock(lock))
 262                         return 0;
 263
 264                 msleep(delay);
 265         }
 266
 267         return -1;
 268 }
 269
 270 /**
 271  * iavf_schedule_reset - Set the flags and schedule a reset event
 272  * @adapter: board private structure
 273  **/
 274 void iavf_schedule_reset(struct iavf_adapter *adapter)
 275 {
 276         if (!(adapter->flags &
 277               (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
 278                 adapter->flags |= IAVF_FLAG_RESET_NEEDED;
 279                 queue_work(adapter->wq, &adapter->reset_task);
 280         }
 281 }
 282
 283 /**
 284  * iavf_schedule_request_stats - Set the flags and schedule statistics request
 285  * @adapter: board private structure
 286  *
 287  * Sets IAVF_FLAG_AQ_REQUEST_STATS flag so iavf_watchdog_task() will explicitly
 288  * request and refresh ethtool stats
 289  **/
 290 void iavf_schedule_request_stats(struct iavf_adapter *adapter)
 291 {
 292         adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_STATS;
 293         mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
 294 }
 295
 296 /**
 297  * iavf_tx_timeout - Respond to a Tx Hang
 298  * @netdev: network interface device structure
 299  * @txqueue: queue number that is timing out
 300  **/
 301 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
 302 {
 303         struct iavf_adapter *adapter = netdev_priv(netdev);
 304
 305         adapter->tx_timeout_count++;
 306         iavf_schedule_reset(adapter);
 307 }
 308
 309 /**
 310  * iavf_misc_irq_disable - Mask off interrupt generation on the NIC
 311  * @adapter: board private structure
 312  **/
 313 static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
 314 {
 315         struct iavf_hw *hw = &adapter->hw;
 316
 317         if (!adapter->msix_entries)
 318                 return;
 319
 320         wr32(hw, IAVF_VFINT_DYN_CTL01, 0);
 321
 322         iavf_flush(hw);
 323
 324         synchronize_irq(adapter->msix_entries[0].vector);
 325 }
 326
 327 /**
 328  * iavf_misc_irq_enable - Enable default interrupt generation settings
 329  * @adapter: board private structure
 330  **/
 331 static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
 332 {
 333         struct iavf_hw *hw = &adapter->hw;
 334
 335         wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |
 336                                        IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
 337         wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
 338
 339         iavf_flush(hw);
 340 }
 341
 342 /**
 343  * iavf_irq_disable - Mask off interrupt generation on the NIC
 344  * @adapter: board private structure
 345  **/
 346 static void iavf_irq_disable(struct iavf_adapter *adapter)
 347 {
 348         int i;
 349         struct iavf_hw *hw = &adapter->hw;
 350
 351         if (!adapter->msix_entries)
 352                 return;
 353
 354         for (i = 1; i < adapter->num_msix_vectors; i++) {
 355                 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);
 356                 synchronize_irq(adapter->msix_entries[i].vector);
 357         }
 358         iavf_flush(hw);
 359 }
 360
 361 /**
 362  * iavf_irq_enable_queues - Enable interrupt for specified queues
 363  * @adapter: board private structure
 364  * @mask: bitmap of queues to enable
 365  **/
 366 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask)
 367 {
 368         struct iavf_hw *hw = &adapter->hw;
 369         int i;
 370
 371         for (i = 1; i < adapter->num_msix_vectors; i++) {
 372                 if (mask & BIT(i - 1)) {
 373                         wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),
 374                              IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
 375                              IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);
 376                 }
 377         }
 378 }
 379
 380 /**
 381  * iavf_irq_enable - Enable default interrupt generation settings
 382  * @adapter: board private structure
 383  * @flush: boolean value whether to run rd32()
 384  **/
 385 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
 386 {
 387         struct iavf_hw *hw = &adapter->hw;
 388
 389         iavf_misc_irq_enable(adapter);
 390         iavf_irq_enable_queues(adapter, ~0);
 391
 392         if (flush)
 393                 iavf_flush(hw);
 394 }
 395
 396 /**
 397  * iavf_msix_aq - Interrupt handler for vector 0
 398  * @irq: interrupt number
 399  * @data: pointer to netdev
 400  **/
 401 static irqreturn_t iavf_msix_aq(int irq, void *data)
 402 {
 403         struct net_device *netdev = data;
 404         struct iavf_adapter *adapter = netdev_priv(netdev);
 405         struct iavf_hw *hw = &adapter->hw;
 406
 407         /* handle non-queue interrupts, these reads clear the registers */
 408         rd32(hw, IAVF_VFINT_ICR01);
 409         rd32(hw, IAVF_VFINT_ICR0_ENA1);
 410
 411         if (adapter->state != __IAVF_REMOVE)
 412                 /* schedule work on the private workqueue */
 413                 queue_work(adapter->wq, &adapter->adminq_task);
 414
 415         return IRQ_HANDLED;
 416 }
 417
 418 /**
 419  * iavf_msix_clean_rings - MSIX mode Interrupt Handler
 420  * @irq: interrupt number
 421  * @data: pointer to a q_vector
 422  **/
 423 static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
 424 {
 425         struct iavf_q_vector *q_vector = data;
 426
 427         if (!q_vector->tx.ring && !q_vector->rx.ring)
 428                 return IRQ_HANDLED;
 429
 430         napi_schedule_irqoff(&q_vector->napi);
 431
 432         return IRQ_HANDLED;
 433 }
 434
 435 /**
 436  * iavf_map_vector_to_rxq - associate irqs with rx queues
 437  * @adapter: board private structure
 438  * @v_idx: interrupt number
 439  * @r_idx: queue number
 440  **/
 441 static void
 442 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
 443 {
 444         struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
 445         struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];
 446         struct iavf_hw *hw = &adapter->hw;
 447
 448         rx_ring->q_vector = q_vector;
 449         rx_ring->next = q_vector->rx.ring;
 450         rx_ring->vsi = &adapter->vsi;
 451         q_vector->rx.ring = rx_ring;
 452         q_vector->rx.count++;
 453         q_vector->rx.next_update = jiffies + 1;
 454         q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
 455         q_vector->ring_mask |= BIT(r_idx);
 456         wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),
 457              q_vector->rx.current_itr >> 1);
 458         q_vector->rx.current_itr = q_vector->rx.target_itr;
 459 }
 460
 461 /**
 462  * iavf_map_vector_to_txq - associate irqs with tx queues
 463  * @adapter: board private structure
 464  * @v_idx: interrupt number
 465  * @t_idx: queue number
 466  **/
 467 static void
 468 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
 469 {
 470         struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
 471         struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];
 472         struct iavf_hw *hw = &adapter->hw;
 473
 474         tx_ring->q_vector = q_vector;
 475         tx_ring->next = q_vector->tx.ring;
 476         tx_ring->vsi = &adapter->vsi;
 477         q_vector->tx.ring = tx_ring;
 478         q_vector->tx.count++;
 479         q_vector->tx.next_update = jiffies + 1;
 480         q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
 481         q_vector->num_ringpairs++;
 482         wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),
 483              q_vector->tx.target_itr >> 1);
 484         q_vector->tx.current_itr = q_vector->tx.target_itr;
 485 }
 486
 487 /**
 488  * iavf_map_rings_to_vectors - Maps descriptor rings to vectors
 489  * @adapter: board private structure to initialize
 490  *
 491  * This function maps descriptor rings to the queue-specific vectors
 492  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
 493  * one vector per ring/queue, but on a constrained vector budget, we
 494  * group the rings as "efficiently" as possible.  You would add new
 495  * mapping configurations in here.
 496  **/
 497 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
 498 {
 499         int rings_remaining = adapter->num_active_queues;
 500         int ridx = 0, vidx = 0;
 501         int q_vectors;
 502
 503         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 504
 505         for (; ridx < rings_remaining; ridx++) {
 506                 iavf_map_vector_to_rxq(adapter, vidx, ridx);
 507                 iavf_map_vector_to_txq(adapter, vidx, ridx);
 508
 509                 /* In the case where we have more queues than vectors, continue
 510                  * round-robin on vectors until all queues are mapped.
 511                  */
 512                 if (++vidx >= q_vectors)
 513                         vidx = 0;
 514         }
 515
 516         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
 517 }
 518
 519 /**
 520  * iavf_irq_affinity_notify - Callback for affinity changes
 521  * @notify: context as to what irq was changed
 522  * @mask: the new affinity mask
 523  *
 524  * This is a callback function used by the irq_set_affinity_notifier function
 525  * so that we may register to receive changes to the irq affinity masks.
 526  **/
 527 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
 528                                      const cpumask_t *mask)
 529 {
 530         struct iavf_q_vector *q_vector =
 531                 container_of(notify, struct iavf_q_vector, affinity_notify);
 532
 533         cpumask_copy(&q_vector->affinity_mask, mask);
 534 }
 535
 536 /**
 537  * iavf_irq_affinity_release - Callback for affinity notifier release
 538  * @ref: internal core kernel usage
 539  *
 540  * This is a callback function used by the irq_set_affinity_notifier function
 541  * to inform the current notification subscriber that they will no longer
 542  * receive notifications.
 543  **/
 544 static void iavf_irq_affinity_release(struct kref *ref) {}
 545
 546 /**
 547  * iavf_request_traffic_irqs - Initialize MSI-X interrupts
 548  * @adapter: board private structure
 549  * @basename: device basename
 550  *
 551  * Allocates MSI-X vectors for tx and rx handling, and requests
 552  * interrupts from the kernel.
 553  **/
 554 static int
 555 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
 556 {
 557         unsigned int vector, q_vectors;
 558         unsigned int rx_int_idx = 0, tx_int_idx = 0;
 559         int irq_num, err;
 560         int cpu;
 561
 562         iavf_irq_disable(adapter);
 563         /* Decrement for Other and TCP Timer vectors */
 564         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 565
 566         for (vector = 0; vector < q_vectors; vector++) {
 567                 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];
 568
 569                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 570
 571                 if (q_vector->tx.ring && q_vector->rx.ring) {
 572                         snprintf(q_vector->name, sizeof(q_vector->name),
 573                                  "iavf-%s-TxRx-%u", basename, rx_int_idx++);
 574                         tx_int_idx++;
 575                 } else if (q_vector->rx.ring) {
 576                         snprintf(q_vector->name, sizeof(q_vector->name),
 577                                  "iavf-%s-rx-%u", basename, rx_int_idx++);
 578                 } else if (q_vector->tx.ring) {
 579                         snprintf(q_vector->name, sizeof(q_vector->name),
 580                                  "iavf-%s-tx-%u", basename, tx_int_idx++);
 581                 } else {
 582                         /* skip this unused q_vector */
 583                         continue;
 584                 }
 585                 err = request_irq(irq_num,
 586                                   iavf_msix_clean_rings,
 587                                   0,
 588                                   q_vector->name,
 589                                   q_vector);
 590                 if (err) {
 591                         dev_info(&adapter->pdev->dev,
 592                                  "Request_irq failed, error: %d\n", err);
 593                         goto free_queue_irqs;
 594                 }
 595                 /* register for affinity change notifications */
 596                 q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
 597                 q_vector->affinity_notify.release =
 598                                                    iavf_irq_affinity_release;
 599                 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
 600                 /* Spread the IRQ affinity hints across online CPUs. Note that
 601                  * get_cpu_mask returns a mask with a permanent lifetime so
 602                  * it's safe to use as a hint for irq_update_affinity_hint.
 603                  */
 604                 cpu = cpumask_local_spread(q_vector->v_idx, -1);
 605                 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
 606         }
 607
 608         return 0;
 609
 610 free_queue_irqs:
 611         while (vector) {
 612                 vector--;
 613                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 614                 irq_set_affinity_notifier(irq_num, NULL);
 615                 irq_update_affinity_hint(irq_num, NULL);
 616                 free_irq(irq_num, &adapter->q_vectors[vector]);
 617         }
 618         return err;
 619 }
 620
 621 /**
 622  * iavf_request_misc_irq - Initialize MSI-X interrupts
 623  * @adapter: board private structure
 624  *
 625  * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
 626  * vector is only for the admin queue, and stays active even when the netdev
 627  * is closed.
 628  **/
 629 static int iavf_request_misc_irq(struct iavf_adapter *adapter)
 630 {
 631         struct net_device *netdev = adapter->netdev;
 632         int err;
 633
 634         snprintf(adapter->misc_vector_name,
 635                  sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",
 636                  dev_name(&adapter->pdev->dev));
 637         err = request_irq(adapter->msix_entries[0].vector,
 638                           &iavf_msix_aq, 0,
 639                           adapter->misc_vector_name, netdev);
 640         if (err) {
 641                 dev_err(&adapter->pdev->dev,
 642                         "request_irq for %s failed: %d\n",
 643                         adapter->misc_vector_name, err);
 644                 free_irq(adapter->msix_entries[0].vector, netdev);
 645         }
 646         return err;
 647 }
 648
 649 /**
 650  * iavf_free_traffic_irqs - Free MSI-X interrupts
 651  * @adapter: board private structure
 652  *
 653  * Frees all MSI-X vectors other than 0.
 654  **/
 655 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
 656 {
 657         int vector, irq_num, q_vectors;
 658
 659         if (!adapter->msix_entries)
 660                 return;
 661
 662         q_vectors = adapter->num_msix_vectors - NONQ_VECS;
 663
 664         for (vector = 0; vector < q_vectors; vector++) {
 665                 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
 666                 irq_set_affinity_notifier(irq_num, NULL);
 667                 irq_update_affinity_hint(irq_num, NULL);
 668                 free_irq(irq_num, &adapter->q_vectors[vector]);
 669         }
 670 }
 671
 672 /**
 673  * iavf_free_misc_irq - Free MSI-X miscellaneous vector
 674  * @adapter: board private structure
 675  *
 676  * Frees MSI-X vector 0.
 677  **/
 678 static void iavf_free_misc_irq(struct iavf_adapter *adapter)
 679 {
 680         struct net_device *netdev = adapter->netdev;
 681
 682         if (!adapter->msix_entries)
 683                 return;
 684
 685         free_irq(adapter->msix_entries[0].vector, netdev);
 686 }
 687
 688 /**
 689  * iavf_configure_tx - Configure Transmit Unit after Reset
 690  * @adapter: board private structure
 691  *
 692  * Configure the Tx unit of the MAC after a reset.
 693  **/
 694 static void iavf_configure_tx(struct iavf_adapter *adapter)
 695 {
 696         struct iavf_hw *hw = &adapter->hw;
 697         int i;
 698
 699         for (i = 0; i < adapter->num_active_queues; i++)
 700                 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);
 701 }
 702
 703 /**
 704  * iavf_configure_rx - Configure Receive Unit after Reset
 705  * @adapter: board private structure
 706  *
 707  * Configure the Rx unit of the MAC after a reset.
 708  **/
 709 static void iavf_configure_rx(struct iavf_adapter *adapter)
 710 {
 711         unsigned int rx_buf_len = IAVF_RXBUFFER_2048;
 712         struct iavf_hw *hw = &adapter->hw;
 713         int i;
 714
 715         /* Legacy Rx will always default to a 2048 buffer size. */
 716 #if (PAGE_SIZE < 8192)
 717         if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) {
 718                 struct net_device *netdev = adapter->netdev;
 719
 720                 /* For jumbo frames on systems with 4K pages we have to use
 721                  * an order 1 page, so we might as well increase the size
 722                  * of our Rx buffer to make better use of the available space
 723                  */
 724                 rx_buf_len = IAVF_RXBUFFER_3072;
 725
 726                 /* We use a 1536 buffer size for configurations with
 727                  * standard Ethernet mtu.  On x86 this gives us enough room
 728                  * for shared info and 192 bytes of padding.
 729                  */
 730                 if (!IAVF_2K_TOO_SMALL_WITH_PADDING &&
 731                     (netdev->mtu <= ETH_DATA_LEN))
 732                         rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN;
 733         }
 734 #endif
 735
 736         for (i = 0; i < adapter->num_active_queues; i++) {
 737                 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);
 738                 adapter->rx_rings[i].rx_buf_len = rx_buf_len;
 739
 740                 if (adapter->flags & IAVF_FLAG_LEGACY_RX)
 741                         clear_ring_build_skb_enabled(&adapter->rx_rings[i]);
 742                 else
 743                         set_ring_build_skb_enabled(&adapter->rx_rings[i]);
 744         }
 745 }
 746
 747 /**
 748  * iavf_find_vlan - Search filter list for specific vlan filter
 749  * @adapter: board private structure
 750  * @vlan: vlan tag
 751  *
 752  * Returns ptr to the filter object or NULL. Must be called while holding the
 753  * mac_vlan_list_lock.
 754  **/
 755 static struct
 756 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter,
 757                                  struct iavf_vlan vlan)
 758 {
 759         struct iavf_vlan_filter *f;
 760
 761         list_for_each_entry(f, &adapter->vlan_filter_list, list) {
 762                 if (f->vlan.vid == vlan.vid &&
 763                     f->vlan.tpid == vlan.tpid)
 764                         return f;
 765         }
 766
 767         return NULL;
 768 }
 769
 770 /**
 771  * iavf_add_vlan - Add a vlan filter to the list
 772  * @adapter: board private structure
 773  * @vlan: VLAN tag
 774  *
 775  * Returns ptr to the filter object or NULL when no memory available.
 776  **/
 777 static struct
 778 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,
 779                                 struct iavf_vlan vlan)
 780 {
 781         struct iavf_vlan_filter *f = NULL;
 782
 783         spin_lock_bh(&adapter->mac_vlan_list_lock);
 784
 785         f = iavf_find_vlan(adapter, vlan);
 786         if (!f) {
 787                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 788                 if (!f)
 789                         goto clearout;
 790
 791                 f->vlan = vlan;
 792
 793                 list_add_tail(&f->list, &adapter->vlan_filter_list);
 794                 f->add = true;
 795                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
 796         }
 797
 798 clearout:
 799         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 800         return f;
 801 }
 802
 803 /**
 804  * iavf_del_vlan - Remove a vlan filter from the list
 805  * @adapter: board private structure
 806  * @vlan: VLAN tag
 807  **/
 808 static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan)
 809 {
 810         struct iavf_vlan_filter *f;
 811
 812         spin_lock_bh(&adapter->mac_vlan_list_lock);
 813
 814         f = iavf_find_vlan(adapter, vlan);
 815         if (f) {
 816                 f->remove = true;
 817                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
 818         }
 819
 820         spin_unlock_bh(&adapter->mac_vlan_list_lock);
 821 }
 822
 823 /**
 824  * iavf_restore_filters
 825  * @adapter: board private structure
 826  *
 827  * Restore existing non MAC filters when VF netdev comes back up
 828  **/
 829 static void iavf_restore_filters(struct iavf_adapter *adapter)
 830 {
 831         u16 vid;
 832
 833         /* re-add all VLAN filters */
 834         for_each_set_bit(vid, adapter->vsi.active_cvlans, VLAN_N_VID)
 835                 iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021Q));
 836
 837         for_each_set_bit(vid, adapter->vsi.active_svlans, VLAN_N_VID)
 838                 iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021AD));
 839 }
 840
 841 /**
 842  * iavf_get_num_vlans_added - get number of VLANs added
 843  * @adapter: board private structure
 844  */
 845 u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter)
 846 {
 847         return bitmap_weight(adapter->vsi.active_cvlans, VLAN_N_VID) +
 848                 bitmap_weight(adapter->vsi.active_svlans, VLAN_N_VID);
 849 }
 850
 851 /**
 852  * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF
 853  * @adapter: board private structure
 854  *
 855  * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN,
 856  * do not impose a limit as that maintains current behavior and for
 857  * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF.
 858  **/
 859 static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter)
 860 {
 861         /* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has
 862          * never been a limit on the VF driver side
 863          */
 864         if (VLAN_ALLOWED(adapter))
 865                 return VLAN_N_VID;
 866         else if (VLAN_V2_ALLOWED(adapter))
 867                 return adapter->vlan_v2_caps.filtering.max_filters;
 868
 869         return 0;
 870 }
 871
 872 /**
 873  * iavf_max_vlans_added - check if maximum VLANs allowed already exist
 874  * @adapter: board private structure
 875  **/
 876 static bool iavf_max_vlans_added(struct iavf_adapter *adapter)
 877 {
 878         if (iavf_get_num_vlans_added(adapter) <
 879             iavf_get_max_vlans_allowed(adapter))
 880                 return false;
 881
 882         return true;
 883 }
 884
 885 /**
 886  * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
 887  * @netdev: network device struct
 888  * @proto: unused protocol data
 889  * @vid: VLAN tag
 890  **/
 891 static int iavf_vlan_rx_add_vid(struct net_device *netdev,
 892                                 __always_unused __be16 proto, u16 vid)
 893 {
 894         struct iavf_adapter *adapter = netdev_priv(netdev);
 895
 896         if (!VLAN_FILTERING_ALLOWED(adapter))
 897                 return -EIO;
 898
 899         if (iavf_max_vlans_added(adapter)) {
 900                 netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n",
 901                            iavf_get_max_vlans_allowed(adapter));
 902                 return -EIO;
 903         }
 904
 905         if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))))
 906                 return -ENOMEM;
 907
 908         return 0;
 909 }
 910
 911 /**
 912  * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
 913  * @netdev: network device struct
 914  * @proto: unused protocol data
 915  * @vid: VLAN tag
 916  **/
 917 static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
 918                                  __always_unused __be16 proto, u16 vid)
 919 {
 920         struct iavf_adapter *adapter = netdev_priv(netdev);
 921
 922         iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)));
 923         if (proto == cpu_to_be16(ETH_P_8021Q))
 924                 clear_bit(vid, adapter->vsi.active_cvlans);
 925         else
 926                 clear_bit(vid, adapter->vsi.active_svlans);
 927
 928         return 0;
 929 }
 930
 931 /**
 932  * iavf_find_filter - Search filter list for specific mac filter
 933  * @adapter: board private structure
 934  * @macaddr: the MAC address
 935  *
 936  * Returns ptr to the filter object or NULL. Must be called while holding the
 937  * mac_vlan_list_lock.
 938  **/
 939 static struct
 940 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
 941                                   const u8 *macaddr)
 942 {
 943         struct iavf_mac_filter *f;
 944
 945         if (!macaddr)
 946                 return NULL;
 947
 948         list_for_each_entry(f, &adapter->mac_filter_list, list) {
 949                 if (ether_addr_equal(macaddr, f->macaddr))
 950                         return f;
 951         }
 952         return NULL;
 953 }
 954
 955 /**
 956  * iavf_add_filter - Add a mac filter to the filter list
 957  * @adapter: board private structure
 958  * @macaddr: the MAC address
 959  *
 960  * Returns ptr to the filter object or NULL when no memory available.
 961  **/
 962 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
 963                                         const u8 *macaddr)
 964 {
 965         struct iavf_mac_filter *f;
 966
 967         if (!macaddr)
 968                 return NULL;
 969
 970         f = iavf_find_filter(adapter, macaddr);
 971         if (!f) {
 972                 f = kzalloc(sizeof(*f), GFP_ATOMIC);
 973                 if (!f)
 974                         return f;
 975
 976                 ether_addr_copy(f->macaddr, macaddr);
 977
 978                 list_add_tail(&f->list, &adapter->mac_filter_list);
 979                 f->add = true;
 980                 f->add_handled = false;
 981                 f->is_new_mac = true;
 982                 f->is_primary = ether_addr_equal(macaddr, adapter->hw.mac.addr);
 983                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
 984         } else {
 985                 f->remove = false;
 986         }
 987
 988         return f;
 989 }
 990
 991 /**
 992  * iavf_replace_primary_mac - Replace current primary address
 993  * @adapter: board private structure
 994  * @new_mac: new MAC address to be applied
 995  *
 996  * Replace current dev_addr and send request to PF for removal of previous
 997  * primary MAC address filter and addition of new primary MAC filter.
 998  * Return 0 for success, -ENOMEM for failure.
 999  *
1000  * Do not call this with mac_vlan_list_lock!
1001  **/
1002 int iavf_replace_primary_mac(struct iavf_adapter *adapter,
1003                              const u8 *new_mac)
1004 {
1005         struct iavf_hw *hw = &adapter->hw;
1006         struct iavf_mac_filter *f;
1007
1008         spin_lock_bh(&adapter->mac_vlan_list_lock);
1009
1010         list_for_each_entry(f, &adapter->mac_filter_list, list) {
1011                 f->is_primary = false;
1012         }
1013
1014         f = iavf_find_filter(adapter, hw->mac.addr);
1015         if (f) {
1016                 f->remove = true;
1017                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1018         }
1019
1020         f = iavf_add_filter(adapter, new_mac);
1021
1022         if (f) {
1023                 /* Always send the request to add if changing primary MAC
1024                  * even if filter is already present on the list
1025                  */
1026                 f->is_primary = true;
1027                 f->add = true;
1028                 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
1029                 ether_addr_copy(hw->mac.addr, new_mac);
1030         }
1031
1032         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1033
1034         /* schedule the watchdog task to immediately process the request */
1035         if (f) {
1036                 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
1037                 return 0;
1038         }
1039         return -ENOMEM;
1040 }
1041
1042 /**
1043  * iavf_is_mac_set_handled - wait for a response to set MAC from PF
1044  * @netdev: network interface device structure
1045  * @macaddr: MAC address to set
1046  *
1047  * Returns true on success, false on failure
1048  */
1049 static bool iavf_is_mac_set_handled(struct net_device *netdev,
1050                                     const u8 *macaddr)
1051 {
1052         struct iavf_adapter *adapter = netdev_priv(netdev);
1053         struct iavf_mac_filter *f;
1054         bool ret = false;
1055
1056         spin_lock_bh(&adapter->mac_vlan_list_lock);
1057
1058         f = iavf_find_filter(adapter, macaddr);
1059
1060         if (!f || (!f->add && f->add_handled))
1061                 ret = true;
1062
1063         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1064
1065         return ret;
1066 }
1067
1068 /**
1069  * iavf_set_mac - NDO callback to set port MAC address
1070  * @netdev: network interface device structure
1071  * @p: pointer to an address structure
1072  *
1073  * Returns 0 on success, negative on failure
1074  */
1075 static int iavf_set_mac(struct net_device *netdev, void *p)
1076 {
1077         struct iavf_adapter *adapter = netdev_priv(netdev);
1078         struct sockaddr *addr = p;
1079         int ret;
1080
1081         if (!is_valid_ether_addr(addr->sa_data))
1082                 return -EADDRNOTAVAIL;
1083
1084         ret = iavf_replace_primary_mac(adapter, addr->sa_data);
1085
1086         if (ret)
1087                 return ret;
1088
1089         ret = wait_event_interruptible_timeout(adapter->vc_waitqueue,
1090                                                iavf_is_mac_set_handled(netdev, addr->sa_data),
1091                                                msecs_to_jiffies(2500));
1092
1093         /* If ret < 0 then it means wait was interrupted.
1094          * If ret == 0 then it means we got a timeout.
1095          * else it means we got response for set MAC from PF,
1096          * check if netdev MAC was updated to requested MAC,
1097          * if yes then set MAC succeeded otherwise it failed return -EACCES
1098          */
1099         if (ret < 0)
1100                 return ret;
1101
1102         if (!ret)
1103                 return -EAGAIN;
1104
1105         if (!ether_addr_equal(netdev->dev_addr, addr->sa_data))
1106                 return -EACCES;
1107
1108         return 0;
1109 }
1110
1111 /**
1112  * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
1113  * @netdev: the netdevice
1114  * @addr: address to add
1115  *
1116  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1117  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1118  */
1119 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
1120 {
1121         struct iavf_adapter *adapter = netdev_priv(netdev);
1122
1123         if (iavf_add_filter(adapter, addr))
1124                 return 0;
1125         else
1126                 return -ENOMEM;
1127 }
1128
1129 /**
1130  * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1131  * @netdev: the netdevice
1132  * @addr: address to add
1133  *
1134  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1135  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1136  */
1137 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
1138 {
1139         struct iavf_adapter *adapter = netdev_priv(netdev);
1140         struct iavf_mac_filter *f;
1141
1142         /* Under some circumstances, we might receive a request to delete
1143          * our own device address from our uc list. Because we store the
1144          * device address in the VSI's MAC/VLAN filter list, we need to ignore
1145          * such requests and not delete our device address from this list.
1146          */
1147         if (ether_addr_equal(addr, netdev->dev_addr))
1148                 return 0;
1149
1150         f = iavf_find_filter(adapter, addr);
1151         if (f) {
1152                 f->remove = true;
1153                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1154         }
1155         return 0;
1156 }
1157
1158 /**
1159  * iavf_set_rx_mode - NDO callback to set the netdev filters
1160  * @netdev: network interface device structure
1161  **/
1162 static void iavf_set_rx_mode(struct net_device *netdev)
1163 {
1164         struct iavf_adapter *adapter = netdev_priv(netdev);
1165
1166         spin_lock_bh(&adapter->mac_vlan_list_lock);
1167         __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1168         __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1169         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1170
1171         if (netdev->flags & IFF_PROMISC &&
1172             !(adapter->flags & IAVF_FLAG_PROMISC_ON))
1173                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC;
1174         else if (!(netdev->flags & IFF_PROMISC) &&
1175                  adapter->flags & IAVF_FLAG_PROMISC_ON)
1176                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC;
1177
1178         if (netdev->flags & IFF_ALLMULTI &&
1179             !(adapter->flags & IAVF_FLAG_ALLMULTI_ON))
1180                 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI;
1181         else if (!(netdev->flags & IFF_ALLMULTI) &&
1182                  adapter->flags & IAVF_FLAG_ALLMULTI_ON)
1183                 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI;
1184 }
1185
1186 /**
1187  * iavf_napi_enable_all - enable NAPI on all queue vectors
1188  * @adapter: board private structure
1189  **/
1190 static void iavf_napi_enable_all(struct iavf_adapter *adapter)
1191 {
1192         int q_idx;
1193         struct iavf_q_vector *q_vector;
1194         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1195
1196         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1197                 struct napi_struct *napi;
1198
1199                 q_vector = &adapter->q_vectors[q_idx];
1200                 napi = &q_vector->napi;
1201                 napi_enable(napi);
1202         }
1203 }
1204
1205 /**
1206  * iavf_napi_disable_all - disable NAPI on all queue vectors
1207  * @adapter: board private structure
1208  **/
1209 static void iavf_napi_disable_all(struct iavf_adapter *adapter)
1210 {
1211         int q_idx;
1212         struct iavf_q_vector *q_vector;
1213         int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1214
1215         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1216                 q_vector = &adapter->q_vectors[q_idx];
1217                 napi_disable(&q_vector->napi);
1218         }
1219 }
1220
1221 /**
1222  * iavf_configure - set up transmit and receive data structures
1223  * @adapter: board private structure
1224  **/
1225 static void iavf_configure(struct iavf_adapter *adapter)
1226 {
1227         struct net_device *netdev = adapter->netdev;
1228         int i;
1229
1230         iavf_set_rx_mode(netdev);
1231
1232         iavf_configure_tx(adapter);
1233         iavf_configure_rx(adapter);
1234         adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
1235
1236         for (i = 0; i < adapter->num_active_queues; i++) {
1237                 struct iavf_ring *ring = &adapter->rx_rings[i];
1238
1239                 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring));
1240         }
1241 }
1242
1243 /**
1244  * iavf_up_complete - Finish the last steps of bringing up a connection
1245  * @adapter: board private structure
1246  *
1247  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
1248  **/
1249 static void iavf_up_complete(struct iavf_adapter *adapter)
1250 {
1251         iavf_change_state(adapter, __IAVF_RUNNING);
1252         clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1253
1254         iavf_napi_enable_all(adapter);
1255
1256         adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES;
1257         if (CLIENT_ENABLED(adapter))
1258                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN;
1259         mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
1260 }
1261
1262 /**
1263  * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF
1264  * yet and mark other to be removed.
1265  * @adapter: board private structure
1266  **/
1267 static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)
1268 {
1269         struct iavf_vlan_filter *vlf, *vlftmp;
1270         struct iavf_mac_filter *f, *ftmp;
1271
1272         spin_lock_bh(&adapter->mac_vlan_list_lock);
1273         /* clear the sync flag on all filters */
1274         __dev_uc_unsync(adapter->netdev, NULL);
1275         __dev_mc_unsync(adapter->netdev, NULL);
1276
1277         /* remove all MAC filters */
1278         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list,
1279                                  list) {
1280                 if (f->add) {
1281                         list_del(&f->list);
1282                         kfree(f);
1283                 } else {
1284                         f->remove = true;
1285                 }
1286         }
1287
1288         /* remove all VLAN filters */
1289         list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
1290                                  list) {
1291                 if (vlf->add) {
1292                         list_del(&vlf->list);
1293                         kfree(vlf);
1294                 } else {
1295                         vlf->remove = true;
1296                 }
1297         }
1298         spin_unlock_bh(&adapter->mac_vlan_list_lock);
1299 }
1300
1301 /**
1302  * iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and
1303  * mark other to be removed.
1304  * @adapter: board private structure
1305  **/
1306 static void iavf_clear_cloud_filters(struct iavf_adapter *adapter)
1307 {
1308         struct iavf_cloud_filter *cf, *cftmp;
1309
1310         /* remove all cloud filters */
1311         spin_lock_bh(&adapter->cloud_filter_list_lock);
1312         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
1313                                  list) {
1314                 if (cf->add) {
1315                         list_del(&cf->list);
1316                         kfree(cf);
1317                         adapter->num_cloud_filters--;
1318                 } else {
1319                         cf->del = true;
1320                 }
1321         }
1322         spin_unlock_bh(&adapter->cloud_filter_list_lock);
1323 }
1324
1325 /**
1326  * iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark
1327  * other to be removed.
1328  * @adapter: board private structure
1329  **/
1330 static void iavf_clear_fdir_filters(struct iavf_adapter *adapter)
1331 {
1332         struct iavf_fdir_fltr *fdir, *fdirtmp;
1333
1334         /* remove all Flow Director filters */
1335         spin_lock_bh(&adapter->fdir_fltr_lock);
1336         list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head,
1337                                  list) {
1338                 if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) {
1339                         list_del(&fdir->list);
1340                         kfree(fdir);
1341                         adapter->fdir_active_fltr--;
1342                 } else {
1343                         fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
1344                 }
1345         }
1346         spin_unlock_bh(&adapter->fdir_fltr_lock);
1347 }
1348
1349 /**
1350  * iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark
1351  * other to be removed.
1352  * @adapter: board private structure
1353  **/
1354 static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter)
1355 {
1356         struct iavf_adv_rss *rss, *rsstmp;
1357
1358         /* remove all advance RSS configuration */
1359         spin_lock_bh(&adapter->adv_rss_lock);
1360         list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
1361                                  list) {
1362                 if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) {
1363                         list_del(&rss->list);
1364                         kfree(rss);
1365                 } else {
1366                         rss->state = IAVF_ADV_RSS_DEL_REQUEST;
1367                 }
1368         }
1369         spin_unlock_bh(&adapter->adv_rss_lock);
1370 }
1371
1372 /**
1373  * iavf_down - Shutdown the connection processing
1374  * @adapter: board private structure
1375  *
1376  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
1377  **/
1378 void iavf_down(struct iavf_adapter *adapter)
1379 {
1380         struct net_device *netdev = adapter->netdev;
1381
1382         if (adapter->state <= __IAVF_DOWN_PENDING)
1383                 return;
1384
1385         netif_carrier_off(netdev);
1386         netif_tx_disable(netdev);
1387         adapter->link_up = false;
1388         iavf_napi_disable_all(adapter);
1389         iavf_irq_disable(adapter);
1390
1391         iavf_clear_mac_vlan_filters(adapter);
1392         iavf_clear_cloud_filters(adapter);
1393         iavf_clear_fdir_filters(adapter);
1394         iavf_clear_adv_rss_conf(adapter);
1395
1396         if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)) {
1397                 /* cancel any current operation */
1398                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1399                 /* Schedule operations to close down the HW. Don't wait
1400                  * here for this to complete. The watchdog is still running
1401                  * and it will take care of this.
1402                  */
1403                 if (!list_empty(&adapter->mac_filter_list))
1404                         adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1405                 if (!list_empty(&adapter->vlan_filter_list))
1406                         adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
1407                 if (!list_empty(&adapter->cloud_filter_list))
1408                         adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
1409                 if (!list_empty(&adapter->fdir_list_head))
1410                         adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
1411                 if (!list_empty(&adapter->adv_rss_list_head))
1412                         adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
1413                 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
1414         }
1415
1416         mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
1417 }
1418
1419 /**
1420  * iavf_acquire_msix_vectors - Setup the MSIX capability
1421  * @adapter: board private structure
1422  * @vectors: number of vectors to request
1423  *
1424  * Work with the OS to set up the MSIX vectors needed.
1425  *
1426  * Returns 0 on success, negative on failure
1427  **/
1428 static int
1429 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
1430 {
1431         int err, vector_threshold;
1432
1433         /* We'll want at least 3 (vector_threshold):
1434          * 0) Other (Admin Queue and link, mostly)
1435          * 1) TxQ[0] Cleanup
1436          * 2) RxQ[0] Cleanup
1437          */
1438         vector_threshold = MIN_MSIX_COUNT;
1439
1440         /* The more we get, the more we will assign to Tx/Rx Cleanup
1441          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1442          * Right now, we simply care about how many we'll get; we'll
1443          * set them up later while requesting irq's.
1444          */
1445         err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1446                                     vector_threshold, vectors);
1447         if (err < 0) {
1448                 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1449                 kfree(adapter->msix_entries);
1450                 adapter->msix_entries = NULL;
1451                 return err;
1452         }
1453
1454         /* Adjust for only the vectors we'll use, which is minimum
1455          * of max_msix_q_vectors + NONQ_VECS, or the number of
1456          * vectors we were allocated.
1457          */
1458         adapter->num_msix_vectors = err;
1459         return 0;
1460 }
1461
1462 /**
1463  * iavf_free_queues - Free memory for all rings
1464  * @adapter: board private structure to initialize
1465  *
1466  * Free all of the memory associated with queue pairs.
1467  **/
1468 static void iavf_free_queues(struct iavf_adapter *adapter)
1469 {
1470         if (!adapter->vsi_res)
1471                 return;
1472         adapter->num_active_queues = 0;
1473         kfree(adapter->tx_rings);
1474         adapter->tx_rings = NULL;
1475         kfree(adapter->rx_rings);
1476         adapter->rx_rings = NULL;
1477 }
1478
1479 /**
1480  * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload
1481  * @adapter: board private structure
1482  *
1483  * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or
1484  * stripped in certain descriptor fields. Instead of checking the offload
1485  * capability bits in the hot path, cache the location the ring specific
1486  * flags.
1487  */
1488 void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter)
1489 {
1490         int i;
1491
1492         for (i = 0; i < adapter->num_active_queues; i++) {
1493                 struct iavf_ring *tx_ring = &adapter->tx_rings[i];
1494                 struct iavf_ring *rx_ring = &adapter->rx_rings[i];
1495
1496                 /* prevent multiple L2TAG bits being set after VFR */
1497                 tx_ring->flags &=
1498                         ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1499                           IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2);
1500                 rx_ring->flags &=
1501                         ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1502                           IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2);
1503
1504                 if (VLAN_ALLOWED(adapter)) {
1505                         tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1506                         rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1507                 } else if (VLAN_V2_ALLOWED(adapter)) {
1508                         struct virtchnl_vlan_supported_caps *stripping_support;
1509                         struct virtchnl_vlan_supported_caps *insertion_support;
1510
1511                         stripping_support =
1512                                 &adapter->vlan_v2_caps.offloads.stripping_support;
1513                         insertion_support =
1514                                 &adapter->vlan_v2_caps.offloads.insertion_support;
1515
1516                         if (stripping_support->outer) {
1517                                 if (stripping_support->outer &
1518                                     VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1519                                         rx_ring->flags |=
1520                                                 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1521                                 else if (stripping_support->outer &
1522                                          VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1523                                         rx_ring->flags |=
1524                                                 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1525                         } else if (stripping_support->inner) {
1526                                 if (stripping_support->inner &
1527                                     VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1528                                         rx_ring->flags |=
1529                                                 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1530                                 else if (stripping_support->inner &
1531                                          VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1532                                         rx_ring->flags |=
1533                                                 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1534                         }
1535
1536                         if (insertion_support->outer) {
1537                                 if (insertion_support->outer &
1538                                     VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1539                                         tx_ring->flags |=
1540                                                 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1541                                 else if (insertion_support->outer &
1542                                          VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1543                                         tx_ring->flags |=
1544                                                 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1545                         } else if (insertion_support->inner) {
1546                                 if (insertion_support->inner &
1547                                     VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1548                                         tx_ring->flags |=
1549                                                 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1550                                 else if (insertion_support->inner &
1551                                          VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1552                                         tx_ring->flags |=
1553                                                 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1554                         }
1555                 }
1556         }
1557 }
1558
1559 /**
1560  * iavf_alloc_queues - Allocate memory for all rings
1561  * @adapter: board private structure to initialize
1562  *
1563  * We allocate one ring per queue at run-time since we don't know the
1564  * number of queues at compile-time.  The polling_netdev array is
1565  * intended for Multiqueue, but should work fine with a single queue.
1566  **/
1567 static int iavf_alloc_queues(struct iavf_adapter *adapter)
1568 {
1569         int i, num_active_queues;
1570
1571         /* If we're in reset reallocating queues we don't actually know yet for
1572          * certain the PF gave us the number of queues we asked for but we'll
1573          * assume it did.  Once basic reset is finished we'll confirm once we
1574          * start negotiating config with PF.
1575          */
1576         if (adapter->num_req_queues)
1577                 num_active_queues = adapter->num_req_queues;
1578         else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1579                  adapter->num_tc)
1580                 num_active_queues = adapter->ch_config.total_qps;
1581         else
1582                 num_active_queues = min_t(int,
1583                                           adapter->vsi_res->num_queue_pairs,
1584                                           (int)(num_online_cpus()));
1585
1586
1587         adapter->tx_rings = kcalloc(num_active_queues,
1588                                     sizeof(struct iavf_ring), GFP_KERNEL);
1589         if (!adapter->tx_rings)
1590                 goto err_out;
1591         adapter->rx_rings = kcalloc(num_active_queues,
1592                                     sizeof(struct iavf_ring), GFP_KERNEL);
1593         if (!adapter->rx_rings)
1594                 goto err_out;
1595
1596         for (i = 0; i < num_active_queues; i++) {
1597                 struct iavf_ring *tx_ring;
1598                 struct iavf_ring *rx_ring;
1599
1600                 tx_ring = &adapter->tx_rings[i];
1601
1602                 tx_ring->queue_index = i;
1603                 tx_ring->netdev = adapter->netdev;
1604                 tx_ring->dev = &adapter->pdev->dev;
1605                 tx_ring->count = adapter->tx_desc_count;
1606                 tx_ring->itr_setting = IAVF_ITR_TX_DEF;
1607                 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
1608                         tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;
1609
1610                 rx_ring = &adapter->rx_rings[i];
1611                 rx_ring->queue_index = i;
1612                 rx_ring->netdev = adapter->netdev;
1613                 rx_ring->dev = &adapter->pdev->dev;
1614                 rx_ring->count = adapter->rx_desc_count;
1615                 rx_ring->itr_setting = IAVF_ITR_RX_DEF;
1616         }
1617
1618         adapter->num_active_queues = num_active_queues;
1619
1620         iavf_set_queue_vlan_tag_loc(adapter);
1621
1622         return 0;
1623
1624 err_out:
1625         iavf_free_queues(adapter);
1626         return -ENOMEM;
1627 }
1628
1629 /**
1630  * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
1631  * @adapter: board private structure to initialize
1632  *
1633  * Attempt to configure the interrupts using the best available
1634  * capabilities of the hardware and the kernel.
1635  **/
1636 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
1637 {
1638         int vector, v_budget;
1639         int pairs = 0;
1640         int err = 0;
1641
1642         if (!adapter->vsi_res) {
1643                 err = -EIO;
1644                 goto out;
1645         }
1646         pairs = adapter->num_active_queues;
1647
1648         /* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1649          * us much good if we have more vectors than CPUs. However, we already
1650          * limit the total number of queues by the number of CPUs so we do not
1651          * need any further limiting here.
1652          */
1653         v_budget = min_t(int, pairs + NONQ_VECS,
1654                          (int)adapter->vf_res->max_vectors);
1655
1656         adapter->msix_entries = kcalloc(v_budget,
1657                                         sizeof(struct msix_entry), GFP_KERNEL);
1658         if (!adapter->msix_entries) {
1659                 err = -ENOMEM;
1660                 goto out;
1661         }
1662
1663         for (vector = 0; vector < v_budget; vector++)
1664                 adapter->msix_entries[vector].entry = vector;
1665
1666         err = iavf_acquire_msix_vectors(adapter, v_budget);
1667
1668 out:
1669         netif_set_real_num_rx_queues(adapter->netdev, pairs);
1670         netif_set_real_num_tx_queues(adapter->netdev, pairs);
1671         return err;
1672 }
1673
1674 /**
1675  * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands
1676  * @adapter: board private structure
1677  *
1678  * Return 0 on success, negative on failure
1679  **/
1680 static int iavf_config_rss_aq(struct iavf_adapter *adapter)
1681 {
1682         struct iavf_aqc_get_set_rss_key_data *rss_key =
1683                 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;
1684         struct iavf_hw *hw = &adapter->hw;
1685         enum iavf_status status;
1686
1687         if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1688                 /* bail because we already have a command pending */
1689                 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1690                         adapter->current_op);
1691                 return -EBUSY;
1692         }
1693
1694         status = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
1695         if (status) {
1696                 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1697                         iavf_stat_str(hw, status),
1698                         iavf_aq_str(hw, hw->aq.asq_last_status));
1699                 return iavf_status_to_errno(status);
1700
1701         }
1702
1703         status = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,
1704                                      adapter->rss_lut, adapter->rss_lut_size);
1705         if (status) {
1706                 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1707                         iavf_stat_str(hw, status),
1708                         iavf_aq_str(hw, hw->aq.asq_last_status));
1709                 return iavf_status_to_errno(status);
1710         }
1711
1712         return 0;
1713
1714 }
1715
1716 /**
1717  * iavf_config_rss_reg - Configure RSS keys and lut by writing registers
1718  * @adapter: board private structure
1719  *
1720  * Returns 0 on success, negative on failure
1721  **/
1722 static int iavf_config_rss_reg(struct iavf_adapter *adapter)
1723 {
1724         struct iavf_hw *hw = &adapter->hw;
1725         u32 *dw;
1726         u16 i;
1727
1728         dw = (u32 *)adapter->rss_key;
1729         for (i = 0; i <= adapter->rss_key_size / 4; i++)
1730                 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);
1731
1732         dw = (u32 *)adapter->rss_lut;
1733         for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1734                 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);
1735
1736         iavf_flush(hw);
1737
1738         return 0;
1739 }
1740
1741 /**
1742  * iavf_config_rss - Configure RSS keys and lut
1743  * @adapter: board private structure
1744  *
1745  * Returns 0 on success, negative on failure
1746  **/
1747 int iavf_config_rss(struct iavf_adapter *adapter)
1748 {
1749
1750         if (RSS_PF(adapter)) {
1751                 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
1752                                         IAVF_FLAG_AQ_SET_RSS_KEY;
1753                 return 0;
1754         } else if (RSS_AQ(adapter)) {
1755                 return iavf_config_rss_aq(adapter);
1756         } else {
1757                 return iavf_config_rss_reg(adapter);
1758         }
1759 }
1760
1761 /**
1762  * iavf_fill_rss_lut - Fill the lut with default values
1763  * @adapter: board private structure
1764  **/
1765 static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
1766 {
1767         u16 i;
1768
1769         for (i = 0; i < adapter->rss_lut_size; i++)
1770                 adapter->rss_lut[i] = i % adapter->num_active_queues;
1771 }
1772
1773 /**
1774  * iavf_init_rss - Prepare for RSS
1775  * @adapter: board private structure
1776  *
1777  * Return 0 on success, negative on failure
1778  **/
1779 static int iavf_init_rss(struct iavf_adapter *adapter)
1780 {
1781         struct iavf_hw *hw = &adapter->hw;
1782
1783         if (!RSS_PF(adapter)) {
1784                 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1785                 if (adapter->vf_res->vf_cap_flags &
1786                     VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1787                         adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;
1788                 else
1789                         adapter->hena = IAVF_DEFAULT_RSS_HENA;
1790
1791                 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);
1792                 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1793         }
1794
1795         iavf_fill_rss_lut(adapter);
1796         netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
1797
1798         return iavf_config_rss(adapter);
1799 }
1800
1801 /**
1802  * iavf_alloc_q_vectors - Allocate memory for interrupt vectors
1803  * @adapter: board private structure to initialize
1804  *
1805  * We allocate one q_vector per queue interrupt.  If allocation fails we
1806  * return -ENOMEM.
1807  **/
1808 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
1809 {
1810         int q_idx = 0, num_q_vectors;
1811         struct iavf_q_vector *q_vector;
1812
1813         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1814         adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
1815                                      GFP_KERNEL);
1816         if (!adapter->q_vectors)
1817                 return -ENOMEM;
1818
1819         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1820                 q_vector = &adapter->q_vectors[q_idx];
1821                 q_vector->adapter = adapter;
1822                 q_vector->vsi = &adapter->vsi;
1823                 q_vector->v_idx = q_idx;
1824                 q_vector->reg_idx = q_idx;
1825                 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
1826                 netif_napi_add(adapter->netdev, &q_vector->napi,
1827                                iavf_napi_poll);
1828         }
1829
1830         return 0;
1831 }
1832
1833 /**
1834  * iavf_free_q_vectors - Free memory allocated for interrupt vectors
1835  * @adapter: board private structure to initialize
1836  *
1837  * This function frees the memory allocated to the q_vectors.  In addition if
1838  * NAPI is enabled it will delete any references to the NAPI struct prior
1839  * to freeing the q_vector.
1840  **/
1841 static void iavf_free_q_vectors(struct iavf_adapter *adapter)
1842 {
1843         int q_idx, num_q_vectors;
1844         int napi_vectors;
1845
1846         if (!adapter->q_vectors)
1847                 return;
1848
1849         num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1850         napi_vectors = adapter->num_active_queues;
1851
1852         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1853                 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];
1854
1855                 if (q_idx < napi_vectors)
1856                         netif_napi_del(&q_vector->napi);
1857         }
1858         kfree(adapter->q_vectors);
1859         adapter->q_vectors = NULL;
1860 }
1861
1862 /**
1863  * iavf_reset_interrupt_capability - Reset MSIX setup
1864  * @adapter: board private structure
1865  *
1866  **/
1867 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
1868 {
1869         if (!adapter->msix_entries)
1870                 return;
1871
1872         pci_disable_msix(adapter->pdev);
1873         kfree(adapter->msix_entries);
1874         adapter->msix_entries = NULL;
1875 }
1876
1877 /**
1878  * iavf_init_interrupt_scheme - Determine if MSIX is supported and init
1879  * @adapter: board private structure to initialize
1880  *
1881  **/
1882 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
1883 {
1884         int err;
1885
1886         err = iavf_alloc_queues(adapter);
1887         if (err) {
1888                 dev_err(&adapter->pdev->dev,
1889                         "Unable to allocate memory for queues\n");
1890                 goto err_alloc_queues;
1891         }
1892
1893         rtnl_lock();
1894         err = iavf_set_interrupt_capability(adapter);
1895         rtnl_unlock();
1896         if (err) {
1897                 dev_err(&adapter->pdev->dev,
1898                         "Unable to setup interrupt capabilities\n");
1899                 goto err_set_interrupt;
1900         }
1901
1902         err = iavf_alloc_q_vectors(adapter);
1903         if (err) {
1904                 dev_err(&adapter->pdev->dev,
1905                         "Unable to allocate memory for queue vectors\n");
1906                 goto err_alloc_q_vectors;
1907         }
1908
1909         /* If we've made it so far while ADq flag being ON, then we haven't
1910          * bailed out anywhere in middle. And ADq isn't just enabled but actual
1911          * resources have been allocated in the reset path.
1912          * Now we can truly claim that ADq is enabled.
1913          */
1914         if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1915             adapter->num_tc)
1916                 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1917                          adapter->num_tc);
1918
1919         dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1920                  (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1921                  adapter->num_active_queues);
1922
1923         return 0;
1924 err_alloc_q_vectors:
1925         iavf_reset_interrupt_capability(adapter);
1926 err_set_interrupt:
1927         iavf_free_queues(adapter);
1928 err_alloc_queues:
1929         return err;
1930 }
1931
1932 /**
1933  * iavf_free_rss - Free memory used by RSS structs
1934  * @adapter: board private structure
1935  **/
1936 static void iavf_free_rss(struct iavf_adapter *adapter)
1937 {
1938         kfree(adapter->rss_key);
1939         adapter->rss_key = NULL;
1940
1941         kfree(adapter->rss_lut);
1942         adapter->rss_lut = NULL;
1943 }
1944
1945 /**
1946  * iavf_reinit_interrupt_scheme - Reallocate queues and vectors
1947  * @adapter: board private structure
1948  *
1949  * Returns 0 on success, negative on failure
1950  **/
1951 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter)
1952 {
1953         struct net_device *netdev = adapter->netdev;
1954         int err;
1955
1956         if (netif_running(netdev))
1957                 iavf_free_traffic_irqs(adapter);
1958         iavf_free_misc_irq(adapter);
1959         iavf_reset_interrupt_capability(adapter);
1960         iavf_free_q_vectors(adapter);
1961         iavf_free_queues(adapter);
1962
1963         err =  iavf_init_interrupt_scheme(adapter);
1964         if (err)
1965                 goto err;
1966
1967         netif_tx_stop_all_queues(netdev);
1968
1969         err = iavf_request_misc_irq(adapter);
1970         if (err)
1971                 goto err;
1972
1973         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1974
1975         iavf_map_rings_to_vectors(adapter);
1976 err:
1977         return err;
1978 }
1979
1980 /**
1981  * iavf_process_aq_command - process aq_required flags
1982  * and sends aq command
1983  * @adapter: pointer to iavf adapter structure
1984  *
1985  * Returns 0 on success
1986  * Returns error code if no command was sent
1987  * or error code if the command failed.
1988  **/
1989 static int iavf_process_aq_command(struct iavf_adapter *adapter)
1990 {
1991         if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)
1992                 return iavf_send_vf_config_msg(adapter);
1993         if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS)
1994                 return iavf_send_vf_offload_vlan_v2_msg(adapter);
1995         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
1996                 iavf_disable_queues(adapter);
1997                 return 0;
1998         }
1999
2000         if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
2001                 iavf_map_queues(adapter);
2002                 return 0;
2003         }
2004
2005         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
2006                 iavf_add_ether_addrs(adapter);
2007                 return 0;
2008         }
2009
2010         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
2011                 iavf_add_vlans(adapter);
2012                 return 0;
2013         }
2014
2015         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
2016                 iavf_del_ether_addrs(adapter);
2017                 return 0;
2018         }
2019
2020         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
2021                 iavf_del_vlans(adapter);
2022                 return 0;
2023         }
2024
2025         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
2026                 iavf_enable_vlan_stripping(adapter);
2027                 return 0;
2028         }
2029
2030         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
2031                 iavf_disable_vlan_stripping(adapter);
2032                 return 0;
2033         }
2034
2035         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
2036                 iavf_configure_queues(adapter);
2037                 return 0;
2038         }
2039
2040         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
2041                 iavf_enable_queues(adapter);
2042                 return 0;
2043         }
2044
2045         if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
2046                 /* This message goes straight to the firmware, not the
2047                  * PF, so we don't have to set current_op as we will
2048                  * not get a response through the ARQ.
2049                  */
2050                 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
2051                 return 0;
2052         }
2053         if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
2054                 iavf_get_hena(adapter);
2055                 return 0;
2056         }
2057         if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
2058                 iavf_set_hena(adapter);
2059                 return 0;
2060         }
2061         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
2062                 iavf_set_rss_key(adapter);
2063                 return 0;
2064         }
2065         if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
2066                 iavf_set_rss_lut(adapter);
2067                 return 0;
2068         }
2069
2070         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) {
2071                 iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC |
2072                                        FLAG_VF_MULTICAST_PROMISC);
2073                 return 0;
2074         }
2075
2076         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) {
2077                 iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC);
2078                 return 0;
2079         }
2080         if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) ||
2081             (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) {
2082                 iavf_set_promiscuous(adapter, 0);
2083                 return 0;
2084         }
2085
2086         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
2087                 iavf_enable_channels(adapter);
2088                 return 0;
2089         }
2090
2091         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
2092                 iavf_disable_channels(adapter);
2093                 return 0;
2094         }
2095         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
2096                 iavf_add_cloud_filter(adapter);
2097                 return 0;
2098         }
2099
2100         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
2101                 iavf_del_cloud_filter(adapter);
2102                 return 0;
2103         }
2104         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
2105                 iavf_del_cloud_filter(adapter);
2106                 return 0;
2107         }
2108         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
2109                 iavf_add_cloud_filter(adapter);
2110                 return 0;
2111         }
2112         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {
2113                 iavf_add_fdir_filter(adapter);
2114                 return IAVF_SUCCESS;
2115         }
2116         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {
2117                 iavf_del_fdir_filter(adapter);
2118                 return IAVF_SUCCESS;
2119         }
2120         if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {
2121                 iavf_add_adv_rss_cfg(adapter);
2122                 return 0;
2123         }
2124         if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {
2125                 iavf_del_adv_rss_cfg(adapter);
2126                 return 0;
2127         }
2128         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) {
2129                 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2130                 return 0;
2131         }
2132         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) {
2133                 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2134                 return 0;
2135         }
2136         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) {
2137                 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2138                 return 0;
2139         }
2140         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) {
2141                 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2142                 return 0;
2143         }
2144         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) {
2145                 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2146                 return 0;
2147         }
2148         if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) {
2149                 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2150                 return 0;
2151         }
2152         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) {
2153                 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2154                 return 0;
2155         }
2156         if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) {
2157                 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2158                 return 0;
2159         }
2160
2161         if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) {
2162                 iavf_request_stats(adapter);
2163                 return 0;
2164         }
2165
2166         return -EAGAIN;
2167 }
2168
2169 /**
2170  * iavf_set_vlan_offload_features - set VLAN offload configuration
2171  * @adapter: board private structure
2172  * @prev_features: previous features used for comparison
2173  * @features: updated features used for configuration
2174  *
2175  * Set the aq_required bit(s) based on the requested features passed in to
2176  * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule
2177  * the watchdog if any changes are requested to expedite the request via
2178  * virtchnl.
2179  **/
2180 void
2181 iavf_set_vlan_offload_features(struct iavf_adapter *adapter,
2182                                netdev_features_t prev_features,
2183                                netdev_features_t features)
2184 {
2185         bool enable_stripping = true, enable_insertion = true;
2186         u16 vlan_ethertype = 0;
2187         u64 aq_required = 0;
2188
2189         /* keep cases separate because one ethertype for offloads can be
2190          * disabled at the same time as another is disabled, so check for an
2191          * enabled ethertype first, then check for disabled. Default to
2192          * ETH_P_8021Q so an ethertype is specified if disabling insertion and
2193          * stripping.
2194          */
2195         if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2196                 vlan_ethertype = ETH_P_8021AD;
2197         else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2198                 vlan_ethertype = ETH_P_8021Q;
2199         else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2200                 vlan_ethertype = ETH_P_8021AD;
2201         else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2202                 vlan_ethertype = ETH_P_8021Q;
2203         else
2204                 vlan_ethertype = ETH_P_8021Q;
2205
2206         if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
2207                 enable_stripping = false;
2208         if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
2209                 enable_insertion = false;
2210
2211         if (VLAN_ALLOWED(adapter)) {
2212                 /* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN
2213                  * stripping via virtchnl. VLAN insertion can be toggled on the
2214                  * netdev, but it doesn't require a virtchnl message
2215                  */
2216                 if (enable_stripping)
2217                         aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
2218                 else
2219                         aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
2220
2221         } else if (VLAN_V2_ALLOWED(adapter)) {
2222                 switch (vlan_ethertype) {
2223                 case ETH_P_8021Q:
2224                         if (enable_stripping)
2225                                 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING;
2226                         else
2227                                 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING;
2228
2229                         if (enable_insertion)
2230                                 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION;
2231                         else
2232                                 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION;
2233                         break;
2234                 case ETH_P_8021AD:
2235                         if (enable_stripping)
2236                                 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING;
2237                         else
2238                                 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING;
2239
2240                         if (enable_insertion)
2241                                 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION;
2242                         else
2243                                 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION;
2244                         break;
2245                 }
2246         }
2247
2248         if (aq_required) {
2249                 adapter->aq_required |= aq_required;
2250                 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
2251         }
2252 }
2253
2254 /**
2255  * iavf_startup - first step of driver startup
2256  * @adapter: board private structure
2257  *
2258  * Function process __IAVF_STARTUP driver state.
2259  * When success the state is changed to __IAVF_INIT_VERSION_CHECK
2260  * when fails the state is changed to __IAVF_INIT_FAILED
2261  **/
2262 static void iavf_startup(struct iavf_adapter *adapter)
2263 {
2264         struct pci_dev *pdev = adapter->pdev;
2265         struct iavf_hw *hw = &adapter->hw;
2266         enum iavf_status status;
2267         int ret;
2268
2269         WARN_ON(adapter->state != __IAVF_STARTUP);
2270
2271         /* driver loaded, probe complete */
2272         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2273         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2274         status = iavf_set_mac_type(hw);
2275         if (status) {
2276                 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", status);
2277                 goto err;
2278         }
2279
2280         ret = iavf_check_reset_complete(hw);
2281         if (ret) {
2282                 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
2283                          ret);
2284                 goto err;
2285         }
2286         hw->aq.num_arq_entries = IAVF_AQ_LEN;
2287         hw->aq.num_asq_entries = IAVF_AQ_LEN;
2288         hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2289         hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2290
2291         status = iavf_init_adminq(hw);
2292         if (status) {
2293                 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
2294                         status);
2295                 goto err;
2296         }
2297         ret = iavf_send_api_ver(adapter);
2298         if (ret) {
2299                 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret);
2300                 iavf_shutdown_adminq(hw);
2301                 goto err;
2302         }
2303         iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK);
2304         return;
2305 err:
2306         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2307 }
2308
2309 /**
2310  * iavf_init_version_check - second step of driver startup
2311  * @adapter: board private structure
2312  *
2313  * Function process __IAVF_INIT_VERSION_CHECK driver state.
2314  * When success the state is changed to __IAVF_INIT_GET_RESOURCES
2315  * when fails the state is changed to __IAVF_INIT_FAILED
2316  **/
2317 static void iavf_init_version_check(struct iavf_adapter *adapter)
2318 {
2319         struct pci_dev *pdev = adapter->pdev;
2320         struct iavf_hw *hw = &adapter->hw;
2321         int err = -EAGAIN;
2322
2323         WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);
2324
2325         if (!iavf_asq_done(hw)) {
2326                 dev_err(&pdev->dev, "Admin queue command never completed\n");
2327                 iavf_shutdown_adminq(hw);
2328                 iavf_change_state(adapter, __IAVF_STARTUP);
2329                 goto err;
2330         }
2331
2332         /* aq msg sent, awaiting reply */
2333         err = iavf_verify_api_ver(adapter);
2334         if (err) {
2335                 if (err == -EALREADY)
2336                         err = iavf_send_api_ver(adapter);
2337                 else
2338                         dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
2339                                 adapter->pf_version.major,
2340                                 adapter->pf_version.minor,
2341                                 VIRTCHNL_VERSION_MAJOR,
2342                                 VIRTCHNL_VERSION_MINOR);
2343                 goto err;
2344         }
2345         err = iavf_send_vf_config_msg(adapter);
2346         if (err) {
2347                 dev_err(&pdev->dev, "Unable to send config request (%d)\n",
2348                         err);
2349                 goto err;
2350         }
2351         iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES);
2352         return;
2353 err:
2354         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2355 }
2356
2357 /**
2358  * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES
2359  * @adapter: board private structure
2360  */
2361 int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter)
2362 {
2363         int i, num_req_queues = adapter->num_req_queues;
2364         struct iavf_vsi *vsi = &adapter->vsi;
2365
2366         for (i = 0; i < adapter->vf_res->num_vsis; i++) {
2367                 if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
2368                         adapter->vsi_res = &adapter->vf_res->vsi_res[i];
2369         }
2370         if (!adapter->vsi_res) {
2371                 dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
2372                 return -ENODEV;
2373         }
2374
2375         if (num_req_queues &&
2376             num_req_queues > adapter->vsi_res->num_queue_pairs) {
2377                 /* Problem.  The PF gave us fewer queues than what we had
2378                  * negotiated in our request.  Need a reset to see if we can't
2379                  * get back to a working state.
2380                  */
2381                 dev_err(&adapter->pdev->dev,
2382                         "Requested %d queues, but PF only gave us %d.\n",
2383                         num_req_queues,
2384                         adapter->vsi_res->num_queue_pairs);
2385                 adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED;
2386                 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
2387                 iavf_schedule_reset(adapter);
2388
2389                 return -EAGAIN;
2390         }
2391         adapter->num_req_queues = 0;
2392         adapter->vsi.id = adapter->vsi_res->vsi_id;
2393
2394         adapter->vsi.back = adapter;
2395         adapter->vsi.base_vector = 1;
2396         vsi->netdev = adapter->netdev;
2397         vsi->qs_handle = adapter->vsi_res->qset_handle;
2398         if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
2399                 adapter->rss_key_size = adapter->vf_res->rss_key_size;
2400                 adapter->rss_lut_size = adapter->vf_res->rss_lut_size;
2401         } else {
2402                 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
2403                 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
2404         }
2405
2406         return 0;
2407 }
2408
2409 /**
2410  * iavf_init_get_resources - third step of driver startup
2411  * @adapter: board private structure
2412  *
2413  * Function process __IAVF_INIT_GET_RESOURCES driver state and
2414  * finishes driver initialization procedure.
2415  * When success the state is changed to __IAVF_DOWN
2416  * when fails the state is changed to __IAVF_INIT_FAILED
2417  **/
2418 static void iavf_init_get_resources(struct iavf_adapter *adapter)
2419 {
2420         struct pci_dev *pdev = adapter->pdev;
2421         struct iavf_hw *hw = &adapter->hw;
2422         int err;
2423
2424         WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
2425         /* aq msg sent, awaiting reply */
2426         if (!adapter->vf_res) {
2427                 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
2428                                           GFP_KERNEL);
2429                 if (!adapter->vf_res) {
2430                         err = -ENOMEM;
2431                         goto err;
2432                 }
2433         }
2434         err = iavf_get_vf_config(adapter);
2435         if (err == -EALREADY) {
2436                 err = iavf_send_vf_config_msg(adapter);
2437                 goto err;
2438         } else if (err == -EINVAL) {
2439                 /* We only get -EINVAL if the device is in a very bad
2440                  * state or if we've been disabled for previous bad
2441                  * behavior. Either way, we're done now.
2442                  */
2443                 iavf_shutdown_adminq(hw);
2444                 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
2445                 return;
2446         }
2447         if (err) {
2448                 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
2449                 goto err_alloc;
2450         }
2451
2452         err = iavf_parse_vf_resource_msg(adapter);
2453         if (err) {
2454                 dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n",
2455                         err);
2456                 goto err_alloc;
2457         }
2458         /* Some features require additional messages to negotiate extended
2459          * capabilities. These are processed in sequence by the
2460          * __IAVF_INIT_EXTENDED_CAPS driver state.
2461          */
2462         adapter->extended_caps = IAVF_EXTENDED_CAPS;
2463
2464         iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS);
2465         return;
2466
2467 err_alloc:
2468         kfree(adapter->vf_res);
2469         adapter->vf_res = NULL;
2470 err:
2471         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2472 }
2473
2474 /**
2475  * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2476  * @adapter: board private structure
2477  *
2478  * Function processes send of the extended VLAN V2 capability message to the
2479  * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent,
2480  * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2.
2481  */
2482 static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2483 {
2484         int ret;
2485
2486         WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2));
2487
2488         ret = iavf_send_vf_offload_vlan_v2_msg(adapter);
2489         if (ret && ret == -EOPNOTSUPP) {
2490                 /* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case,
2491                  * we did not send the capability exchange message and do not
2492                  * expect a response.
2493                  */
2494                 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2495         }
2496
2497         /* We sent the message, so move on to the next step */
2498         adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2499 }
2500
2501 /**
2502  * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2503  * @adapter: board private structure
2504  *
2505  * Function processes receipt of the extended VLAN V2 capability message from
2506  * the PF.
2507  **/
2508 static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2509 {
2510         int ret;
2511
2512         WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2));
2513
2514         memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps));
2515
2516         ret = iavf_get_vf_vlan_v2_caps(adapter);
2517         if (ret)
2518                 goto err;
2519
2520         /* We've processed receipt of the VLAN V2 caps message */
2521         adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2522         return;
2523 err:
2524         /* We didn't receive a reply. Make sure we try sending again when
2525          * __IAVF_INIT_FAILED attempts to recover.
2526          */
2527         adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2528         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2529 }
2530
2531 /**
2532  * iavf_init_process_extended_caps - Part of driver startup
2533  * @adapter: board private structure
2534  *
2535  * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state
2536  * handles negotiating capabilities for features which require an additional
2537  * message.
2538  *
2539  * Once all extended capabilities exchanges are finished, the driver will
2540  * transition into __IAVF_INIT_CONFIG_ADAPTER.
2541  */
2542 static void iavf_init_process_extended_caps(struct iavf_adapter *adapter)
2543 {
2544         WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS);
2545
2546         /* Process capability exchange for VLAN V2 */
2547         if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) {
2548                 iavf_init_send_offload_vlan_v2_caps(adapter);
2549                 return;
2550         } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) {
2551                 iavf_init_recv_offload_vlan_v2_caps(adapter);
2552                 return;
2553         }
2554
2555         /* When we reach here, no further extended capabilities exchanges are
2556          * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER
2557          */
2558         iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER);
2559 }
2560
2561 /**
2562  * iavf_init_config_adapter - last part of driver startup
2563  * @adapter: board private structure
2564  *
2565  * After all the supported capabilities are negotiated, then the
2566  * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization.
2567  */
2568 static void iavf_init_config_adapter(struct iavf_adapter *adapter)
2569 {
2570         struct net_device *netdev = adapter->netdev;
2571         struct pci_dev *pdev = adapter->pdev;
2572         int err;
2573
2574         WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER);
2575
2576         if (iavf_process_config(adapter))
2577                 goto err;
2578
2579         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2580
2581         adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
2582
2583         netdev->netdev_ops = &iavf_netdev_ops;
2584         iavf_set_ethtool_ops(netdev);
2585         netdev->watchdog_timeo = 5 * HZ;
2586
2587         /* MTU range: 68 - 9710 */
2588         netdev->min_mtu = ETH_MIN_MTU;
2589         netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD;
2590
2591         if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
2592                 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
2593                          adapter->hw.mac.addr);
2594                 eth_hw_addr_random(netdev);
2595                 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
2596         } else {
2597                 eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2598                 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
2599         }
2600
2601         adapter->tx_desc_count = IAVF_DEFAULT_TXD;
2602         adapter->rx_desc_count = IAVF_DEFAULT_RXD;
2603         err = iavf_init_interrupt_scheme(adapter);
2604         if (err)
2605                 goto err_sw_init;
2606         iavf_map_rings_to_vectors(adapter);
2607         if (adapter->vf_res->vf_cap_flags &
2608                 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
2609                 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
2610
2611         err = iavf_request_misc_irq(adapter);
2612         if (err)
2613                 goto err_sw_init;
2614
2615         netif_carrier_off(netdev);
2616         adapter->link_up = false;
2617
2618         /* set the semaphore to prevent any callbacks after device registration
2619          * up to time when state of driver will be set to __IAVF_DOWN
2620          */
2621         rtnl_lock();
2622         if (!adapter->netdev_registered) {
2623                 err = register_netdevice(netdev);
2624                 if (err) {
2625                         rtnl_unlock();
2626                         goto err_register;
2627                 }
2628         }
2629
2630         adapter->netdev_registered = true;
2631
2632         netif_tx_stop_all_queues(netdev);
2633         if (CLIENT_ALLOWED(adapter)) {
2634                 err = iavf_lan_add_device(adapter);
2635                 if (err)
2636                         dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n",
2637                                  err);
2638         }
2639         dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
2640         if (netdev->features & NETIF_F_GRO)
2641                 dev_info(&pdev->dev, "GRO is enabled\n");
2642
2643         iavf_change_state(adapter, __IAVF_DOWN);
2644         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2645         rtnl_unlock();
2646
2647         iavf_misc_irq_enable(adapter);
2648         wake_up(&adapter->down_waitqueue);
2649
2650         adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
2651         adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
2652         if (!adapter->rss_key || !adapter->rss_lut) {
2653                 err = -ENOMEM;
2654                 goto err_mem;
2655         }
2656         if (RSS_AQ(adapter))
2657                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
2658         else
2659                 iavf_init_rss(adapter);
2660
2661         if (VLAN_V2_ALLOWED(adapter))
2662                 /* request initial VLAN offload settings */
2663                 iavf_set_vlan_offload_features(adapter, 0, netdev->features);
2664
2665         return;
2666 err_mem:
2667         iavf_free_rss(adapter);
2668 err_register:
2669         iavf_free_misc_irq(adapter);
2670 err_sw_init:
2671         iavf_reset_interrupt_capability(adapter);
2672 err:
2673         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2674 }
2675
2676 /**
2677  * iavf_watchdog_task - Periodic call-back task
2678  * @work: pointer to work_struct
2679  **/
2680 static void iavf_watchdog_task(struct work_struct *work)
2681 {
2682         struct iavf_adapter *adapter = container_of(work,
2683                                                     struct iavf_adapter,
2684                                                     watchdog_task.work);
2685         struct iavf_hw *hw = &adapter->hw;
2686         u32 reg_val;
2687
2688         if (!mutex_trylock(&adapter->crit_lock)) {
2689                 if (adapter->state == __IAVF_REMOVE)
2690                         return;
2691
2692                 goto restart_watchdog;
2693         }
2694
2695         if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES) &&
2696             adapter->netdev_registered &&
2697             !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section) &&
2698             rtnl_trylock()) {
2699                 netdev_update_features(adapter->netdev);
2700                 rtnl_unlock();
2701                 adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES;
2702         }
2703
2704         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
2705                 iavf_change_state(adapter, __IAVF_COMM_FAILED);
2706
2707         if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
2708                 adapter->aq_required = 0;
2709                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2710                 mutex_unlock(&adapter->crit_lock);
2711                 queue_work(adapter->wq, &adapter->reset_task);
2712                 return;
2713         }
2714
2715         switch (adapter->state) {
2716         case __IAVF_STARTUP:
2717                 iavf_startup(adapter);
2718                 mutex_unlock(&adapter->crit_lock);
2719                 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2720                                    msecs_to_jiffies(30));
2721                 return;
2722         case __IAVF_INIT_VERSION_CHECK:
2723                 iavf_init_version_check(adapter);
2724                 mutex_unlock(&adapter->crit_lock);
2725                 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2726                                    msecs_to_jiffies(30));
2727                 return;
2728         case __IAVF_INIT_GET_RESOURCES:
2729                 iavf_init_get_resources(adapter);
2730                 mutex_unlock(&adapter->crit_lock);
2731                 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2732                                    msecs_to_jiffies(1));
2733                 return;
2734         case __IAVF_INIT_EXTENDED_CAPS:
2735                 iavf_init_process_extended_caps(adapter);
2736                 mutex_unlock(&adapter->crit_lock);
2737                 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2738                                    msecs_to_jiffies(1));
2739                 return;
2740         case __IAVF_INIT_CONFIG_ADAPTER:
2741                 iavf_init_config_adapter(adapter);
2742                 mutex_unlock(&adapter->crit_lock);
2743                 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2744                                    msecs_to_jiffies(1));
2745                 return;
2746         case __IAVF_INIT_FAILED:
2747                 if (test_bit(__IAVF_IN_REMOVE_TASK,
2748                              &adapter->crit_section)) {
2749                         /* Do not update the state and do not reschedule
2750                          * watchdog task, iavf_remove should handle this state
2751                          * as it can loop forever
2752                          */
2753                         mutex_unlock(&adapter->crit_lock);
2754                         return;
2755                 }
2756                 if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
2757                         dev_err(&adapter->pdev->dev,
2758                                 "Failed to communicate with PF; waiting before retry\n");
2759                         adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2760                         iavf_shutdown_adminq(hw);
2761                         mutex_unlock(&adapter->crit_lock);
2762                         queue_delayed_work(adapter->wq,
2763                                            &adapter->watchdog_task, (5 * HZ));
2764                         return;
2765                 }
2766                 /* Try again from failed step*/
2767                 iavf_change_state(adapter, adapter->last_state);
2768                 mutex_unlock(&adapter->crit_lock);
2769                 queue_delayed_work(adapter->wq, &adapter->watchdog_task, HZ);
2770                 return;
2771         case __IAVF_COMM_FAILED:
2772                 if (test_bit(__IAVF_IN_REMOVE_TASK,
2773                              &adapter->crit_section)) {
2774                         /* Set state to __IAVF_INIT_FAILED and perform remove
2775                          * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task
2776                          * doesn't bring the state back to __IAVF_COMM_FAILED.
2777                          */
2778                         iavf_change_state(adapter, __IAVF_INIT_FAILED);
2779                         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2780                         mutex_unlock(&adapter->crit_lock);
2781                         return;
2782                 }
2783                 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
2784                           IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
2785                 if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
2786                     reg_val == VIRTCHNL_VFR_COMPLETED) {
2787                         /* A chance for redemption! */
2788                         dev_err(&adapter->pdev->dev,
2789                                 "Hardware came out of reset. Attempting reinit.\n");
2790                         /* When init task contacts the PF and
2791                          * gets everything set up again, it'll restart the
2792                          * watchdog for us. Down, boy. Sit. Stay. Woof.
2793                          */
2794                         iavf_change_state(adapter, __IAVF_STARTUP);
2795                         adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2796                 }
2797                 adapter->aq_required = 0;
2798                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2799                 mutex_unlock(&adapter->crit_lock);
2800                 queue_delayed_work(adapter->wq,
2801                                    &adapter->watchdog_task,
2802                                    msecs_to_jiffies(10));
2803                 return;
2804         case __IAVF_RESETTING:
2805                 mutex_unlock(&adapter->crit_lock);
2806                 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2807                                    HZ * 2);
2808                 return;
2809         case __IAVF_DOWN:
2810         case __IAVF_DOWN_PENDING:
2811         case __IAVF_TESTING:
2812         case __IAVF_RUNNING:
2813                 if (adapter->current_op) {
2814                         if (!iavf_asq_done(hw)) {
2815                                 dev_dbg(&adapter->pdev->dev,
2816                                         "Admin queue timeout\n");
2817                                 iavf_send_api_ver(adapter);
2818                         }
2819                 } else {
2820                         int ret = iavf_process_aq_command(adapter);
2821
2822                         /* An error will be returned if no commands were
2823                          * processed; use this opportunity to update stats
2824                          * if the error isn't -ENOTSUPP
2825                          */
2826                         if (ret && ret != -EOPNOTSUPP &&
2827                             adapter->state == __IAVF_RUNNING)
2828                                 iavf_request_stats(adapter);
2829                 }
2830                 if (adapter->state == __IAVF_RUNNING)
2831                         iavf_detect_recover_hung(&adapter->vsi);
2832                 break;
2833         case __IAVF_REMOVE:
2834         default:
2835                 mutex_unlock(&adapter->crit_lock);
2836                 return;
2837         }
2838
2839         /* check for hw reset */
2840         reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2841         if (!reg_val) {
2842                 adapter->flags |= IAVF_FLAG_RESET_PENDING;
2843                 adapter->aq_required = 0;
2844                 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2845                 dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
2846                 queue_work(adapter->wq, &adapter->reset_task);
2847                 mutex_unlock(&adapter->crit_lock);
2848                 queue_delayed_work(adapter->wq,
2849                                    &adapter->watchdog_task, HZ * 2);
2850                 return;
2851         }
2852
2853         schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
2854         mutex_unlock(&adapter->crit_lock);
2855 restart_watchdog:
2856         if (adapter->state >= __IAVF_DOWN)
2857                 queue_work(adapter->wq, &adapter->adminq_task);
2858         if (adapter->aq_required)
2859                 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2860                                    msecs_to_jiffies(20));
2861         else
2862                 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2863                                    HZ * 2);
2864 }
2865
2866 /**
2867  * iavf_disable_vf - disable VF
2868  * @adapter: board private structure
2869  *
2870  * Set communication failed flag and free all resources.
2871  * NOTE: This function is expected to be called with crit_lock being held.
2872  **/
2873 static void iavf_disable_vf(struct iavf_adapter *adapter)
2874 {
2875         struct iavf_mac_filter *f, *ftmp;
2876         struct iavf_vlan_filter *fv, *fvtmp;
2877         struct iavf_cloud_filter *cf, *cftmp;
2878
2879         adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2880
2881         /* We don't use netif_running() because it may be true prior to
2882          * ndo_open() returning, so we can't assume it means all our open
2883          * tasks have finished, since we're not holding the rtnl_lock here.
2884          */
2885         if (adapter->state == __IAVF_RUNNING) {
2886                 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2887                 netif_carrier_off(adapter->netdev);
2888                 netif_tx_disable(adapter->netdev);
2889                 adapter->link_up = false;
2890                 iavf_napi_disable_all(adapter);
2891                 iavf_irq_disable(adapter);
2892                 iavf_free_traffic_irqs(adapter);
2893                 iavf_free_all_tx_resources(adapter);
2894                 iavf_free_all_rx_resources(adapter);
2895         }
2896
2897         spin_lock_bh(&adapter->mac_vlan_list_lock);
2898
2899         /* Delete all of the filters */
2900         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2901                 list_del(&f->list);
2902                 kfree(f);
2903         }
2904
2905         list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
2906                 list_del(&fv->list);
2907                 kfree(fv);
2908         }
2909
2910         spin_unlock_bh(&adapter->mac_vlan_list_lock);
2911
2912         spin_lock_bh(&adapter->cloud_filter_list_lock);
2913         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
2914                 list_del(&cf->list);
2915                 kfree(cf);
2916                 adapter->num_cloud_filters--;
2917         }
2918         spin_unlock_bh(&adapter->cloud_filter_list_lock);
2919
2920         iavf_free_misc_irq(adapter);
2921         iavf_reset_interrupt_capability(adapter);
2922         iavf_free_q_vectors(adapter);
2923         iavf_free_queues(adapter);
2924         memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
2925         iavf_shutdown_adminq(&adapter->hw);
2926         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2927         iavf_change_state(adapter, __IAVF_DOWN);
2928         wake_up(&adapter->down_waitqueue);
2929         dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
2930 }
2931
2932 /**
2933  * iavf_reset_task - Call-back task to handle hardware reset
2934  * @work: pointer to work_struct
2935  *
2936  * During reset we need to shut down and reinitialize the admin queue
2937  * before we can use it to communicate with the PF again. We also clear
2938  * and reinit the rings because that context is lost as well.
2939  **/
2940 static void iavf_reset_task(struct work_struct *work)
2941 {
2942         struct iavf_adapter *adapter = container_of(work,
2943                                                       struct iavf_adapter,
2944                                                       reset_task);
2945         struct virtchnl_vf_resource *vfres = adapter->vf_res;
2946         struct net_device *netdev = adapter->netdev;
2947         struct iavf_hw *hw = &adapter->hw;
2948         struct iavf_mac_filter *f, *ftmp;
2949         struct iavf_cloud_filter *cf;
2950         enum iavf_status status;
2951         u32 reg_val;
2952         int i = 0, err;
2953         bool running;
2954
2955         /* Detach interface to avoid subsequent NDO callbacks */
2956         rtnl_lock();
2957         netif_device_detach(netdev);
2958         rtnl_unlock();
2959
2960         /* When device is being removed it doesn't make sense to run the reset
2961          * task, just return in such a case.
2962          */
2963         if (!mutex_trylock(&adapter->crit_lock)) {
2964                 if (adapter->state != __IAVF_REMOVE)
2965                         queue_work(adapter->wq, &adapter->reset_task);
2966
2967                 goto reset_finish;
2968         }
2969
2970         while (!mutex_trylock(&adapter->client_lock))
2971                 usleep_range(500, 1000);
2972         if (CLIENT_ENABLED(adapter)) {
2973                 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN |
2974                                     IAVF_FLAG_CLIENT_NEEDS_CLOSE |
2975                                     IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS |
2976                                     IAVF_FLAG_SERVICE_CLIENT_REQUESTED);
2977                 cancel_delayed_work_sync(&adapter->client_task);
2978                 iavf_notify_client_close(&adapter->vsi, true);
2979         }
2980         iavf_misc_irq_disable(adapter);
2981         if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
2982                 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
2983                 /* Restart the AQ here. If we have been reset but didn't
2984                  * detect it, or if the PF had to reinit, our AQ will be hosed.
2985                  */
2986                 iavf_shutdown_adminq(hw);
2987                 iavf_init_adminq(hw);
2988                 iavf_request_reset(adapter);
2989         }
2990         adapter->flags |= IAVF_FLAG_RESET_PENDING;
2991
2992         /* poll until we see the reset actually happen */
2993         for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
2994                 reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
2995                           IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2996                 if (!reg_val)
2997                         break;
2998                 usleep_range(5000, 10000);
2999         }
3000         if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
3001                 dev_info(&adapter->pdev->dev, "Never saw reset\n");
3002                 goto continue_reset; /* act like the reset happened */
3003         }
3004
3005         /* wait until the reset is complete and the PF is responding to us */
3006         for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
3007                 /* sleep first to make sure a minimum wait time is met */
3008                 msleep(IAVF_RESET_WAIT_MS);
3009
3010                 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
3011                           IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
3012                 if (reg_val == VIRTCHNL_VFR_VFACTIVE)
3013                         break;
3014         }
3015
3016         pci_set_master(adapter->pdev);
3017         pci_restore_msi_state(adapter->pdev);
3018
3019         if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
3020                 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
3021                         reg_val);
3022                 iavf_disable_vf(adapter);
3023                 mutex_unlock(&adapter->client_lock);
3024                 mutex_unlock(&adapter->crit_lock);
3025                 if (netif_running(netdev)) {
3026                         rtnl_lock();
3027                         dev_close(netdev);
3028                         rtnl_unlock();
3029                 }
3030                 return; /* Do not attempt to reinit. It's dead, Jim. */
3031         }
3032
3033 continue_reset:
3034         /* We don't use netif_running() because it may be true prior to
3035          * ndo_open() returning, so we can't assume it means all our open
3036          * tasks have finished, since we're not holding the rtnl_lock here.
3037          */
3038         running = adapter->state == __IAVF_RUNNING;
3039
3040         if (running) {
3041                 netif_carrier_off(netdev);
3042                 netif_tx_stop_all_queues(netdev);
3043                 adapter->link_up = false;
3044                 iavf_napi_disable_all(adapter);
3045         }
3046         iavf_irq_disable(adapter);
3047
3048         iavf_change_state(adapter, __IAVF_RESETTING);
3049         adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
3050
3051         /* free the Tx/Rx rings and descriptors, might be better to just
3052          * re-use them sometime in the future
3053          */
3054         iavf_free_all_rx_resources(adapter);
3055         iavf_free_all_tx_resources(adapter);
3056
3057         adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
3058         /* kill and reinit the admin queue */
3059         iavf_shutdown_adminq(hw);
3060         adapter->current_op = VIRTCHNL_OP_UNKNOWN;
3061         status = iavf_init_adminq(hw);
3062         if (status) {
3063                 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
3064                          status);
3065                 goto reset_err;
3066         }
3067         adapter->aq_required = 0;
3068
3069         if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3070             (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3071                 err = iavf_reinit_interrupt_scheme(adapter);
3072                 if (err)
3073                         goto reset_err;
3074         }
3075
3076         if (RSS_AQ(adapter)) {
3077                 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
3078         } else {
3079                 err = iavf_init_rss(adapter);
3080                 if (err)
3081                         goto reset_err;
3082         }
3083
3084         adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
3085         /* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been
3086          * sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here,
3087          * however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until
3088          * VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have
3089          * been successfully sent and negotiated
3090          */
3091         adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS;
3092         adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
3093
3094         spin_lock_bh(&adapter->mac_vlan_list_lock);
3095
3096         /* Delete filter for the current MAC address, it could have
3097          * been changed by the PF via administratively set MAC.
3098          * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.
3099          */
3100         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
3101                 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) {
3102                         list_del(&f->list);
3103                         kfree(f);
3104                 }
3105         }
3106         /* re-add all MAC filters */
3107         list_for_each_entry(f, &adapter->mac_filter_list, list) {
3108                 f->add = true;
3109         }
3110         spin_unlock_bh(&adapter->mac_vlan_list_lock);
3111
3112         /* check if TCs are running and re-add all cloud filters */
3113         spin_lock_bh(&adapter->cloud_filter_list_lock);
3114         if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
3115             adapter->num_tc) {
3116                 list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
3117                         cf->add = true;
3118                 }
3119         }
3120         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3121
3122         adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
3123         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
3124         iavf_misc_irq_enable(adapter);
3125
3126         bitmap_clear(adapter->vsi.active_cvlans, 0, VLAN_N_VID);
3127         bitmap_clear(adapter->vsi.active_svlans, 0, VLAN_N_VID);
3128
3129         mod_delayed_work(adapter->wq, &adapter->watchdog_task, 2);
3130
3131         /* We were running when the reset started, so we need to restore some
3132          * state here.
3133          */
3134         if (running) {
3135                 /* allocate transmit descriptors */
3136                 err = iavf_setup_all_tx_resources(adapter);
3137                 if (err)
3138                         goto reset_err;
3139
3140                 /* allocate receive descriptors */
3141                 err = iavf_setup_all_rx_resources(adapter);
3142                 if (err)
3143                         goto reset_err;
3144
3145                 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3146                     (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3147                         err = iavf_request_traffic_irqs(adapter, netdev->name);
3148                         if (err)
3149                                 goto reset_err;
3150
3151                         adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED;
3152                 }
3153
3154                 iavf_configure(adapter);
3155
3156                 /* iavf_up_complete() will switch device back
3157                  * to __IAVF_RUNNING
3158                  */
3159                 iavf_up_complete(adapter);
3160
3161                 iavf_irq_enable(adapter, true);
3162         } else {
3163                 iavf_change_state(adapter, __IAVF_DOWN);
3164                 wake_up(&adapter->down_waitqueue);
3165         }
3166
3167         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
3168
3169         mutex_unlock(&adapter->client_lock);
3170         mutex_unlock(&adapter->crit_lock);
3171
3172         goto reset_finish;
3173 reset_err:
3174         if (running) {
3175                 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
3176                 iavf_free_traffic_irqs(adapter);
3177         }
3178         iavf_disable_vf(adapter);
3179
3180         mutex_unlock(&adapter->client_lock);
3181         mutex_unlock(&adapter->crit_lock);
3182
3183         if (netif_running(netdev)) {
3184                 /* Close device to ensure that Tx queues will not be started
3185                  * during netif_device_attach() at the end of the reset task.
3186                  */
3187                 rtnl_lock();
3188                 dev_close(netdev);
3189                 rtnl_unlock();
3190         }
3191
3192         dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
3193 reset_finish:
3194         rtnl_lock();
3195         netif_device_attach(netdev);
3196         rtnl_unlock();
3197 }
3198
3199 /**
3200  * iavf_adminq_task - worker thread to clean the admin queue
3201  * @work: pointer to work_struct containing our data
3202  **/
3203 static void iavf_adminq_task(struct work_struct *work)
3204 {
3205         struct iavf_adapter *adapter =
3206                 container_of(work, struct iavf_adapter, adminq_task);
3207         struct iavf_hw *hw = &adapter->hw;
3208         struct iavf_arq_event_info event;
3209         enum virtchnl_ops v_op;
3210         enum iavf_status ret, v_ret;
3211         u32 val, oldval;
3212         u16 pending;
3213
3214         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
3215                 goto out;
3216
3217         if (!mutex_trylock(&adapter->crit_lock)) {
3218                 if (adapter->state == __IAVF_REMOVE)
3219                         return;
3220
3221                 queue_work(adapter->wq, &adapter->adminq_task);
3222                 goto out;
3223         }
3224
3225         event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
3226         event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
3227         if (!event.msg_buf)
3228                 goto out;
3229
3230         do {
3231                 ret = iavf_clean_arq_element(hw, &event, &pending);
3232                 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
3233                 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);
3234
3235                 if (ret || !v_op)
3236                         break; /* No event to process or error cleaning ARQ */
3237
3238                 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
3239                                          event.msg_len);
3240                 if (pending != 0)
3241                         memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
3242         } while (pending);
3243         mutex_unlock(&adapter->crit_lock);
3244
3245         if ((adapter->flags &
3246              (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) ||
3247             adapter->state == __IAVF_RESETTING)
3248                 goto freedom;
3249
3250         /* check for error indications */
3251         val = rd32(hw, hw->aq.arq.len);
3252         if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */
3253                 goto freedom;
3254         oldval = val;
3255         if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
3256                 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
3257                 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;
3258         }
3259         if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {
3260                 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
3261                 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;
3262         }
3263         if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {
3264                 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
3265                 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;
3266         }
3267         if (oldval != val)
3268                 wr32(hw, hw->aq.arq.len, val);
3269
3270         val = rd32(hw, hw->aq.asq.len);
3271         oldval = val;
3272         if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {
3273                 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
3274                 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;
3275         }
3276         if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {
3277                 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
3278                 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;
3279         }
3280         if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
3281                 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
3282                 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;
3283         }
3284         if (oldval != val)
3285                 wr32(hw, hw->aq.asq.len, val);
3286
3287 freedom:
3288         kfree(event.msg_buf);
3289 out:
3290         /* re-enable Admin queue interrupt cause */
3291         iavf_misc_irq_enable(adapter);
3292 }
3293
3294 /**
3295  * iavf_client_task - worker thread to perform client work
3296  * @work: pointer to work_struct containing our data
3297  *
3298  * This task handles client interactions. Because client calls can be
3299  * reentrant, we can't handle them in the watchdog.
3300  **/
3301 static void iavf_client_task(struct work_struct *work)
3302 {
3303         struct iavf_adapter *adapter =
3304                 container_of(work, struct iavf_adapter, client_task.work);
3305
3306         /* If we can't get the client bit, just give up. We'll be rescheduled
3307          * later.
3308          */
3309
3310         if (!mutex_trylock(&adapter->client_lock))
3311                 return;
3312
3313         if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) {
3314                 iavf_client_subtask(adapter);
3315                 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
3316                 goto out;
3317         }
3318         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) {
3319                 iavf_notify_client_l2_params(&adapter->vsi);
3320                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS;
3321                 goto out;
3322         }
3323         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) {
3324                 iavf_notify_client_close(&adapter->vsi, false);
3325                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE;
3326                 goto out;
3327         }
3328         if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) {
3329                 iavf_notify_client_open(&adapter->vsi);
3330                 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN;
3331         }
3332 out:
3333         mutex_unlock(&adapter->client_lock);
3334 }
3335
3336 /**
3337  * iavf_free_all_tx_resources - Free Tx Resources for All Queues
3338  * @adapter: board private structure
3339  *
3340  * Free all transmit software resources
3341  **/
3342 void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
3343 {
3344         int i;
3345
3346         if (!adapter->tx_rings)
3347                 return;
3348
3349         for (i = 0; i < adapter->num_active_queues; i++)
3350                 if (adapter->tx_rings[i].desc)
3351                         iavf_free_tx_resources(&adapter->tx_rings[i]);
3352 }
3353
3354 /**
3355  * iavf_setup_all_tx_resources - allocate all queues Tx resources
3356  * @adapter: board private structure
3357  *
3358  * If this function returns with an error, then it's possible one or
3359  * more of the rings is populated (while the rest are not).  It is the
3360  * callers duty to clean those orphaned rings.
3361  *
3362  * Return 0 on success, negative on failure
3363  **/
3364 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
3365 {
3366         int i, err = 0;
3367
3368         for (i = 0; i < adapter->num_active_queues; i++) {
3369                 adapter->tx_rings[i].count = adapter->tx_desc_count;
3370                 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);
3371                 if (!err)
3372                         continue;
3373                 dev_err(&adapter->pdev->dev,
3374                         "Allocation for Tx Queue %u failed\n", i);
3375                 break;
3376         }
3377
3378         return err;
3379 }
3380
3381 /**
3382  * iavf_setup_all_rx_resources - allocate all queues Rx resources
3383  * @adapter: board private structure
3384  *
3385  * If this function returns with an error, then it's possible one or
3386  * more of the rings is populated (while the rest are not).  It is the
3387  * callers duty to clean those orphaned rings.
3388  *
3389  * Return 0 on success, negative on failure
3390  **/
3391 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
3392 {
3393         int i, err = 0;
3394
3395         for (i = 0; i < adapter->num_active_queues; i++) {
3396                 adapter->rx_rings[i].count = adapter->rx_desc_count;
3397                 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
3398                 if (!err)
3399                         continue;
3400                 dev_err(&adapter->pdev->dev,
3401                         "Allocation for Rx Queue %u failed\n", i);
3402                 break;
3403         }
3404         return err;
3405 }
3406
3407 /**
3408  * iavf_free_all_rx_resources - Free Rx Resources for All Queues
3409  * @adapter: board private structure
3410  *
3411  * Free all receive software resources
3412  **/
3413 void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
3414 {
3415         int i;
3416
3417         if (!adapter->rx_rings)
3418                 return;
3419
3420         for (i = 0; i < adapter->num_active_queues; i++)
3421                 if (adapter->rx_rings[i].desc)
3422                         iavf_free_rx_resources(&adapter->rx_rings[i]);
3423 }
3424
3425 /**
3426  * iavf_validate_tx_bandwidth - validate the max Tx bandwidth
3427  * @adapter: board private structure
3428  * @max_tx_rate: max Tx bw for a tc
3429  **/
3430 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
3431                                       u64 max_tx_rate)
3432 {
3433         int speed = 0, ret = 0;
3434
3435         if (ADV_LINK_SUPPORT(adapter)) {
3436                 if (adapter->link_speed_mbps < U32_MAX) {
3437                         speed = adapter->link_speed_mbps;
3438                         goto validate_bw;
3439                 } else {
3440                         dev_err(&adapter->pdev->dev, "Unknown link speed\n");
3441                         return -EINVAL;
3442                 }
3443         }
3444
3445         switch (adapter->link_speed) {
3446         case VIRTCHNL_LINK_SPEED_40GB:
3447                 speed = SPEED_40000;
3448                 break;
3449         case VIRTCHNL_LINK_SPEED_25GB:
3450                 speed = SPEED_25000;
3451                 break;
3452         case VIRTCHNL_LINK_SPEED_20GB:
3453                 speed = SPEED_20000;
3454                 break;
3455         case VIRTCHNL_LINK_SPEED_10GB:
3456                 speed = SPEED_10000;
3457                 break;
3458         case VIRTCHNL_LINK_SPEED_5GB:
3459                 speed = SPEED_5000;
3460                 break;
3461         case VIRTCHNL_LINK_SPEED_2_5GB:
3462                 speed = SPEED_2500;
3463                 break;
3464         case VIRTCHNL_LINK_SPEED_1GB:
3465                 speed = SPEED_1000;
3466                 break;
3467         case VIRTCHNL_LINK_SPEED_100MB:
3468                 speed = SPEED_100;
3469                 break;
3470         default:
3471                 break;
3472         }
3473
3474 validate_bw:
3475         if (max_tx_rate > speed) {
3476                 dev_err(&adapter->pdev->dev,
3477                         "Invalid tx rate specified\n");
3478                 ret = -EINVAL;
3479         }
3480
3481         return ret;
3482 }
3483
3484 /**
3485  * iavf_validate_ch_config - validate queue mapping info
3486  * @adapter: board private structure
3487  * @mqprio_qopt: queue parameters
3488  *
3489  * This function validates if the config provided by the user to
3490  * configure queue channels is valid or not. Returns 0 on a valid
3491  * config.
3492  **/
3493 static int iavf_validate_ch_config(struct iavf_adapter *adapter,
3494                                    struct tc_mqprio_qopt_offload *mqprio_qopt)
3495 {
3496         u64 total_max_rate = 0;
3497         u32 tx_rate_rem = 0;
3498         int i, num_qps = 0;
3499         u64 tx_rate = 0;
3500         int ret = 0;
3501
3502         if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
3503             mqprio_qopt->qopt.num_tc < 1)
3504                 return -EINVAL;
3505
3506         for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
3507                 if (!mqprio_qopt->qopt.count[i] ||
3508                     mqprio_qopt->qopt.offset[i] != num_qps)
3509                         return -EINVAL;
3510                 if (mqprio_qopt->min_rate[i]) {
3511                         dev_err(&adapter->pdev->dev,
3512                                 "Invalid min tx rate (greater than 0) specified for TC%d\n",
3513                                 i);
3514                         return -EINVAL;
3515                 }
3516
3517                 /* convert to Mbps */
3518                 tx_rate = div_u64(mqprio_qopt->max_rate[i],
3519                                   IAVF_MBPS_DIVISOR);
3520
3521                 if (mqprio_qopt->max_rate[i] &&
3522                     tx_rate < IAVF_MBPS_QUANTA) {
3523                         dev_err(&adapter->pdev->dev,
3524                                 "Invalid max tx rate for TC%d, minimum %dMbps\n",
3525                                 i, IAVF_MBPS_QUANTA);
3526                         return -EINVAL;
3527                 }
3528
3529                 (void)div_u64_rem(tx_rate, IAVF_MBPS_QUANTA, &tx_rate_rem);
3530
3531                 if (tx_rate_rem != 0) {
3532                         dev_err(&adapter->pdev->dev,
3533                                 "Invalid max tx rate for TC%d, not divisible by %d\n",
3534                                 i, IAVF_MBPS_QUANTA);
3535                         return -EINVAL;
3536                 }
3537
3538                 total_max_rate += tx_rate;
3539                 num_qps += mqprio_qopt->qopt.count[i];
3540         }
3541         if (num_qps > adapter->num_active_queues) {
3542                 dev_err(&adapter->pdev->dev,
3543                         "Cannot support requested number of queues\n");
3544                 return -EINVAL;
3545         }
3546
3547         ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
3548         return ret;
3549 }
3550
3551 /**
3552  * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes
3553  * @adapter: board private structure
3554  **/
3555 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
3556 {
3557         struct iavf_cloud_filter *cf, *cftmp;
3558
3559         spin_lock_bh(&adapter->cloud_filter_list_lock);
3560         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
3561                                  list) {
3562                 list_del(&cf->list);
3563                 kfree(cf);
3564                 adapter->num_cloud_filters--;
3565         }
3566         spin_unlock_bh(&adapter->cloud_filter_list_lock);
3567 }
3568
3569 /**
3570  * __iavf_setup_tc - configure multiple traffic classes
3571  * @netdev: network interface device structure
3572  * @type_data: tc offload data
3573  *
3574  * This function processes the config information provided by the
3575  * user to configure traffic classes/queue channels and packages the
3576  * information to request the PF to setup traffic classes.
3577  *
3578  * Returns 0 on success.
3579  **/
3580 static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
3581 {
3582         struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
3583         struct iavf_adapter *adapter = netdev_priv(netdev);
3584         struct virtchnl_vf_resource *vfres = adapter->vf_res;
3585         u8 num_tc = 0, total_qps = 0;
3586         int ret = 0, netdev_tc = 0;
3587         u64 max_tx_rate;
3588         u16 mode;
3589         int i;
3590
3591         num_tc = mqprio_qopt->qopt.num_tc;
3592         mode = mqprio_qopt->mode;
3593
3594         /* delete queue_channel */
3595         if (!mqprio_qopt->qopt.hw) {
3596                 if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
3597                         /* reset the tc configuration */
3598                         netdev_reset_tc(netdev);
3599                         adapter->num_tc = 0;
3600                         netif_tx_stop_all_queues(netdev);
3601                         netif_tx_disable(netdev);
3602                         iavf_del_all_cloud_filters(adapter);
3603                         adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
3604                         total_qps = adapter->orig_num_active_queues;
3605                         goto exit;
3606                 } else {
3607                         return -EINVAL;
3608                 }
3609         }
3610
3611         /* add queue channel */
3612         if (mode == TC_MQPRIO_MODE_CHANNEL) {
3613                 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
3614                         dev_err(&adapter->pdev->dev, "ADq not supported\n");
3615                         return -EOPNOTSUPP;
3616                 }
3617                 if (adapter->ch_config.state != __IAVF_TC_INVALID) {
3618                         dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
3619                         return -EINVAL;
3620                 }
3621
3622                 ret = iavf_validate_ch_config(adapter, mqprio_qopt);
3623                 if (ret)
3624                         return ret;
3625                 /* Return if same TC config is requested */
3626                 if (adapter->num_tc == num_tc)
3627                         return 0;
3628                 adapter->num_tc = num_tc;
3629
3630                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3631                         if (i < num_tc) {
3632                                 adapter->ch_config.ch_info[i].count =
3633                                         mqprio_qopt->qopt.count[i];
3634                                 adapter->ch_config.ch_info[i].offset =
3635                                         mqprio_qopt->qopt.offset[i];
3636                                 total_qps += mqprio_qopt->qopt.count[i];
3637                                 max_tx_rate = mqprio_qopt->max_rate[i];
3638                                 /* convert to Mbps */
3639                                 max_tx_rate = div_u64(max_tx_rate,
3640                                                       IAVF_MBPS_DIVISOR);
3641                                 adapter->ch_config.ch_info[i].max_tx_rate =
3642                                         max_tx_rate;
3643                         } else {
3644                                 adapter->ch_config.ch_info[i].count = 1;
3645                                 adapter->ch_config.ch_info[i].offset = 0;
3646                         }
3647                 }
3648
3649                 /* Take snapshot of original config such as "num_active_queues"
3650                  * It is used later when delete ADQ flow is exercised, so that
3651                  * once delete ADQ flow completes, VF shall go back to its
3652                  * original queue configuration
3653                  */
3654
3655                 adapter->orig_num_active_queues = adapter->num_active_queues;
3656
3657                 /* Store queue info based on TC so that VF gets configured
3658                  * with correct number of queues when VF completes ADQ config
3659                  * flow
3660                  */
3661                 adapter->ch_config.total_qps = total_qps;
3662
3663                 netif_tx_stop_all_queues(netdev);
3664                 netif_tx_disable(netdev);
3665                 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
3666                 netdev_reset_tc(netdev);
3667                 /* Report the tc mapping up the stack */
3668                 netdev_set_num_tc(adapter->netdev, num_tc);
3669                 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3670                         u16 qcount = mqprio_qopt->qopt.count[i];
3671                         u16 qoffset = mqprio_qopt->qopt.offset[i];
3672
3673                         if (i < num_tc)
3674                                 netdev_set_tc_queue(netdev, netdev_tc++, qcount,
3675                                                     qoffset);
3676                 }
3677         }
3678 exit:
3679         if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
3680                 return 0;
3681
3682         netif_set_real_num_rx_queues(netdev, total_qps);
3683         netif_set_real_num_tx_queues(netdev, total_qps);
3684
3685         return ret;
3686 }
3687
3688 /**
3689  * iavf_parse_cls_flower - Parse tc flower filters provided by kernel
3690  * @adapter: board private structure
3691  * @f: pointer to struct flow_cls_offload
3692  * @filter: pointer to cloud filter structure
3693  */
3694 static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
3695                                  struct flow_cls_offload *f,
3696                                  struct iavf_cloud_filter *filter)
3697 {
3698         struct flow_rule *rule = flow_cls_offload_flow_rule(f);
3699         struct flow_dissector *dissector = rule->match.dissector;
3700         u16 n_proto_mask = 0;
3701         u16 n_proto_key = 0;
3702         u8 field_flags = 0;
3703         u16 addr_type = 0;
3704         u16 n_proto = 0;
3705         int i = 0;
3706         struct virtchnl_filter *vf = &filter->f;
3707
3708         if (dissector->used_keys &
3709             ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
3710               BIT(FLOW_DISSECTOR_KEY_BASIC) |
3711               BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
3712               BIT(FLOW_DISSECTOR_KEY_VLAN) |
3713               BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
3714               BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
3715               BIT(FLOW_DISSECTOR_KEY_PORTS) |
3716               BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
3717                 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n",
3718                         dissector->used_keys);
3719                 return -EOPNOTSUPP;
3720         }
3721
3722         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
3723                 struct flow_match_enc_keyid match;
3724
3725                 flow_rule_match_enc_keyid(rule, &match);
3726                 if (match.mask->keyid != 0)
3727                         field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
3728         }
3729
3730         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
3731                 struct flow_match_basic match;
3732
3733                 flow_rule_match_basic(rule, &match);
3734                 n_proto_key = ntohs(match.key->n_proto);
3735                 n_proto_mask = ntohs(match.mask->n_proto);
3736
3737                 if (n_proto_key == ETH_P_ALL) {
3738                         n_proto_key = 0;
3739                         n_proto_mask = 0;
3740                 }
3741                 n_proto = n_proto_key & n_proto_mask;
3742                 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
3743                         return -EINVAL;
3744                 if (n_proto == ETH_P_IPV6) {
3745                         /* specify flow type as TCP IPv6 */
3746                         vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
3747                 }
3748
3749                 if (match.key->ip_proto != IPPROTO_TCP) {
3750                         dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
3751                         return -EINVAL;
3752                 }
3753         }
3754
3755         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
3756                 struct flow_match_eth_addrs match;
3757
3758                 flow_rule_match_eth_addrs(rule, &match);
3759
3760                 /* use is_broadcast and is_zero to check for all 0xf or 0 */
3761                 if (!is_zero_ether_addr(match.mask->dst)) {
3762                         if (is_broadcast_ether_addr(match.mask->dst)) {
3763                                 field_flags |= IAVF_CLOUD_FIELD_OMAC;
3764                         } else {
3765                                 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
3766                                         match.mask->dst);
3767                                 return -EINVAL;
3768                         }
3769                 }
3770
3771                 if (!is_zero_ether_addr(match.mask->src)) {
3772                         if (is_broadcast_ether_addr(match.mask->src)) {
3773                                 field_flags |= IAVF_CLOUD_FIELD_IMAC;
3774                         } else {
3775                                 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
3776                                         match.mask->src);
3777                                 return -EINVAL;
3778                         }
3779                 }
3780
3781                 if (!is_zero_ether_addr(match.key->dst))
3782                         if (is_valid_ether_addr(match.key->dst) ||
3783                             is_multicast_ether_addr(match.key->dst)) {
3784                                 /* set the mask if a valid dst_mac address */
3785                                 for (i = 0; i < ETH_ALEN; i++)
3786                                         vf->mask.tcp_spec.dst_mac[i] |= 0xff;
3787                                 ether_addr_copy(vf->data.tcp_spec.dst_mac,
3788                                                 match.key->dst);
3789                         }
3790
3791                 if (!is_zero_ether_addr(match.key->src))
3792                         if (is_valid_ether_addr(match.key->src) ||
3793                             is_multicast_ether_addr(match.key->src)) {
3794                                 /* set the mask if a valid dst_mac address */
3795                                 for (i = 0; i < ETH_ALEN; i++)
3796                                         vf->mask.tcp_spec.src_mac[i] |= 0xff;
3797                                 ether_addr_copy(vf->data.tcp_spec.src_mac,
3798                                                 match.key->src);
3799                 }
3800         }
3801
3802         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
3803                 struct flow_match_vlan match;
3804
3805                 flow_rule_match_vlan(rule, &match);
3806                 if (match.mask->vlan_id) {
3807                         if (match.mask->vlan_id == VLAN_VID_MASK) {
3808                                 field_flags |= IAVF_CLOUD_FIELD_IVLAN;
3809                         } else {
3810                                 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
3811                                         match.mask->vlan_id);
3812                                 return -EINVAL;
3813                         }
3814                 }
3815                 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
3816                 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);
3817         }
3818
3819         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
3820                 struct flow_match_control match;
3821
3822                 flow_rule_match_control(rule, &match);
3823                 addr_type = match.key->addr_type;
3824         }
3825
3826         if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
3827                 struct flow_match_ipv4_addrs match;
3828
3829                 flow_rule_match_ipv4_addrs(rule, &match);
3830                 if (match.mask->dst) {
3831                         if (match.mask->dst == cpu_to_be32(0xffffffff)) {
3832                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
3833                         } else {
3834                                 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
3835                                         be32_to_cpu(match.mask->dst));
3836                                 return -EINVAL;
3837                         }
3838                 }
3839
3840                 if (match.mask->src) {
3841                         if (match.mask->src == cpu_to_be32(0xffffffff)) {
3842                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
3843                         } else {
3844                                 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
3845                                         be32_to_cpu(match.mask->src));
3846                                 return -EINVAL;
3847                         }
3848                 }
3849
3850                 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
3851                         dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
3852                         return -EINVAL;
3853                 }
3854                 if (match.key->dst) {
3855                         vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
3856                         vf->data.tcp_spec.dst_ip[0] = match.key->dst;
3857                 }
3858                 if (match.key->src) {
3859                         vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
3860                         vf->data.tcp_spec.src_ip[0] = match.key->src;
3861                 }
3862         }
3863
3864         if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
3865                 struct flow_match_ipv6_addrs match;
3866
3867                 flow_rule_match_ipv6_addrs(rule, &match);
3868
3869                 /* validate mask, make sure it is not IPV6_ADDR_ANY */
3870                 if (ipv6_addr_any(&match.mask->dst)) {
3871                         dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
3872                                 IPV6_ADDR_ANY);
3873                         return -EINVAL;
3874                 }
3875
3876                 /* src and dest IPv6 address should not be LOOPBACK
3877                  * (0:0:0:0:0:0:0:1) which can be represented as ::1
3878                  */
3879                 if (ipv6_addr_loopback(&match.key->dst) ||
3880                     ipv6_addr_loopback(&match.key->src)) {
3881                         dev_err(&adapter->pdev->dev,
3882                                 "ipv6 addr should not be loopback\n");
3883                         return -EINVAL;
3884                 }
3885                 if (!ipv6_addr_any(&match.mask->dst) ||
3886                     !ipv6_addr_any(&match.mask->src))
3887                         field_flags |= IAVF_CLOUD_FIELD_IIP;
3888
3889                 for (i = 0; i < 4; i++)
3890                         vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
3891                 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,
3892                        sizeof(vf->data.tcp_spec.dst_ip));
3893                 for (i = 0; i < 4; i++)
3894                         vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
3895                 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,
3896                        sizeof(vf->data.tcp_spec.src_ip));
3897         }
3898         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
3899                 struct flow_match_ports match;
3900
3901                 flow_rule_match_ports(rule, &match);
3902                 if (match.mask->src) {
3903                         if (match.mask->src == cpu_to_be16(0xffff)) {
3904                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
3905                         } else {
3906                                 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
3907                                         be16_to_cpu(match.mask->src));
3908                                 return -EINVAL;
3909                         }
3910                 }
3911
3912                 if (match.mask->dst) {
3913                         if (match.mask->dst == cpu_to_be16(0xffff)) {
3914                                 field_flags |= IAVF_CLOUD_FIELD_IIP;
3915                         } else {
3916                                 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
3917                                         be16_to_cpu(match.mask->dst));
3918                                 return -EINVAL;
3919                         }
3920                 }
3921                 if (match.key->dst) {
3922                         vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
3923                         vf->data.tcp_spec.dst_port = match.key->dst;
3924                 }
3925
3926                 if (match.key->src) {
3927                         vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
3928                         vf->data.tcp_spec.src_port = match.key->src;
3929                 }
3930         }
3931         vf->field_flags = field_flags;
3932
3933         return 0;
3934 }
3935
3936 /**
3937  * iavf_handle_tclass - Forward to a traffic class on the device
3938  * @adapter: board private structure
3939  * @tc: traffic class index on the device
3940  * @filter: pointer to cloud filter structure
3941  */
3942 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
3943                               struct iavf_cloud_filter *filter)
3944 {
3945         if (tc == 0)
3946                 return 0;
3947         if (tc < adapter->num_tc) {
3948                 if (!filter->f.data.tcp_spec.dst_port) {
3949                         dev_err(&adapter->pdev->dev,
3950                                 "Specify destination port to redirect to traffic class other than TC0\n");
3951                         return -EINVAL;
3952                 }
3953         }
3954         /* redirect to a traffic class on the same device */
3955         filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
3956         filter->f.action_meta = tc;
3957         return 0;
3958 }
3959
3960 /**
3961  * iavf_find_cf - Find the cloud filter in the list
3962  * @adapter: Board private structure
3963  * @cookie: filter specific cookie
3964  *
3965  * Returns ptr to the filter object or NULL. Must be called while holding the
3966  * cloud_filter_list_lock.
3967  */
3968 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
3969                                               unsigned long *cookie)
3970 {
3971         struct iavf_cloud_filter *filter = NULL;
3972
3973         if (!cookie)
3974                 return NULL;
3975
3976         list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
3977                 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
3978                         return filter;
3979         }
3980         return NULL;
3981 }
3982
3983 /**
3984  * iavf_configure_clsflower - Add tc flower filters
3985  * @adapter: board private structure
3986  * @cls_flower: Pointer to struct flow_cls_offload
3987  */
3988 static int iavf_configure_clsflower(struct iavf_adapter *adapter,
3989                                     struct flow_cls_offload *cls_flower)
3990 {
3991         int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
3992         struct iavf_cloud_filter *filter = NULL;
3993         int err = -EINVAL, count = 50;
3994
3995         if (tc < 0) {
3996                 dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
3997                 return -EINVAL;
3998         }
3999
4000         filter = kzalloc(sizeof(*filter), GFP_KERNEL);
4001         if (!filter)
4002                 return -ENOMEM;
4003
4004         while (!mutex_trylock(&adapter->crit_lock)) {
4005                 if (--count == 0) {
4006                         kfree(filter);
4007                         return err;
4008                 }
4009                 udelay(1);
4010         }
4011
4012         filter->cookie = cls_flower->cookie;
4013
4014         /* bail out here if filter already exists */
4015         spin_lock_bh(&adapter->cloud_filter_list_lock);
4016         if (iavf_find_cf(adapter, &cls_flower->cookie)) {
4017                 dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n");
4018                 err = -EEXIST;
4019                 goto spin_unlock;
4020         }
4021         spin_unlock_bh(&adapter->cloud_filter_list_lock);
4022
4023         /* set the mask to all zeroes to begin with */
4024         memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
4025         /* start out with flow type and eth type IPv4 to begin with */
4026         filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
4027         err = iavf_parse_cls_flower(adapter, cls_flower, filter);
4028         if (err)
4029                 goto err;
4030
4031         err = iavf_handle_tclass(adapter, tc, filter);
4032         if (err)
4033                 goto err;
4034
4035         /* add filter to the list */
4036         spin_lock_bh(&adapter->cloud_filter_list_lock);
4037         list_add_tail(&filter->list, &adapter->cloud_filter_list);
4038         adapter->num_cloud_filters++;
4039         filter->add = true;
4040         adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
4041 spin_unlock:
4042         spin_unlock_bh(&adapter->cloud_filter_list_lock);
4043 err:
4044         if (err)
4045                 kfree(filter);
4046
4047         mutex_unlock(&adapter->crit_lock);
4048         return err;
4049 }
4050
4051 /**
4052  * iavf_delete_clsflower - Remove tc flower filters
4053  * @adapter: board private structure
4054  * @cls_flower: Pointer to struct flow_cls_offload
4055  */
4056 static int iavf_delete_clsflower(struct iavf_adapter *adapter,
4057                                  struct flow_cls_offload *cls_flower)
4058 {
4059         struct iavf_cloud_filter *filter = NULL;
4060         int err = 0;
4061
4062         spin_lock_bh(&adapter->cloud_filter_list_lock);
4063         filter = iavf_find_cf(adapter, &cls_flower->cookie);
4064         if (filter) {
4065                 filter->del = true;
4066                 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
4067         } else {
4068                 err = -EINVAL;
4069         }
4070         spin_unlock_bh(&adapter->cloud_filter_list_lock);
4071
4072         return err;
4073 }
4074
4075 /**
4076  * iavf_setup_tc_cls_flower - flower classifier offloads
4077  * @adapter: board private structure
4078  * @cls_flower: pointer to flow_cls_offload struct with flow info
4079  */
4080 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
4081                                     struct flow_cls_offload *cls_flower)
4082 {
4083         switch (cls_flower->command) {
4084         case FLOW_CLS_REPLACE:
4085                 return iavf_configure_clsflower(adapter, cls_flower);
4086         case FLOW_CLS_DESTROY:
4087                 return iavf_delete_clsflower(adapter, cls_flower);
4088         case FLOW_CLS_STATS:
4089                 return -EOPNOTSUPP;
4090         default:
4091                 return -EOPNOTSUPP;
4092         }
4093 }
4094
4095 /**
4096  * iavf_setup_tc_block_cb - block callback for tc
4097  * @type: type of offload
4098  * @type_data: offload data
4099  * @cb_priv:
4100  *
4101  * This function is the block callback for traffic classes
4102  **/
4103 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4104                                   void *cb_priv)
4105 {
4106         struct iavf_adapter *adapter = cb_priv;
4107
4108         if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
4109                 return -EOPNOTSUPP;
4110
4111         switch (type) {
4112         case TC_SETUP_CLSFLOWER:
4113                 return iavf_setup_tc_cls_flower(cb_priv, type_data);
4114         default:
4115                 return -EOPNOTSUPP;
4116         }
4117 }
4118
4119 static LIST_HEAD(iavf_block_cb_list);
4120
4121 /**
4122  * iavf_setup_tc - configure multiple traffic classes
4123  * @netdev: network interface device structure
4124  * @type: type of offload
4125  * @type_data: tc offload data
4126  *
4127  * This function is the callback to ndo_setup_tc in the
4128  * netdev_ops.
4129  *
4130  * Returns 0 on success
4131  **/
4132 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
4133                          void *type_data)
4134 {
4135         struct iavf_adapter *adapter = netdev_priv(netdev);
4136
4137         switch (type) {
4138         case TC_SETUP_QDISC_MQPRIO:
4139                 return __iavf_setup_tc(netdev, type_data);
4140         case TC_SETUP_BLOCK:
4141                 return flow_block_cb_setup_simple(type_data,
4142                                                   &iavf_block_cb_list,
4143                                                   iavf_setup_tc_block_cb,
4144                                                   adapter, adapter, true);
4145         default:
4146                 return -EOPNOTSUPP;
4147         }
4148 }
4149
4150 /**
4151  * iavf_open - Called when a network interface is made active
4152  * @netdev: network interface device structure
4153  *
4154  * Returns 0 on success, negative value on failure
4155  *
4156  * The open entry point is called when a network interface is made
4157  * active by the system (IFF_UP).  At this point all resources needed
4158  * for transmit and receive operations are allocated, the interrupt
4159  * handler is registered with the OS, the watchdog is started,
4160  * and the stack is notified that the interface is ready.
4161  **/
4162 static int iavf_open(struct net_device *netdev)
4163 {
4164         struct iavf_adapter *adapter = netdev_priv(netdev);
4165         int err;
4166
4167         if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
4168                 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
4169                 return -EIO;
4170         }
4171
4172         while (!mutex_trylock(&adapter->crit_lock)) {
4173                 /* If we are in __IAVF_INIT_CONFIG_ADAPTER state the crit_lock
4174                  * is already taken and iavf_open is called from an upper
4175                  * device's notifier reacting on NETDEV_REGISTER event.
4176                  * We have to leave here to avoid dead lock.
4177                  */
4178                 if (adapter->state == __IAVF_INIT_CONFIG_ADAPTER)
4179                         return -EBUSY;
4180
4181                 usleep_range(500, 1000);
4182         }
4183
4184         if (adapter->state != __IAVF_DOWN) {
4185                 err = -EBUSY;
4186                 goto err_unlock;
4187         }
4188
4189         if (adapter->state == __IAVF_RUNNING &&
4190             !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) {
4191                 dev_dbg(&adapter->pdev->dev, "VF is already open.\n");
4192                 err = 0;
4193                 goto err_unlock;
4194         }
4195
4196         /* allocate transmit descriptors */
4197         err = iavf_setup_all_tx_resources(adapter);
4198         if (err)
4199                 goto err_setup_tx;
4200
4201         /* allocate receive descriptors */
4202         err = iavf_setup_all_rx_resources(adapter);
4203         if (err)
4204                 goto err_setup_rx;
4205
4206         /* clear any pending interrupts, may auto mask */
4207         err = iavf_request_traffic_irqs(adapter, netdev->name);
4208         if (err)
4209                 goto err_req_irq;
4210
4211         spin_lock_bh(&adapter->mac_vlan_list_lock);
4212
4213         iavf_add_filter(adapter, adapter->hw.mac.addr);
4214
4215         spin_unlock_bh(&adapter->mac_vlan_list_lock);
4216
4217         /* Restore VLAN filters that were removed with IFF_DOWN */
4218         iavf_restore_filters(adapter);
4219
4220         iavf_configure(adapter);
4221
4222         iavf_up_complete(adapter);
4223
4224         iavf_irq_enable(adapter, true);
4225
4226         mutex_unlock(&adapter->crit_lock);
4227
4228         return 0;
4229
4230 err_req_irq:
4231         iavf_down(adapter);
4232         iavf_free_traffic_irqs(adapter);
4233 err_setup_rx:
4234         iavf_free_all_rx_resources(adapter);
4235 err_setup_tx:
4236         iavf_free_all_tx_resources(adapter);
4237 err_unlock:
4238         mutex_unlock(&adapter->crit_lock);
4239
4240         return err;
4241 }
4242
4243 /**
4244  * iavf_close - Disables a network interface
4245  * @netdev: network interface device structure
4246  *
4247  * Returns 0, this is not allowed to fail
4248  *
4249  * The close entry point is called when an interface is de-activated
4250  * by the OS.  The hardware is still under the drivers control, but
4251  * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
4252  * are freed, along with all transmit and receive resources.
4253  **/
4254 static int iavf_close(struct net_device *netdev)
4255 {
4256         struct iavf_adapter *adapter = netdev_priv(netdev);
4257         u64 aq_to_restore;
4258         int status;
4259
4260         mutex_lock(&adapter->crit_lock);
4261
4262         if (adapter->state <= __IAVF_DOWN_PENDING) {
4263                 mutex_unlock(&adapter->crit_lock);
4264                 return 0;
4265         }
4266
4267         set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
4268         if (CLIENT_ENABLED(adapter))
4269                 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
4270         /* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before
4271          * IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl
4272          * deadlock with adminq_task() until iavf_close timeouts. We must send
4273          * IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make
4274          * disable queues possible for vf. Give only necessary flags to
4275          * iavf_down and save other to set them right before iavf_close()
4276          * returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and
4277          * iavf will be in DOWN state.
4278          */
4279         aq_to_restore = adapter->aq_required;
4280         adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG;
4281
4282         /* Remove flags which we do not want to send after close or we want to
4283          * send before disable queues.
4284          */
4285         aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG              |
4286                            IAVF_FLAG_AQ_ENABLE_QUEUES           |
4287                            IAVF_FLAG_AQ_CONFIGURE_QUEUES        |
4288                            IAVF_FLAG_AQ_ADD_VLAN_FILTER         |
4289                            IAVF_FLAG_AQ_ADD_MAC_FILTER          |
4290                            IAVF_FLAG_AQ_ADD_CLOUD_FILTER        |
4291                            IAVF_FLAG_AQ_ADD_FDIR_FILTER         |
4292                            IAVF_FLAG_AQ_ADD_ADV_RSS_CFG);
4293
4294         iavf_down(adapter);
4295         iavf_change_state(adapter, __IAVF_DOWN_PENDING);
4296         iavf_free_traffic_irqs(adapter);
4297
4298         mutex_unlock(&adapter->crit_lock);
4299
4300         /* We explicitly don't free resources here because the hardware is
4301          * still active and can DMA into memory. Resources are cleared in
4302          * iavf_virtchnl_completion() after we get confirmation from the PF
4303          * driver that the rings have been stopped.
4304          *
4305          * Also, we wait for state to transition to __IAVF_DOWN before
4306          * returning. State change occurs in iavf_virtchnl_completion() after
4307          * VF resources are released (which occurs after PF driver processes and
4308          * responds to admin queue commands).
4309          */
4310
4311         status = wait_event_timeout(adapter->down_waitqueue,
4312                                     adapter->state == __IAVF_DOWN,
4313                                     msecs_to_jiffies(500));
4314         if (!status)
4315                 netdev_warn(netdev, "Device resources not yet released\n");
4316
4317         mutex_lock(&adapter->crit_lock);
4318         adapter->aq_required |= aq_to_restore;
4319         mutex_unlock(&adapter->crit_lock);
4320         return 0;
4321 }
4322
4323 /**
4324  * iavf_change_mtu - Change the Maximum Transfer Unit
4325  * @netdev: network interface device structure
4326  * @new_mtu: new value for maximum frame size
4327  *
4328  * Returns 0 on success, negative on failure
4329  **/
4330 static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
4331 {
4332         struct iavf_adapter *adapter = netdev_priv(netdev);
4333
4334         netdev_dbg(netdev, "changing MTU from %d to %d\n",
4335                    netdev->mtu, new_mtu);
4336         netdev->mtu = new_mtu;
4337         if (CLIENT_ENABLED(adapter)) {
4338                 iavf_notify_client_l2_params(&adapter->vsi);
4339                 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
4340         }
4341
4342         if (netif_running(netdev)) {
4343                 adapter->flags |= IAVF_FLAG_RESET_NEEDED;
4344                 queue_work(adapter->wq, &adapter->reset_task);
4345         }
4346
4347         return 0;
4348 }
4349
4350 #define NETIF_VLAN_OFFLOAD_FEATURES     (NETIF_F_HW_VLAN_CTAG_RX | \
4351                                          NETIF_F_HW_VLAN_CTAG_TX | \
4352                                          NETIF_F_HW_VLAN_STAG_RX | \
4353                                          NETIF_F_HW_VLAN_STAG_TX)
4354
4355 /**
4356  * iavf_set_features - set the netdev feature flags
4357  * @netdev: ptr to the netdev being adjusted
4358  * @features: the feature set that the stack is suggesting
4359  * Note: expects to be called while under rtnl_lock()
4360  **/
4361 static int iavf_set_features(struct net_device *netdev,
4362                              netdev_features_t features)
4363 {
4364         struct iavf_adapter *adapter = netdev_priv(netdev);
4365
4366         /* trigger update on any VLAN feature change */
4367         if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^
4368             (features & NETIF_VLAN_OFFLOAD_FEATURES))
4369                 iavf_set_vlan_offload_features(adapter, netdev->features,
4370                                                features);
4371
4372         return 0;
4373 }
4374
4375 /**
4376  * iavf_features_check - Validate encapsulated packet conforms to limits
4377  * @skb: skb buff
4378  * @dev: This physical port's netdev
4379  * @features: Offload features that the stack believes apply
4380  **/
4381 static netdev_features_t iavf_features_check(struct sk_buff *skb,
4382                                              struct net_device *dev,
4383                                              netdev_features_t features)
4384 {
4385         size_t len;
4386
4387         /* No point in doing any of this if neither checksum nor GSO are
4388          * being requested for this frame.  We can rule out both by just
4389          * checking for CHECKSUM_PARTIAL
4390          */
4391         if (skb->ip_summed != CHECKSUM_PARTIAL)
4392                 return features;
4393
4394         /* We cannot support GSO if the MSS is going to be less than
4395          * 64 bytes.  If it is then we need to drop support for GSO.
4396          */
4397         if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
4398                 features &= ~NETIF_F_GSO_MASK;
4399
4400         /* MACLEN can support at most 63 words */
4401         len = skb_network_header(skb) - skb->data;
4402         if (len & ~(63 * 2))
4403                 goto out_err;
4404
4405         /* IPLEN and EIPLEN can support at most 127 dwords */
4406         len = skb_transport_header(skb) - skb_network_header(skb);
4407         if (len & ~(127 * 4))
4408                 goto out_err;
4409
4410         if (skb->encapsulation) {
4411                 /* L4TUNLEN can support 127 words */
4412                 len = skb_inner_network_header(skb) - skb_transport_header(skb);
4413                 if (len & ~(127 * 2))
4414                         goto out_err;
4415
4416                 /* IPLEN can support at most 127 dwords */
4417                 len = skb_inner_transport_header(skb) -
4418                       skb_inner_network_header(skb);
4419                 if (len & ~(127 * 4))
4420                         goto out_err;
4421         }
4422
4423         /* No need to validate L4LEN as TCP is the only protocol with a
4424          * flexible value and we support all possible values supported
4425          * by TCP, which is at most 15 dwords
4426          */
4427
4428         return features;
4429 out_err:
4430         return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4431 }
4432
4433 /**
4434  * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off
4435  * @adapter: board private structure
4436  *
4437  * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4438  * were negotiated determine the VLAN features that can be toggled on and off.
4439  **/
4440 static netdev_features_t
4441 iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter)
4442 {
4443         netdev_features_t hw_features = 0;
4444
4445         if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4446                 return hw_features;
4447
4448         /* Enable VLAN features if supported */
4449         if (VLAN_ALLOWED(adapter)) {
4450                 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
4451                                 NETIF_F_HW_VLAN_CTAG_RX);
4452         } else if (VLAN_V2_ALLOWED(adapter)) {
4453                 struct virtchnl_vlan_caps *vlan_v2_caps =
4454                         &adapter->vlan_v2_caps;
4455                 struct virtchnl_vlan_supported_caps *stripping_support =
4456                         &vlan_v2_caps->offloads.stripping_support;
4457                 struct virtchnl_vlan_supported_caps *insertion_support =
4458                         &vlan_v2_caps->offloads.insertion_support;
4459
4460                 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4461                     stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4462                         if (stripping_support->outer &
4463                             VIRTCHNL_VLAN_ETHERTYPE_8100)
4464                                 hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4465                         if (stripping_support->outer &
4466                             VIRTCHNL_VLAN_ETHERTYPE_88A8)
4467                                 hw_features |= NETIF_F_HW_VLAN_STAG_RX;
4468                 } else if (stripping_support->inner !=
4469                            VIRTCHNL_VLAN_UNSUPPORTED &&
4470                            stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4471                         if (stripping_support->inner &
4472                             VIRTCHNL_VLAN_ETHERTYPE_8100)
4473                                 hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4474                 }
4475
4476                 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4477                     insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4478                         if (insertion_support->outer &
4479                             VIRTCHNL_VLAN_ETHERTYPE_8100)
4480                                 hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4481                         if (insertion_support->outer &
4482                             VIRTCHNL_VLAN_ETHERTYPE_88A8)
4483                                 hw_features |= NETIF_F_HW_VLAN_STAG_TX;
4484                 } else if (insertion_support->inner &&
4485                            insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4486                         if (insertion_support->inner &
4487                             VIRTCHNL_VLAN_ETHERTYPE_8100)
4488                                 hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4489                 }
4490         }
4491
4492         return hw_features;
4493 }
4494
4495 /**
4496  * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures
4497  * @adapter: board private structure
4498  *
4499  * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4500  * were negotiated determine the VLAN features that are enabled by default.
4501  **/
4502 static netdev_features_t
4503 iavf_get_netdev_vlan_features(struct iavf_adapter *adapter)
4504 {
4505         netdev_features_t features = 0;
4506
4507         if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4508                 return features;
4509
4510         if (VLAN_ALLOWED(adapter)) {
4511                 features |= NETIF_F_HW_VLAN_CTAG_FILTER |
4512                         NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX;
4513         } else if (VLAN_V2_ALLOWED(adapter)) {
4514                 struct virtchnl_vlan_caps *vlan_v2_caps =
4515                         &adapter->vlan_v2_caps;
4516                 struct virtchnl_vlan_supported_caps *filtering_support =
4517                         &vlan_v2_caps->filtering.filtering_support;
4518                 struct virtchnl_vlan_supported_caps *stripping_support =
4519                         &vlan_v2_caps->offloads.stripping_support;
4520                 struct virtchnl_vlan_supported_caps *insertion_support =
4521                         &vlan_v2_caps->offloads.insertion_support;
4522                 u32 ethertype_init;
4523
4524                 /* give priority to outer stripping and don't support both outer
4525                  * and inner stripping
4526                  */
4527                 ethertype_init = vlan_v2_caps->offloads.ethertype_init;
4528                 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4529                         if (stripping_support->outer &
4530                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4531                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4532                                 features |= NETIF_F_HW_VLAN_CTAG_RX;
4533                         else if (stripping_support->outer &
4534                                  VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4535                                  ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4536                                 features |= NETIF_F_HW_VLAN_STAG_RX;
4537                 } else if (stripping_support->inner !=
4538                            VIRTCHNL_VLAN_UNSUPPORTED) {
4539                         if (stripping_support->inner &
4540                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4541                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4542                                 features |= NETIF_F_HW_VLAN_CTAG_RX;
4543                 }
4544
4545                 /* give priority to outer insertion and don't support both outer
4546                  * and inner insertion
4547                  */
4548                 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4549                         if (insertion_support->outer &
4550                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4551                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4552                                 features |= NETIF_F_HW_VLAN_CTAG_TX;
4553                         else if (insertion_support->outer &
4554                                  VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4555                                  ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4556                                 features |= NETIF_F_HW_VLAN_STAG_TX;
4557                 } else if (insertion_support->inner !=
4558                            VIRTCHNL_VLAN_UNSUPPORTED) {
4559                         if (insertion_support->inner &
4560                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4561                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4562                                 features |= NETIF_F_HW_VLAN_CTAG_TX;
4563                 }
4564
4565                 /* give priority to outer filtering and don't bother if both
4566                  * outer and inner filtering are enabled
4567                  */
4568                 ethertype_init = vlan_v2_caps->filtering.ethertype_init;
4569                 if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4570                         if (filtering_support->outer &
4571                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4572                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4573                                 features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4574                         if (filtering_support->outer &
4575                             VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4576                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4577                                 features |= NETIF_F_HW_VLAN_STAG_FILTER;
4578                 } else if (filtering_support->inner !=
4579                            VIRTCHNL_VLAN_UNSUPPORTED) {
4580                         if (filtering_support->inner &
4581                             VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4582                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4583                                 features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4584                         if (filtering_support->inner &
4585                             VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4586                             ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4587                                 features |= NETIF_F_HW_VLAN_STAG_FILTER;
4588                 }
4589         }
4590
4591         return features;
4592 }
4593
4594 #define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \
4595         (!(((requested) & (feature_bit)) && \
4596            !((allowed) & (feature_bit))))
4597
4598 /**
4599  * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support
4600  * @adapter: board private structure
4601  * @requested_features: stack requested NETDEV features
4602  **/
4603 static netdev_features_t
4604 iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter,
4605                               netdev_features_t requested_features)
4606 {
4607         netdev_features_t allowed_features;
4608
4609         allowed_features = iavf_get_netdev_vlan_hw_features(adapter) |
4610                 iavf_get_netdev_vlan_features(adapter);
4611
4612         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4613                                               allowed_features,
4614                                               NETIF_F_HW_VLAN_CTAG_TX))
4615                 requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX;
4616
4617         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4618                                               allowed_features,
4619                                               NETIF_F_HW_VLAN_CTAG_RX))
4620                 requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
4621
4622         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4623                                               allowed_features,
4624                                               NETIF_F_HW_VLAN_STAG_TX))
4625                 requested_features &= ~NETIF_F_HW_VLAN_STAG_TX;
4626         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4627                                               allowed_features,
4628                                               NETIF_F_HW_VLAN_STAG_RX))
4629                 requested_features &= ~NETIF_F_HW_VLAN_STAG_RX;
4630
4631         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4632                                               allowed_features,
4633                                               NETIF_F_HW_VLAN_CTAG_FILTER))
4634                 requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
4635
4636         if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4637                                               allowed_features,
4638                                               NETIF_F_HW_VLAN_STAG_FILTER))
4639                 requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER;
4640
4641         if ((requested_features &
4642              (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
4643             (requested_features &
4644              (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) &&
4645             adapter->vlan_v2_caps.offloads.ethertype_match ==
4646             VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) {
4647                 netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
4648                 requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX |
4649                                         NETIF_F_HW_VLAN_STAG_TX);
4650         }
4651
4652         return requested_features;
4653 }
4654
4655 /**
4656  * iavf_fix_features - fix up the netdev feature bits
4657  * @netdev: our net device
4658  * @features: desired feature bits
4659  *
4660  * Returns fixed-up features bits
4661  **/
4662 static netdev_features_t iavf_fix_features(struct net_device *netdev,
4663                                            netdev_features_t features)
4664 {
4665         struct iavf_adapter *adapter = netdev_priv(netdev);
4666
4667         return iavf_fix_netdev_vlan_features(adapter, features);
4668 }
4669
4670 static const struct net_device_ops iavf_netdev_ops = {
4671         .ndo_open               = iavf_open,
4672         .ndo_stop               = iavf_close,
4673         .ndo_start_xmit         = iavf_xmit_frame,
4674         .ndo_set_rx_mode        = iavf_set_rx_mode,
4675         .ndo_validate_addr      = eth_validate_addr,
4676         .ndo_set_mac_address    = iavf_set_mac,
4677         .ndo_change_mtu         = iavf_change_mtu,
4678         .ndo_tx_timeout         = iavf_tx_timeout,
4679         .ndo_vlan_rx_add_vid    = iavf_vlan_rx_add_vid,
4680         .ndo_vlan_rx_kill_vid   = iavf_vlan_rx_kill_vid,
4681         .ndo_features_check     = iavf_features_check,
4682         .ndo_fix_features       = iavf_fix_features,
4683         .ndo_set_features       = iavf_set_features,
4684         .ndo_setup_tc           = iavf_setup_tc,
4685 };
4686
4687 /**
4688  * iavf_check_reset_complete - check that VF reset is complete
4689  * @hw: pointer to hw struct
4690  *
4691  * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
4692  **/
4693 static int iavf_check_reset_complete(struct iavf_hw *hw)
4694 {
4695         u32 rstat;
4696         int i;
4697
4698         for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
4699                 rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
4700                              IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
4701                 if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
4702                     (rstat == VIRTCHNL_VFR_COMPLETED))
4703                         return 0;
4704                 usleep_range(10, 20);
4705         }
4706         return -EBUSY;
4707 }
4708
4709 /**
4710  * iavf_process_config - Process the config information we got from the PF
4711  * @adapter: board private structure
4712  *
4713  * Verify that we have a valid config struct, and set up our netdev features
4714  * and our VSI struct.
4715  **/
4716 int iavf_process_config(struct iavf_adapter *adapter)
4717 {
4718         struct virtchnl_vf_resource *vfres = adapter->vf_res;
4719         netdev_features_t hw_vlan_features, vlan_features;
4720         struct net_device *netdev = adapter->netdev;
4721         netdev_features_t hw_enc_features;
4722         netdev_features_t hw_features;
4723
4724         hw_enc_features = NETIF_F_SG                    |
4725                           NETIF_F_IP_CSUM               |
4726                           NETIF_F_IPV6_CSUM             |
4727                           NETIF_F_HIGHDMA               |
4728                           NETIF_F_SOFT_FEATURES |
4729                           NETIF_F_TSO                   |
4730                           NETIF_F_TSO_ECN               |
4731                           NETIF_F_TSO6                  |
4732                           NETIF_F_SCTP_CRC              |
4733                           NETIF_F_RXHASH                |
4734                           NETIF_F_RXCSUM                |
4735                           0;
4736
4737         /* advertise to stack only if offloads for encapsulated packets is
4738          * supported
4739          */
4740         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
4741                 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL       |
4742                                    NETIF_F_GSO_GRE              |
4743                                    NETIF_F_GSO_GRE_CSUM         |
4744                                    NETIF_F_GSO_IPXIP4           |
4745                                    NETIF_F_GSO_IPXIP6           |
4746                                    NETIF_F_GSO_UDP_TUNNEL_CSUM  |
4747                                    NETIF_F_GSO_PARTIAL          |
4748                                    0;
4749
4750                 if (!(vfres->vf_cap_flags &
4751                       VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
4752                         netdev->gso_partial_features |=
4753                                 NETIF_F_GSO_UDP_TUNNEL_CSUM;
4754
4755                 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
4756                 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
4757                 netdev->hw_enc_features |= hw_enc_features;
4758         }
4759         /* record features VLANs can make use of */
4760         netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
4761
4762         /* Write features and hw_features separately to avoid polluting
4763          * with, or dropping, features that are set when we registered.
4764          */
4765         hw_features = hw_enc_features;
4766
4767         /* get HW VLAN features that can be toggled */
4768         hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter);
4769
4770         /* Enable cloud filter if ADQ is supported */
4771         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
4772                 hw_features |= NETIF_F_HW_TC;
4773         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)
4774                 hw_features |= NETIF_F_GSO_UDP_L4;
4775
4776         netdev->hw_features |= hw_features | hw_vlan_features;
4777         vlan_features = iavf_get_netdev_vlan_features(adapter);
4778
4779         netdev->features |= hw_features | vlan_features;
4780
4781         if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
4782                 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4783
4784         netdev->priv_flags |= IFF_UNICAST_FLT;
4785
4786         /* Do not turn on offloads when they are requested to be turned off.
4787          * TSO needs minimum 576 bytes to work correctly.
4788          */
4789         if (netdev->wanted_features) {
4790                 if (!(netdev->wanted_features & NETIF_F_TSO) ||
4791                     netdev->mtu < 576)
4792                         netdev->features &= ~NETIF_F_TSO;
4793                 if (!(netdev->wanted_features & NETIF_F_TSO6) ||
4794                     netdev->mtu < 576)
4795                         netdev->features &= ~NETIF_F_TSO6;
4796                 if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
4797                         netdev->features &= ~NETIF_F_TSO_ECN;
4798                 if (!(netdev->wanted_features & NETIF_F_GRO))
4799                         netdev->features &= ~NETIF_F_GRO;
4800                 if (!(netdev->wanted_features & NETIF_F_GSO))
4801                         netdev->features &= ~NETIF_F_GSO;
4802         }
4803
4804         return 0;
4805 }
4806
4807 /**
4808  * iavf_shutdown - Shutdown the device in preparation for a reboot
4809  * @pdev: pci device structure
4810  **/
4811 static void iavf_shutdown(struct pci_dev *pdev)
4812 {
4813         struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
4814         struct net_device *netdev = adapter->netdev;
4815
4816         netif_device_detach(netdev);
4817
4818         if (netif_running(netdev))
4819                 iavf_close(netdev);
4820
4821         if (iavf_lock_timeout(&adapter->crit_lock, 5000))
4822                 dev_warn(&adapter->pdev->dev, "%s: failed to acquire crit_lock\n", __func__);
4823         /* Prevent the watchdog from running. */
4824         iavf_change_state(adapter, __IAVF_REMOVE);
4825         adapter->aq_required = 0;
4826         mutex_unlock(&adapter->crit_lock);
4827
4828 #ifdef CONFIG_PM
4829         pci_save_state(pdev);
4830
4831 #endif
4832         pci_disable_device(pdev);
4833 }
4834
4835 /**
4836  * iavf_probe - Device Initialization Routine
4837  * @pdev: PCI device information struct
4838  * @ent: entry in iavf_pci_tbl
4839  *
4840  * Returns 0 on success, negative on failure
4841  *
4842  * iavf_probe initializes an adapter identified by a pci_dev structure.
4843  * The OS initialization, configuring of the adapter private structure,
4844  * and a hardware reset occur.
4845  **/
4846 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4847 {
4848         struct net_device *netdev;
4849         struct iavf_adapter *adapter = NULL;
4850         struct iavf_hw *hw = NULL;
4851         int err;
4852
4853         err = pci_enable_device(pdev);
4854         if (err)
4855                 return err;
4856
4857         err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4858         if (err) {
4859                 dev_err(&pdev->dev,
4860                         "DMA configuration failed: 0x%x\n", err);
4861                 goto err_dma;
4862         }
4863
4864         err = pci_request_regions(pdev, iavf_driver_name);
4865         if (err) {
4866                 dev_err(&pdev->dev,
4867                         "pci_request_regions failed 0x%x\n", err);
4868                 goto err_pci_reg;
4869         }
4870
4871         pci_enable_pcie_error_reporting(pdev);
4872
4873         pci_set_master(pdev);
4874
4875         netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
4876                                    IAVF_MAX_REQ_QUEUES);
4877         if (!netdev) {
4878                 err = -ENOMEM;
4879                 goto err_alloc_etherdev;
4880         }
4881
4882         SET_NETDEV_DEV(netdev, &pdev->dev);
4883
4884         pci_set_drvdata(pdev, netdev);
4885         adapter = netdev_priv(netdev);
4886
4887         adapter->netdev = netdev;
4888         adapter->pdev = pdev;
4889
4890         hw = &adapter->hw;
4891         hw->back = adapter;
4892
4893         adapter->wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM,
4894                                               iavf_driver_name);
4895         if (!adapter->wq) {
4896                 err = -ENOMEM;
4897                 goto err_alloc_wq;
4898         }
4899
4900         adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
4901         iavf_change_state(adapter, __IAVF_STARTUP);
4902
4903         /* Call save state here because it relies on the adapter struct. */
4904         pci_save_state(pdev);
4905
4906         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
4907                               pci_resource_len(pdev, 0));
4908         if (!hw->hw_addr) {
4909                 err = -EIO;
4910                 goto err_ioremap;
4911         }
4912         hw->vendor_id = pdev->vendor;
4913         hw->device_id = pdev->device;
4914         pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
4915         hw->subsystem_vendor_id = pdev->subsystem_vendor;
4916         hw->subsystem_device_id = pdev->subsystem_device;
4917         hw->bus.device = PCI_SLOT(pdev->devfn);
4918         hw->bus.func = PCI_FUNC(pdev->devfn);
4919         hw->bus.bus_id = pdev->bus->number;
4920
4921         /* set up the locks for the AQ, do this only once in probe
4922          * and destroy them only once in remove
4923          */
4924         mutex_init(&adapter->crit_lock);
4925         mutex_init(&adapter->client_lock);
4926         mutex_init(&hw->aq.asq_mutex);
4927         mutex_init(&hw->aq.arq_mutex);
4928
4929         spin_lock_init(&adapter->mac_vlan_list_lock);
4930         spin_lock_init(&adapter->cloud_filter_list_lock);
4931         spin_lock_init(&adapter->fdir_fltr_lock);
4932         spin_lock_init(&adapter->adv_rss_lock);
4933
4934         INIT_LIST_HEAD(&adapter->mac_filter_list);
4935         INIT_LIST_HEAD(&adapter->vlan_filter_list);
4936         INIT_LIST_HEAD(&adapter->cloud_filter_list);
4937         INIT_LIST_HEAD(&adapter->fdir_list_head);
4938         INIT_LIST_HEAD(&adapter->adv_rss_list_head);
4939
4940         INIT_WORK(&adapter->reset_task, iavf_reset_task);
4941         INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
4942         INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
4943         INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task);
4944         queue_delayed_work(adapter->wq, &adapter->watchdog_task,
4945                            msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
4946
4947         /* Setup the wait queue for indicating transition to down status */
4948         init_waitqueue_head(&adapter->down_waitqueue);
4949
4950         /* Setup the wait queue for indicating virtchannel events */
4951         init_waitqueue_head(&adapter->vc_waitqueue);
4952
4953         return 0;
4954
4955 err_ioremap:
4956         destroy_workqueue(adapter->wq);
4957 err_alloc_wq:
4958         free_netdev(netdev);
4959 err_alloc_etherdev:
4960         pci_disable_pcie_error_reporting(pdev);
4961         pci_release_regions(pdev);
4962 err_pci_reg:
4963 err_dma:
4964         pci_disable_device(pdev);
4965         return err;
4966 }
4967
4968 /**
4969  * iavf_suspend - Power management suspend routine
4970  * @dev_d: device info pointer
4971  *
4972  * Called when the system (VM) is entering sleep/suspend.
4973  **/
4974 static int __maybe_unused iavf_suspend(struct device *dev_d)
4975 {
4976         struct net_device *netdev = dev_get_drvdata(dev_d);
4977         struct iavf_adapter *adapter = netdev_priv(netdev);
4978
4979         netif_device_detach(netdev);
4980
4981         while (!mutex_trylock(&adapter->crit_lock))
4982                 usleep_range(500, 1000);
4983
4984         if (netif_running(netdev)) {
4985                 rtnl_lock();
4986                 iavf_down(adapter);
4987                 rtnl_unlock();
4988         }
4989         iavf_free_misc_irq(adapter);
4990         iavf_reset_interrupt_capability(adapter);
4991
4992         mutex_unlock(&adapter->crit_lock);
4993
4994         return 0;
4995 }
4996
4997 /**
4998  * iavf_resume - Power management resume routine
4999  * @dev_d: device info pointer
5000  *
5001  * Called when the system (VM) is resumed from sleep/suspend.
5002  **/
5003 static int __maybe_unused iavf_resume(struct device *dev_d)
5004 {
5005         struct pci_dev *pdev = to_pci_dev(dev_d);
5006         struct iavf_adapter *adapter;
5007         u32 err;
5008
5009         adapter = iavf_pdev_to_adapter(pdev);
5010
5011         pci_set_master(pdev);
5012
5013         rtnl_lock();
5014         err = iavf_set_interrupt_capability(adapter);
5015         if (err) {
5016                 rtnl_unlock();
5017                 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
5018                 return err;
5019         }
5020         err = iavf_request_misc_irq(adapter);
5021         rtnl_unlock();
5022         if (err) {
5023                 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
5024                 return err;
5025         }
5026
5027         queue_work(adapter->wq, &adapter->reset_task);
5028
5029         netif_device_attach(adapter->netdev);
5030
5031         return err;
5032 }
5033
5034 /**
5035  * iavf_remove - Device Removal Routine
5036  * @pdev: PCI device information struct
5037  *
5038  * iavf_remove is called by the PCI subsystem to alert the driver
5039  * that it should release a PCI device.  The could be caused by a
5040  * Hot-Plug event, or because the driver is going to be removed from
5041  * memory.
5042  **/
5043 static void iavf_remove(struct pci_dev *pdev)
5044 {
5045         struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
5046         struct iavf_fdir_fltr *fdir, *fdirtmp;
5047         struct iavf_vlan_filter *vlf, *vlftmp;
5048         struct iavf_cloud_filter *cf, *cftmp;
5049         struct iavf_adv_rss *rss, *rsstmp;
5050         struct iavf_mac_filter *f, *ftmp;
5051         struct net_device *netdev;
5052         struct iavf_hw *hw;
5053         int err;
5054
5055         netdev = adapter->netdev;
5056         hw = &adapter->hw;
5057
5058         if (test_and_set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
5059                 return;
5060
5061         /* Wait until port initialization is complete.
5062          * There are flows where register/unregister netdev may race.
5063          */
5064         while (1) {
5065                 mutex_lock(&adapter->crit_lock);
5066                 if (adapter->state == __IAVF_RUNNING ||
5067                     adapter->state == __IAVF_DOWN ||
5068                     adapter->state == __IAVF_INIT_FAILED) {
5069                         mutex_unlock(&adapter->crit_lock);
5070                         break;
5071                 }
5072
5073                 mutex_unlock(&adapter->crit_lock);
5074                 usleep_range(500, 1000);
5075         }
5076         cancel_delayed_work_sync(&adapter->watchdog_task);
5077
5078         if (adapter->netdev_registered) {
5079                 rtnl_lock();
5080                 unregister_netdevice(netdev);
5081                 adapter->netdev_registered = false;
5082                 rtnl_unlock();
5083         }
5084         if (CLIENT_ALLOWED(adapter)) {
5085                 err = iavf_lan_del_device(adapter);
5086                 if (err)
5087                         dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
5088                                  err);
5089         }
5090
5091         mutex_lock(&adapter->crit_lock);
5092         dev_info(&adapter->pdev->dev, "Removing device\n");
5093         iavf_change_state(adapter, __IAVF_REMOVE);
5094
5095         iavf_request_reset(adapter);
5096         msleep(50);
5097         /* If the FW isn't responding, kick it once, but only once. */
5098         if (!iavf_asq_done(hw)) {
5099                 iavf_request_reset(adapter);
5100                 msleep(50);
5101         }
5102
5103         iavf_misc_irq_disable(adapter);
5104         /* Shut down all the garbage mashers on the detention level */
5105         cancel_work_sync(&adapter->reset_task);
5106         cancel_delayed_work_sync(&adapter->watchdog_task);
5107         cancel_work_sync(&adapter->adminq_task);
5108         cancel_delayed_work_sync(&adapter->client_task);
5109
5110         adapter->aq_required = 0;
5111         adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
5112
5113         iavf_free_all_tx_resources(adapter);
5114         iavf_free_all_rx_resources(adapter);
5115         iavf_free_misc_irq(adapter);
5116
5117         iavf_reset_interrupt_capability(adapter);
5118         iavf_free_q_vectors(adapter);
5119
5120         iavf_free_rss(adapter);
5121
5122         if (hw->aq.asq.count)
5123                 iavf_shutdown_adminq(hw);
5124
5125         /* destroy the locks only once, here */
5126         mutex_destroy(&hw->aq.arq_mutex);
5127         mutex_destroy(&hw->aq.asq_mutex);
5128         mutex_destroy(&adapter->client_lock);
5129         mutex_unlock(&adapter->crit_lock);
5130         mutex_destroy(&adapter->crit_lock);
5131
5132         iounmap(hw->hw_addr);
5133         pci_release_regions(pdev);
5134         iavf_free_queues(adapter);
5135         kfree(adapter->vf_res);
5136         spin_lock_bh(&adapter->mac_vlan_list_lock);
5137         /* If we got removed before an up/down sequence, we've got a filter
5138          * hanging out there that we need to get rid of.
5139          */
5140         list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
5141                 list_del(&f->list);
5142                 kfree(f);
5143         }
5144         list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
5145                                  list) {
5146                 list_del(&vlf->list);
5147                 kfree(vlf);
5148         }
5149
5150         spin_unlock_bh(&adapter->mac_vlan_list_lock);
5151
5152         spin_lock_bh(&adapter->cloud_filter_list_lock);
5153         list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
5154                 list_del(&cf->list);
5155                 kfree(cf);
5156         }
5157         spin_unlock_bh(&adapter->cloud_filter_list_lock);
5158
5159         spin_lock_bh(&adapter->fdir_fltr_lock);
5160         list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {
5161                 list_del(&fdir->list);
5162                 kfree(fdir);
5163         }
5164         spin_unlock_bh(&adapter->fdir_fltr_lock);
5165
5166         spin_lock_bh(&adapter->adv_rss_lock);
5167         list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
5168                                  list) {
5169                 list_del(&rss->list);
5170                 kfree(rss);
5171         }
5172         spin_unlock_bh(&adapter->adv_rss_lock);
5173
5174         destroy_workqueue(adapter->wq);
5175
5176         free_netdev(netdev);
5177
5178         pci_disable_pcie_error_reporting(pdev);
5179
5180         pci_disable_device(pdev);
5181 }
5182
5183 static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);
5184
5185 static struct pci_driver iavf_driver = {
5186         .name      = iavf_driver_name,
5187         .id_table  = iavf_pci_tbl,
5188         .probe     = iavf_probe,
5189         .remove    = iavf_remove,
5190         .driver.pm = &iavf_pm_ops,
5191         .shutdown  = iavf_shutdown,
5192 };
5193
5194 /**
5195  * iavf_init_module - Driver Registration Routine
5196  *
5197  * iavf_init_module is the first routine called when the driver is
5198  * loaded. All it does is register with the PCI subsystem.
5199  **/
5200 static int __init iavf_init_module(void)
5201 {
5202         pr_info("iavf: %s\n", iavf_driver_string);
5203
5204         pr_info("%s\n", iavf_copyright);
5205
5206         return pci_register_driver(&iavf_driver);
5207 }
5208
5209 module_init(iavf_init_module);
5210
5211 /**
5212  * iavf_exit_module - Driver Exit Cleanup Routine
5213  *
5214  * iavf_exit_module is called just before the driver is removed
5215  * from memory.
5216  **/
5217 static void __exit iavf_exit_module(void)
5218 {
5219         pci_unregister_driver(&iavf_driver);
5220 }
5221
5222 module_exit(iavf_exit_module);
5223
5224 /* iavf_main.c */