drivers/net/ethernet/sfc/efx_channels.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2018 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include "net_driver.h"
  12 #include <linux/module.h>
  13 #include <linux/filter.h>
  14 #include "efx_channels.h"
  15 #include "efx.h"
  16 #include "efx_common.h"
  17 #include "tx_common.h"
  18 #include "rx_common.h"
  19 #include "nic.h"
  20 #include "sriov.h"
  21 #include "workarounds.h"
  22
  23 /* This is the first interrupt mode to try out of:
  24  * 0 => MSI-X
  25  * 1 => MSI
  26  * 2 => legacy
  27  */
  28 unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX;
  29
  30 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
  31  * i.e. the number of CPUs among which we may distribute simultaneous
  32  * interrupt handling.
  33  *
  34  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
  35  * The default (0) means to assign an interrupt to each core.
  36  */
  37 unsigned int rss_cpus;
  38
  39 static unsigned int irq_adapt_low_thresh = 8000;
  40 module_param(irq_adapt_low_thresh, uint, 0644);
  41 MODULE_PARM_DESC(irq_adapt_low_thresh,
  42                  "Threshold score for reducing IRQ moderation");
  43
  44 static unsigned int irq_adapt_high_thresh = 16000;
  45 module_param(irq_adapt_high_thresh, uint, 0644);
  46 MODULE_PARM_DESC(irq_adapt_high_thresh,
  47                  "Threshold score for increasing IRQ moderation");
  48
  49 static const struct efx_channel_type efx_default_channel_type;
  50
  51 /*************
  52  * INTERRUPTS
  53  *************/
  54
  55 static unsigned int count_online_cores(struct efx_nic *efx, bool local_node)
  56 {
  57         cpumask_var_t filter_mask;
  58         unsigned int count;
  59         int cpu;
  60
  61         if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) {
  62                 netif_warn(efx, probe, efx->net_dev,
  63                            "RSS disabled due to allocation failure\n");
  64                 return 1;
  65         }
  66
  67         cpumask_copy(filter_mask, cpu_online_mask);
  68         if (local_node)
  69                 cpumask_and(filter_mask, filter_mask,
  70                             cpumask_of_pcibus(efx->pci_dev->bus));
  71
  72         count = 0;
  73         for_each_cpu(cpu, filter_mask) {
  74                 ++count;
  75                 cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu));
  76         }
  77
  78         free_cpumask_var(filter_mask);
  79
  80         return count;
  81 }
  82
  83 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
  84 {
  85         unsigned int count;
  86
  87         if (rss_cpus) {
  88                 count = rss_cpus;
  89         } else {
  90                 count = count_online_cores(efx, true);
  91
  92                 /* If no online CPUs in local node, fallback to any online CPUs */
  93                 if (count == 0)
  94                         count = count_online_cores(efx, false);
  95         }
  96
  97         if (count > EFX_MAX_RX_QUEUES) {
  98                 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
  99                                "Reducing number of rx queues from %u to %u.\n",
 100                                count, EFX_MAX_RX_QUEUES);
 101                 count = EFX_MAX_RX_QUEUES;
 102         }
 103
 104         /* If RSS is requested for the PF *and* VFs then we can't write RSS
 105          * table entries that are inaccessible to VFs
 106          */
 107 #ifdef CONFIG_SFC_SRIOV
 108         if (efx->type->sriov_wanted) {
 109                 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
 110                     count > efx_vf_size(efx)) {
 111                         netif_warn(efx, probe, efx->net_dev,
 112                                    "Reducing number of RSS channels from %u to %u for "
 113                                    "VF support. Increase vf-msix-limit to use more "
 114                                    "channels on the PF.\n",
 115                                    count, efx_vf_size(efx));
 116                         count = efx_vf_size(efx);
 117                 }
 118         }
 119 #endif
 120
 121         return count;
 122 }
 123
 124 static int efx_allocate_msix_channels(struct efx_nic *efx,
 125                                       unsigned int max_channels,
 126                                       unsigned int extra_channels,
 127                                       unsigned int parallelism)
 128 {
 129         unsigned int n_channels = parallelism;
 130         int vec_count;
 131         int tx_per_ev;
 132         int n_xdp_tx;
 133         int n_xdp_ev;
 134
 135         if (efx_separate_tx_channels)
 136                 n_channels *= 2;
 137         n_channels += extra_channels;
 138
 139         /* To allow XDP transmit to happen from arbitrary NAPI contexts
 140          * we allocate a TX queue per CPU. We share event queues across
 141          * multiple tx queues, assuming tx and ev queues are both
 142          * maximum size.
 143          */
 144         tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx);
 145         tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL);
 146         n_xdp_tx = num_possible_cpus();
 147         n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev);
 148
 149         vec_count = pci_msix_vec_count(efx->pci_dev);
 150         if (vec_count < 0)
 151                 return vec_count;
 152
 153         max_channels = min_t(unsigned int, vec_count, max_channels);
 154
 155         /* Check resources.
 156          * We need a channel per event queue, plus a VI per tx queue.
 157          * This may be more pessimistic than it needs to be.
 158          */
 159         if (n_channels >= max_channels) {
 160                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 161                 netif_warn(efx, drv, efx->net_dev,
 162                            "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
 163                            n_xdp_ev, n_channels, max_channels);
 164                 netif_warn(efx, drv, efx->net_dev,
 165                            "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
 166         } else if (n_channels + n_xdp_tx > efx->max_vis) {
 167                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 168                 netif_warn(efx, drv, efx->net_dev,
 169                            "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n",
 170                            n_xdp_tx, n_channels, efx->max_vis);
 171                 netif_warn(efx, drv, efx->net_dev,
 172                            "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
 173         } else if (n_channels + n_xdp_ev > max_channels) {
 174                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED;
 175                 netif_warn(efx, drv, efx->net_dev,
 176                            "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
 177                            n_xdp_ev, n_channels, max_channels);
 178
 179                 n_xdp_ev = max_channels - n_channels;
 180                 netif_warn(efx, drv, efx->net_dev,
 181                            "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n",
 182                            DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev));
 183         } else {
 184                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED;
 185         }
 186
 187         if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) {
 188                 efx->n_xdp_channels = n_xdp_ev;
 189                 efx->xdp_tx_per_channel = tx_per_ev;
 190                 efx->xdp_tx_queue_count = n_xdp_tx;
 191                 n_channels += n_xdp_ev;
 192                 netif_dbg(efx, drv, efx->net_dev,
 193                           "Allocating %d TX and %d event queues for XDP\n",
 194                           n_xdp_ev * tx_per_ev, n_xdp_ev);
 195         } else {
 196                 efx->n_xdp_channels = 0;
 197                 efx->xdp_tx_per_channel = 0;
 198                 efx->xdp_tx_queue_count = n_xdp_tx;
 199         }
 200
 201         if (vec_count < n_channels) {
 202                 netif_err(efx, drv, efx->net_dev,
 203                           "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
 204                           vec_count, n_channels);
 205                 netif_err(efx, drv, efx->net_dev,
 206                           "WARNING: Performance may be reduced.\n");
 207                 n_channels = vec_count;
 208         }
 209
 210         n_channels = min(n_channels, max_channels);
 211
 212         efx->n_channels = n_channels;
 213
 214         /* Ignore XDP tx channels when creating rx channels. */
 215         n_channels -= efx->n_xdp_channels;
 216
 217         if (efx_separate_tx_channels) {
 218                 efx->n_tx_channels =
 219                         min(max(n_channels / 2, 1U),
 220                             efx->max_tx_channels);
 221                 efx->tx_channel_offset =
 222                         n_channels - efx->n_tx_channels;
 223                 efx->n_rx_channels =
 224                         max(n_channels -
 225                             efx->n_tx_channels, 1U);
 226         } else {
 227                 efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
 228                 efx->tx_channel_offset = 0;
 229                 efx->n_rx_channels = n_channels;
 230         }
 231
 232         efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
 233         efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
 234
 235         efx->xdp_channel_offset = n_channels;
 236
 237         netif_dbg(efx, drv, efx->net_dev,
 238                   "Allocating %u RX channels\n",
 239                   efx->n_rx_channels);
 240
 241         return efx->n_channels;
 242 }
 243
 244 /* Probe the number and type of interrupts we are able to obtain, and
 245  * the resulting numbers of channels and RX queues.
 246  */
 247 int efx_probe_interrupts(struct efx_nic *efx)
 248 {
 249         unsigned int extra_channels = 0;
 250         unsigned int rss_spread;
 251         unsigned int i, j;
 252         int rc;
 253
 254         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
 255                 if (efx->extra_channel_type[i])
 256                         ++extra_channels;
 257
 258         if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
 259                 unsigned int parallelism = efx_wanted_parallelism(efx);
 260                 struct msix_entry xentries[EFX_MAX_CHANNELS];
 261                 unsigned int n_channels;
 262
 263                 rc = efx_allocate_msix_channels(efx, efx->max_channels,
 264                                                 extra_channels, parallelism);
 265                 if (rc >= 0) {
 266                         n_channels = rc;
 267                         for (i = 0; i < n_channels; i++)
 268                                 xentries[i].entry = i;
 269                         rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
 270                                                    n_channels);
 271                 }
 272                 if (rc < 0) {
 273                         /* Fall back to single channel MSI */
 274                         netif_err(efx, drv, efx->net_dev,
 275                                   "could not enable MSI-X\n");
 276                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
 277                                 efx->interrupt_mode = EFX_INT_MODE_MSI;
 278                         else
 279                                 return rc;
 280                 } else if (rc < n_channels) {
 281                         netif_err(efx, drv, efx->net_dev,
 282                                   "WARNING: Insufficient MSI-X vectors"
 283                                   " available (%d < %u).\n", rc, n_channels);
 284                         netif_err(efx, drv, efx->net_dev,
 285                                   "WARNING: Performance may be reduced.\n");
 286                         n_channels = rc;
 287                 }
 288
 289                 if (rc > 0) {
 290                         for (i = 0; i < efx->n_channels; i++)
 291                                 efx_get_channel(efx, i)->irq =
 292                                         xentries[i].vector;
 293                 }
 294         }
 295
 296         /* Try single interrupt MSI */
 297         if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
 298                 efx->n_channels = 1;
 299                 efx->n_rx_channels = 1;
 300                 efx->n_tx_channels = 1;
 301                 efx->tx_channel_offset = 0;
 302                 efx->n_xdp_channels = 0;
 303                 efx->xdp_channel_offset = efx->n_channels;
 304                 rc = pci_enable_msi(efx->pci_dev);
 305                 if (rc == 0) {
 306                         efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
 307                 } else {
 308                         netif_err(efx, drv, efx->net_dev,
 309                                   "could not enable MSI\n");
 310                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
 311                                 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
 312                         else
 313                                 return rc;
 314                 }
 315         }
 316
 317         /* Assume legacy interrupts */
 318         if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
 319                 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
 320                 efx->n_rx_channels = 1;
 321                 efx->n_tx_channels = 1;
 322                 efx->tx_channel_offset = efx_separate_tx_channels ? 1 : 0;
 323                 efx->n_xdp_channels = 0;
 324                 efx->xdp_channel_offset = efx->n_channels;
 325                 efx->legacy_irq = efx->pci_dev->irq;
 326         }
 327
 328         /* Assign extra channels if possible, before XDP channels */
 329         efx->n_extra_tx_channels = 0;
 330         j = efx->xdp_channel_offset;
 331         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
 332                 if (!efx->extra_channel_type[i])
 333                         continue;
 334                 if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
 335                         efx->extra_channel_type[i]->handle_no_channel(efx);
 336                 } else {
 337                         --j;
 338                         efx_get_channel(efx, j)->type =
 339                                 efx->extra_channel_type[i];
 340                         if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
 341                                 efx->n_extra_tx_channels++;
 342                 }
 343         }
 344
 345         rss_spread = efx->n_rx_channels;
 346         /* RSS might be usable on VFs even if it is disabled on the PF */
 347 #ifdef CONFIG_SFC_SRIOV
 348         if (efx->type->sriov_wanted) {
 349                 efx->rss_spread = ((rss_spread > 1 ||
 350                                     !efx->type->sriov_wanted(efx)) ?
 351                                    rss_spread : efx_vf_size(efx));
 352                 return 0;
 353         }
 354 #endif
 355         efx->rss_spread = rss_spread;
 356
 357         return 0;
 358 }
 359
 360 #if defined(CONFIG_SMP)
 361 void efx_set_interrupt_affinity(struct efx_nic *efx)
 362 {
 363         const struct cpumask *numa_mask = cpumask_of_pcibus(efx->pci_dev->bus);
 364         struct efx_channel *channel;
 365         unsigned int cpu;
 366
 367         /* If no online CPUs in local node, fallback to any online CPU */
 368         if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids)
 369                 numa_mask = cpu_online_mask;
 370
 371         cpu = -1;
 372         efx_for_each_channel(channel, efx) {
 373                 cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask);
 374                 if (cpu >= nr_cpu_ids)
 375                         cpu = cpumask_first_and(cpu_online_mask, numa_mask);
 376                 irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
 377         }
 378 }
 379
 380 void efx_clear_interrupt_affinity(struct efx_nic *efx)
 381 {
 382         struct efx_channel *channel;
 383
 384         efx_for_each_channel(channel, efx)
 385                 irq_set_affinity_hint(channel->irq, NULL);
 386 }
 387 #else
 388 void
 389 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 390 {
 391 }
 392
 393 void
 394 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 395 {
 396 }
 397 #endif /* CONFIG_SMP */
 398
 399 void efx_remove_interrupts(struct efx_nic *efx)
 400 {
 401         struct efx_channel *channel;
 402
 403         /* Remove MSI/MSI-X interrupts */
 404         efx_for_each_channel(channel, efx)
 405                 channel->irq = 0;
 406         pci_disable_msi(efx->pci_dev);
 407         pci_disable_msix(efx->pci_dev);
 408
 409         /* Remove legacy interrupt */
 410         efx->legacy_irq = 0;
 411 }
 412
 413 /***************
 414  * EVENT QUEUES
 415  ***************/
 416
 417 /* Create event queue
 418  * Event queue memory allocations are done only once.  If the channel
 419  * is reset, the memory buffer will be reused; this guards against
 420  * errors during channel reset and also simplifies interrupt handling.
 421  */
 422 int efx_probe_eventq(struct efx_channel *channel)
 423 {
 424         struct efx_nic *efx = channel->efx;
 425         unsigned long entries;
 426
 427         netif_dbg(efx, probe, efx->net_dev,
 428                   "chan %d create event queue\n", channel->channel);
 429
 430         /* Build an event queue with room for one event per tx and rx buffer,
 431          * plus some extra for link state events and MCDI completions.
 432          */
 433         entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
 434         EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
 435         channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
 436
 437         return efx_nic_probe_eventq(channel);
 438 }
 439
 440 /* Prepare channel's event queue */
 441 int efx_init_eventq(struct efx_channel *channel)
 442 {
 443         struct efx_nic *efx = channel->efx;
 444         int rc;
 445
 446         EFX_WARN_ON_PARANOID(channel->eventq_init);
 447
 448         netif_dbg(efx, drv, efx->net_dev,
 449                   "chan %d init event queue\n", channel->channel);
 450
 451         rc = efx_nic_init_eventq(channel);
 452         if (rc == 0) {
 453                 efx->type->push_irq_moderation(channel);
 454                 channel->eventq_read_ptr = 0;
 455                 channel->eventq_init = true;
 456         }
 457         return rc;
 458 }
 459
 460 /* Enable event queue processing and NAPI */
 461 void efx_start_eventq(struct efx_channel *channel)
 462 {
 463         netif_dbg(channel->efx, ifup, channel->efx->net_dev,
 464                   "chan %d start event queue\n", channel->channel);
 465
 466         /* Make sure the NAPI handler sees the enabled flag set */
 467         channel->enabled = true;
 468         smp_wmb();
 469
 470         napi_enable(&channel->napi_str);
 471         efx_nic_eventq_read_ack(channel);
 472 }
 473
 474 /* Disable event queue processing and NAPI */
 475 void efx_stop_eventq(struct efx_channel *channel)
 476 {
 477         if (!channel->enabled)
 478                 return;
 479
 480         napi_disable(&channel->napi_str);
 481         channel->enabled = false;
 482 }
 483
 484 void efx_fini_eventq(struct efx_channel *channel)
 485 {
 486         if (!channel->eventq_init)
 487                 return;
 488
 489         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 490                   "chan %d fini event queue\n", channel->channel);
 491
 492         efx_nic_fini_eventq(channel);
 493         channel->eventq_init = false;
 494 }
 495
 496 void efx_remove_eventq(struct efx_channel *channel)
 497 {
 498         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 499                   "chan %d remove event queue\n", channel->channel);
 500
 501         efx_nic_remove_eventq(channel);
 502 }
 503
 504 /**************************************************************************
 505  *
 506  * Channel handling
 507  *
 508  *************************************************************************/
 509
 510 #ifdef CONFIG_RFS_ACCEL
 511 static void efx_filter_rfs_expire(struct work_struct *data)
 512 {
 513         struct delayed_work *dwork = to_delayed_work(data);
 514         struct efx_channel *channel;
 515         unsigned int time, quota;
 516
 517         channel = container_of(dwork, struct efx_channel, filter_work);
 518         time = jiffies - channel->rfs_last_expiry;
 519         quota = channel->rfs_filter_count * time / (30 * HZ);
 520         if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota)))
 521                 channel->rfs_last_expiry += time;
 522         /* Ensure we do more work eventually even if NAPI poll is not happening */
 523         schedule_delayed_work(dwork, 30 * HZ);
 524 }
 525 #endif
 526
 527 /* Allocate and initialise a channel structure. */
 528 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)
 529 {
 530         struct efx_rx_queue *rx_queue;
 531         struct efx_tx_queue *tx_queue;
 532         struct efx_channel *channel;
 533         int j;
 534
 535         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 536         if (!channel)
 537                 return NULL;
 538
 539         channel->efx = efx;
 540         channel->channel = i;
 541         channel->type = &efx_default_channel_type;
 542
 543         for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
 544                 tx_queue = &channel->tx_queue[j];
 545                 tx_queue->efx = efx;
 546                 tx_queue->queue = -1;
 547                 tx_queue->label = j;
 548                 tx_queue->channel = channel;
 549         }
 550
 551 #ifdef CONFIG_RFS_ACCEL
 552         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 553 #endif
 554
 555         rx_queue = &channel->rx_queue;
 556         rx_queue->efx = efx;
 557         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 558
 559         return channel;
 560 }
 561
 562 int efx_init_channels(struct efx_nic *efx)
 563 {
 564         unsigned int i;
 565
 566         for (i = 0; i < EFX_MAX_CHANNELS; i++) {
 567                 efx->channel[i] = efx_alloc_channel(efx, i);
 568                 if (!efx->channel[i])
 569                         return -ENOMEM;
 570                 efx->msi_context[i].efx = efx;
 571                 efx->msi_context[i].index = i;
 572         }
 573
 574         /* Higher numbered interrupt modes are less capable! */
 575         efx->interrupt_mode = min(efx->type->min_interrupt_mode,
 576                                   efx_interrupt_mode);
 577
 578         efx->max_channels = EFX_MAX_CHANNELS;
 579         efx->max_tx_channels = EFX_MAX_CHANNELS;
 580
 581         return 0;
 582 }
 583
 584 void efx_fini_channels(struct efx_nic *efx)
 585 {
 586         unsigned int i;
 587
 588         for (i = 0; i < EFX_MAX_CHANNELS; i++)
 589                 if (efx->channel[i]) {
 590                         kfree(efx->channel[i]);
 591                         efx->channel[i] = NULL;
 592                 }
 593 }
 594
 595 /* Allocate and initialise a channel structure, copying parameters
 596  * (but not resources) from an old channel structure.
 597  */
 598 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
 599 {
 600         struct efx_rx_queue *rx_queue;
 601         struct efx_tx_queue *tx_queue;
 602         struct efx_channel *channel;
 603         int j;
 604
 605         channel = kmalloc(sizeof(*channel), GFP_KERNEL);
 606         if (!channel)
 607                 return NULL;
 608
 609         *channel = *old_channel;
 610
 611         channel->napi_dev = NULL;
 612         INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
 613         channel->napi_str.napi_id = 0;
 614         channel->napi_str.state = 0;
 615         memset(&channel->eventq, 0, sizeof(channel->eventq));
 616
 617         for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
 618                 tx_queue = &channel->tx_queue[j];
 619                 if (tx_queue->channel)
 620                         tx_queue->channel = channel;
 621                 tx_queue->buffer = NULL;
 622                 tx_queue->cb_page = NULL;
 623                 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
 624         }
 625
 626         rx_queue = &channel->rx_queue;
 627         rx_queue->buffer = NULL;
 628         memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
 629         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 630 #ifdef CONFIG_RFS_ACCEL
 631         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 632 #endif
 633
 634         return channel;
 635 }
 636
 637 static int efx_probe_channel(struct efx_channel *channel)
 638 {
 639         struct efx_tx_queue *tx_queue;
 640         struct efx_rx_queue *rx_queue;
 641         int rc;
 642
 643         netif_dbg(channel->efx, probe, channel->efx->net_dev,
 644                   "creating channel %d\n", channel->channel);
 645
 646         rc = channel->type->pre_probe(channel);
 647         if (rc)
 648                 goto fail;
 649
 650         rc = efx_probe_eventq(channel);
 651         if (rc)
 652                 goto fail;
 653
 654         efx_for_each_channel_tx_queue(tx_queue, channel) {
 655                 rc = efx_probe_tx_queue(tx_queue);
 656                 if (rc)
 657                         goto fail;
 658         }
 659
 660         efx_for_each_channel_rx_queue(rx_queue, channel) {
 661                 rc = efx_probe_rx_queue(rx_queue);
 662                 if (rc)
 663                         goto fail;
 664         }
 665
 666         channel->rx_list = NULL;
 667
 668         return 0;
 669
 670 fail:
 671         efx_remove_channel(channel);
 672         return rc;
 673 }
 674
 675 static void efx_get_channel_name(struct efx_channel *channel, char *buf,
 676                                  size_t len)
 677 {
 678         struct efx_nic *efx = channel->efx;
 679         const char *type;
 680         int number;
 681
 682         number = channel->channel;
 683
 684         if (number >= efx->xdp_channel_offset &&
 685             !WARN_ON_ONCE(!efx->n_xdp_channels)) {
 686                 type = "-xdp";
 687                 number -= efx->xdp_channel_offset;
 688         } else if (efx->tx_channel_offset == 0) {
 689                 type = "";
 690         } else if (number < efx->tx_channel_offset) {
 691                 type = "-rx";
 692         } else {
 693                 type = "-tx";
 694                 number -= efx->tx_channel_offset;
 695         }
 696         snprintf(buf, len, "%s%s-%d", efx->name, type, number);
 697 }
 698
 699 void efx_set_channel_names(struct efx_nic *efx)
 700 {
 701         struct efx_channel *channel;
 702
 703         efx_for_each_channel(channel, efx)
 704                 channel->type->get_name(channel,
 705                                         efx->msi_context[channel->channel].name,
 706                                         sizeof(efx->msi_context[0].name));
 707 }
 708
 709 int efx_probe_channels(struct efx_nic *efx)
 710 {
 711         struct efx_channel *channel;
 712         int rc;
 713
 714         /* Restart special buffer allocation */
 715         efx->next_buffer_table = 0;
 716
 717         /* Probe channels in reverse, so that any 'extra' channels
 718          * use the start of the buffer table. This allows the traffic
 719          * channels to be resized without moving them or wasting the
 720          * entries before them.
 721          */
 722         efx_for_each_channel_rev(channel, efx) {
 723                 rc = efx_probe_channel(channel);
 724                 if (rc) {
 725                         netif_err(efx, probe, efx->net_dev,
 726                                   "failed to create channel %d\n",
 727                                   channel->channel);
 728                         goto fail;
 729                 }
 730         }
 731         efx_set_channel_names(efx);
 732
 733         return 0;
 734
 735 fail:
 736         efx_remove_channels(efx);
 737         return rc;
 738 }
 739
 740 void efx_remove_channel(struct efx_channel *channel)
 741 {
 742         struct efx_tx_queue *tx_queue;
 743         struct efx_rx_queue *rx_queue;
 744
 745         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 746                   "destroy chan %d\n", channel->channel);
 747
 748         efx_for_each_channel_rx_queue(rx_queue, channel)
 749                 efx_remove_rx_queue(rx_queue);
 750         efx_for_each_channel_tx_queue(tx_queue, channel)
 751                 efx_remove_tx_queue(tx_queue);
 752         efx_remove_eventq(channel);
 753         channel->type->post_remove(channel);
 754 }
 755
 756 void efx_remove_channels(struct efx_nic *efx)
 757 {
 758         struct efx_channel *channel;
 759
 760         efx_for_each_channel(channel, efx)
 761                 efx_remove_channel(channel);
 762
 763         kfree(efx->xdp_tx_queues);
 764 }
 765
 766 static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
 767                                 struct efx_tx_queue *tx_queue)
 768 {
 769         if (xdp_queue_number >= efx->xdp_tx_queue_count)
 770                 return -EINVAL;
 771
 772         netif_dbg(efx, drv, efx->net_dev,
 773                   "Channel %u TXQ %u is XDP %u, HW %u\n",
 774                   tx_queue->channel->channel, tx_queue->label,
 775                   xdp_queue_number, tx_queue->queue);
 776         efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
 777         return 0;
 778 }
 779
 780 static void efx_set_xdp_channels(struct efx_nic *efx)
 781 {
 782         struct efx_tx_queue *tx_queue;
 783         struct efx_channel *channel;
 784         unsigned int next_queue = 0;
 785         int xdp_queue_number = 0;
 786         int rc;
 787
 788         /* We need to mark which channels really have RX and TX
 789          * queues, and adjust the TX queue numbers if we have separate
 790          * RX-only and TX-only channels.
 791          */
 792         efx_for_each_channel(channel, efx) {
 793                 if (channel->channel < efx->tx_channel_offset)
 794                         continue;
 795
 796                 if (efx_channel_is_xdp_tx(channel)) {
 797                         efx_for_each_channel_tx_queue(tx_queue, channel) {
 798                                 tx_queue->queue = next_queue++;
 799                                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
 800                                                           tx_queue);
 801                                 if (rc == 0)
 802                                         xdp_queue_number++;
 803                         }
 804                 } else {
 805                         efx_for_each_channel_tx_queue(tx_queue, channel) {
 806                                 tx_queue->queue = next_queue++;
 807                                 netif_dbg(efx, drv, efx->net_dev,
 808                                           "Channel %u TXQ %u is HW %u\n",
 809                                           channel->channel, tx_queue->label,
 810                                           tx_queue->queue);
 811                         }
 812
 813                         /* If XDP is borrowing queues from net stack, it must
 814                          * use the queue with no csum offload, which is the
 815                          * first one of the channel
 816                          * (note: tx_queue_by_type is not initialized yet)
 817                          */
 818                         if (efx->xdp_txq_queues_mode ==
 819                             EFX_XDP_TX_QUEUES_BORROWED) {
 820                                 tx_queue = &channel->tx_queue[0];
 821                                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
 822                                                           tx_queue);
 823                                 if (rc == 0)
 824                                         xdp_queue_number++;
 825                         }
 826                 }
 827         }
 828         WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
 829                 xdp_queue_number != efx->xdp_tx_queue_count);
 830         WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
 831                 xdp_queue_number > efx->xdp_tx_queue_count);
 832
 833         /* If we have more CPUs than assigned XDP TX queues, assign the already
 834          * existing queues to the exceeding CPUs
 835          */
 836         next_queue = 0;
 837         while (xdp_queue_number < efx->xdp_tx_queue_count) {
 838                 tx_queue = efx->xdp_tx_queues[next_queue++];
 839                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
 840                 if (rc == 0)
 841                         xdp_queue_number++;
 842         }
 843 }
 844
 845 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
 846 {
 847         struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel,
 848                            *ptp_channel = efx_ptp_channel(efx);
 849         struct efx_ptp_data *ptp_data = efx->ptp_data;
 850         unsigned int i, next_buffer_table = 0;
 851         u32 old_rxq_entries, old_txq_entries;
 852         int rc, rc2;
 853
 854         rc = efx_check_disabled(efx);
 855         if (rc)
 856                 return rc;
 857
 858         /* Not all channels should be reallocated. We must avoid
 859          * reallocating their buffer table entries.
 860          */
 861         efx_for_each_channel(channel, efx) {
 862                 struct efx_rx_queue *rx_queue;
 863                 struct efx_tx_queue *tx_queue;
 864
 865                 if (channel->type->copy)
 866                         continue;
 867                 next_buffer_table = max(next_buffer_table,
 868                                         channel->eventq.index +
 869                                         channel->eventq.entries);
 870                 efx_for_each_channel_rx_queue(rx_queue, channel)
 871                         next_buffer_table = max(next_buffer_table,
 872                                                 rx_queue->rxd.index +
 873                                                 rx_queue->rxd.entries);
 874                 efx_for_each_channel_tx_queue(tx_queue, channel)
 875                         next_buffer_table = max(next_buffer_table,
 876                                                 tx_queue->txd.index +
 877                                                 tx_queue->txd.entries);
 878         }
 879
 880         efx_device_detach_sync(efx);
 881         efx_stop_all(efx);
 882         efx_soft_disable_interrupts(efx);
 883
 884         /* Clone channels (where possible) */
 885         memset(other_channel, 0, sizeof(other_channel));
 886         for (i = 0; i < efx->n_channels; i++) {
 887                 channel = efx->channel[i];
 888                 if (channel->type->copy)
 889                         channel = channel->type->copy(channel);
 890                 if (!channel) {
 891                         rc = -ENOMEM;
 892                         goto out;
 893                 }
 894                 other_channel[i] = channel;
 895         }
 896
 897         /* Swap entry counts and channel pointers */
 898         old_rxq_entries = efx->rxq_entries;
 899         old_txq_entries = efx->txq_entries;
 900         efx->rxq_entries = rxq_entries;
 901         efx->txq_entries = txq_entries;
 902         for (i = 0; i < efx->n_channels; i++)
 903                 swap(efx->channel[i], other_channel[i]);
 904
 905         /* Restart buffer table allocation */
 906         efx->next_buffer_table = next_buffer_table;
 907
 908         for (i = 0; i < efx->n_channels; i++) {
 909                 channel = efx->channel[i];
 910                 if (!channel->type->copy)
 911                         continue;
 912                 rc = efx_probe_channel(channel);
 913                 if (rc)
 914                         goto rollback;
 915                 efx_init_napi_channel(efx->channel[i]);
 916         }
 917
 918         efx_set_xdp_channels(efx);
 919 out:
 920         efx->ptp_data = NULL;
 921         /* Destroy unused channel structures */
 922         for (i = 0; i < efx->n_channels; i++) {
 923                 channel = other_channel[i];
 924                 if (channel && channel->type->copy) {
 925                         efx_fini_napi_channel(channel);
 926                         efx_remove_channel(channel);
 927                         kfree(channel);
 928                 }
 929         }
 930
 931         efx->ptp_data = ptp_data;
 932         rc2 = efx_soft_enable_interrupts(efx);
 933         if (rc2) {
 934                 rc = rc ? rc : rc2;
 935                 netif_err(efx, drv, efx->net_dev,
 936                           "unable to restart interrupts on channel reallocation\n");
 937                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
 938         } else {
 939                 efx_start_all(efx);
 940                 efx_device_attach_if_not_resetting(efx);
 941         }
 942         return rc;
 943
 944 rollback:
 945         /* Swap back */
 946         efx->rxq_entries = old_rxq_entries;
 947         efx->txq_entries = old_txq_entries;
 948         for (i = 0; i < efx->n_channels; i++)
 949                 swap(efx->channel[i], other_channel[i]);
 950         efx_ptp_update_channel(efx, ptp_channel);
 951         goto out;
 952 }
 953
 954 int efx_set_channels(struct efx_nic *efx)
 955 {
 956         struct efx_channel *channel;
 957         int rc;
 958
 959         if (efx->xdp_tx_queue_count) {
 960                 EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
 961
 962                 /* Allocate array for XDP TX queue lookup. */
 963                 efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
 964                                              sizeof(*efx->xdp_tx_queues),
 965                                              GFP_KERNEL);
 966                 if (!efx->xdp_tx_queues)
 967                         return -ENOMEM;
 968         }
 969
 970         efx_for_each_channel(channel, efx) {
 971                 if (channel->channel < efx->n_rx_channels)
 972                         channel->rx_queue.core_index = channel->channel;
 973                 else
 974                         channel->rx_queue.core_index = -1;
 975         }
 976
 977         efx_set_xdp_channels(efx);
 978
 979         rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
 980         if (rc)
 981                 return rc;
 982         return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
 983 }
 984
 985 static bool efx_default_channel_want_txqs(struct efx_channel *channel)
 986 {
 987         return channel->channel - channel->efx->tx_channel_offset <
 988                 channel->efx->n_tx_channels;
 989 }
 990
 991 /*************
 992  * START/STOP
 993  *************/
 994
 995 int efx_soft_enable_interrupts(struct efx_nic *efx)
 996 {
 997         struct efx_channel *channel, *end_channel;
 998         int rc;
 999
1000         BUG_ON(efx->state == STATE_DISABLED);
1001
1002         efx->irq_soft_enabled = true;
1003         smp_wmb();
1004
1005         efx_for_each_channel(channel, efx) {
1006                 if (!channel->type->keep_eventq) {
1007                         rc = efx_init_eventq(channel);
1008                         if (rc)
1009                                 goto fail;
1010                 }
1011                 efx_start_eventq(channel);
1012         }
1013
1014         efx_mcdi_mode_event(efx);
1015
1016         return 0;
1017 fail:
1018         end_channel = channel;
1019         efx_for_each_channel(channel, efx) {
1020                 if (channel == end_channel)
1021                         break;
1022                 efx_stop_eventq(channel);
1023                 if (!channel->type->keep_eventq)
1024                         efx_fini_eventq(channel);
1025         }
1026
1027         return rc;
1028 }
1029
1030 void efx_soft_disable_interrupts(struct efx_nic *efx)
1031 {
1032         struct efx_channel *channel;
1033
1034         if (efx->state == STATE_DISABLED)
1035                 return;
1036
1037         efx_mcdi_mode_poll(efx);
1038
1039         efx->irq_soft_enabled = false;
1040         smp_wmb();
1041
1042         if (efx->legacy_irq)
1043                 synchronize_irq(efx->legacy_irq);
1044
1045         efx_for_each_channel(channel, efx) {
1046                 if (channel->irq)
1047                         synchronize_irq(channel->irq);
1048
1049                 efx_stop_eventq(channel);
1050                 if (!channel->type->keep_eventq)
1051                         efx_fini_eventq(channel);
1052         }
1053
1054         /* Flush the asynchronous MCDI request queue */
1055         efx_mcdi_flush_async(efx);
1056 }
1057
1058 int efx_enable_interrupts(struct efx_nic *efx)
1059 {
1060         struct efx_channel *channel, *end_channel;
1061         int rc;
1062
1063         /* TODO: Is this really a bug? */
1064         BUG_ON(efx->state == STATE_DISABLED);
1065
1066         if (efx->eeh_disabled_legacy_irq) {
1067                 enable_irq(efx->legacy_irq);
1068                 efx->eeh_disabled_legacy_irq = false;
1069         }
1070
1071         efx->type->irq_enable_master(efx);
1072
1073         efx_for_each_channel(channel, efx) {
1074                 if (channel->type->keep_eventq) {
1075                         rc = efx_init_eventq(channel);
1076                         if (rc)
1077                                 goto fail;
1078                 }
1079         }
1080
1081         rc = efx_soft_enable_interrupts(efx);
1082         if (rc)
1083                 goto fail;
1084
1085         return 0;
1086
1087 fail:
1088         end_channel = channel;
1089         efx_for_each_channel(channel, efx) {
1090                 if (channel == end_channel)
1091                         break;
1092                 if (channel->type->keep_eventq)
1093                         efx_fini_eventq(channel);
1094         }
1095
1096         efx->type->irq_disable_non_ev(efx);
1097
1098         return rc;
1099 }
1100
1101 void efx_disable_interrupts(struct efx_nic *efx)
1102 {
1103         struct efx_channel *channel;
1104
1105         efx_soft_disable_interrupts(efx);
1106
1107         efx_for_each_channel(channel, efx) {
1108                 if (channel->type->keep_eventq)
1109                         efx_fini_eventq(channel);
1110         }
1111
1112         efx->type->irq_disable_non_ev(efx);
1113 }
1114
1115 void efx_start_channels(struct efx_nic *efx)
1116 {
1117         struct efx_tx_queue *tx_queue;
1118         struct efx_rx_queue *rx_queue;
1119         struct efx_channel *channel;
1120
1121         efx_for_each_channel_rev(channel, efx) {
1122                 if (channel->type->start)
1123                         channel->type->start(channel);
1124                 efx_for_each_channel_tx_queue(tx_queue, channel) {
1125                         efx_init_tx_queue(tx_queue);
1126                         atomic_inc(&efx->active_queues);
1127                 }
1128
1129                 efx_for_each_channel_rx_queue(rx_queue, channel) {
1130                         efx_init_rx_queue(rx_queue);
1131                         atomic_inc(&efx->active_queues);
1132                         efx_stop_eventq(channel);
1133                         efx_fast_push_rx_descriptors(rx_queue, false);
1134                         efx_start_eventq(channel);
1135                 }
1136
1137                 WARN_ON(channel->rx_pkt_n_frags);
1138         }
1139 }
1140
1141 void efx_stop_channels(struct efx_nic *efx)
1142 {
1143         struct efx_tx_queue *tx_queue;
1144         struct efx_rx_queue *rx_queue;
1145         struct efx_channel *channel;
1146         int rc = 0;
1147
1148         /* Stop special channels and RX refill.
1149          * The channel's stop has to be called first, since it might wait
1150          * for a sentinel RX to indicate the channel has fully drained.
1151          */
1152         efx_for_each_channel(channel, efx) {
1153                 if (channel->type->stop)
1154                         channel->type->stop(channel);
1155                 efx_for_each_channel_rx_queue(rx_queue, channel)
1156                         rx_queue->refill_enabled = false;
1157         }
1158
1159         efx_for_each_channel(channel, efx) {
1160                 /* RX packet processing is pipelined, so wait for the
1161                  * NAPI handler to complete.  At least event queue 0
1162                  * might be kept active by non-data events, so don't
1163                  * use napi_synchronize() but actually disable NAPI
1164                  * temporarily.
1165                  */
1166                 if (efx_channel_has_rx_queue(channel)) {
1167                         efx_stop_eventq(channel);
1168                         efx_start_eventq(channel);
1169                 }
1170         }
1171
1172         if (efx->type->fini_dmaq)
1173                 rc = efx->type->fini_dmaq(efx);
1174
1175         if (rc) {
1176                 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
1177         } else {
1178                 netif_dbg(efx, drv, efx->net_dev,
1179                           "successfully flushed all queues\n");
1180         }
1181
1182         efx_for_each_channel(channel, efx) {
1183                 efx_for_each_channel_rx_queue(rx_queue, channel)
1184                         efx_fini_rx_queue(rx_queue);
1185                 efx_for_each_channel_tx_queue(tx_queue, channel)
1186                         efx_fini_tx_queue(tx_queue);
1187         }
1188 }
1189
1190 /**************************************************************************
1191  *
1192  * NAPI interface
1193  *
1194  *************************************************************************/
1195
1196 /* Process channel's event queue
1197  *
1198  * This function is responsible for processing the event queue of a
1199  * single channel.  The caller must guarantee that this function will
1200  * never be concurrently called more than once on the same channel,
1201  * though different channels may be being processed concurrently.
1202  */
1203 static int efx_process_channel(struct efx_channel *channel, int budget)
1204 {
1205         struct efx_tx_queue *tx_queue;
1206         struct list_head rx_list;
1207         int spent;
1208
1209         if (unlikely(!channel->enabled))
1210                 return 0;
1211
1212         /* Prepare the batch receive list */
1213         EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1214         INIT_LIST_HEAD(&rx_list);
1215         channel->rx_list = &rx_list;
1216
1217         efx_for_each_channel_tx_queue(tx_queue, channel) {
1218                 tx_queue->pkts_compl = 0;
1219                 tx_queue->bytes_compl = 0;
1220         }
1221
1222         spent = efx_nic_process_eventq(channel, budget);
1223         if (spent && efx_channel_has_rx_queue(channel)) {
1224                 struct efx_rx_queue *rx_queue =
1225                         efx_channel_get_rx_queue(channel);
1226
1227                 efx_rx_flush_packet(channel);
1228                 efx_fast_push_rx_descriptors(rx_queue, true);
1229         }
1230
1231         /* Update BQL */
1232         efx_for_each_channel_tx_queue(tx_queue, channel) {
1233                 if (tx_queue->bytes_compl) {
1234                         netdev_tx_completed_queue(tx_queue->core_txq,
1235                                                   tx_queue->pkts_compl,
1236                                                   tx_queue->bytes_compl);
1237                 }
1238         }
1239
1240         /* Receive any packets we queued up */
1241         netif_receive_skb_list(channel->rx_list);
1242         channel->rx_list = NULL;
1243
1244         return spent;
1245 }
1246
1247 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1248 {
1249         int step = efx->irq_mod_step_us;
1250
1251         if (channel->irq_mod_score < irq_adapt_low_thresh) {
1252                 if (channel->irq_moderation_us > step) {
1253                         channel->irq_moderation_us -= step;
1254                         efx->type->push_irq_moderation(channel);
1255                 }
1256         } else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1257                 if (channel->irq_moderation_us <
1258                     efx->irq_rx_moderation_us) {
1259                         channel->irq_moderation_us += step;
1260                         efx->type->push_irq_moderation(channel);
1261                 }
1262         }
1263
1264         channel->irq_count = 0;
1265         channel->irq_mod_score = 0;
1266 }
1267
1268 /* NAPI poll handler
1269  *
1270  * NAPI guarantees serialisation of polls of the same device, which
1271  * provides the guarantee required by efx_process_channel().
1272  */
1273 static int efx_poll(struct napi_struct *napi, int budget)
1274 {
1275         struct efx_channel *channel =
1276                 container_of(napi, struct efx_channel, napi_str);
1277         struct efx_nic *efx = channel->efx;
1278 #ifdef CONFIG_RFS_ACCEL
1279         unsigned int time;
1280 #endif
1281         int spent;
1282
1283         netif_vdbg(efx, intr, efx->net_dev,
1284                    "channel %d NAPI poll executing on CPU %d\n",
1285                    channel->channel, raw_smp_processor_id());
1286
1287         spent = efx_process_channel(channel, budget);
1288
1289         xdp_do_flush_map();
1290
1291         if (spent < budget) {
1292                 if (efx_channel_has_rx_queue(channel) &&
1293                     efx->irq_rx_adaptive &&
1294                     unlikely(++channel->irq_count == 1000)) {
1295                         efx_update_irq_mod(efx, channel);
1296                 }
1297
1298 #ifdef CONFIG_RFS_ACCEL
1299                 /* Perhaps expire some ARFS filters */
1300                 time = jiffies - channel->rfs_last_expiry;
1301                 /* Would our quota be >= 20? */
1302                 if (channel->rfs_filter_count * time >= 600 * HZ)
1303                         mod_delayed_work(system_wq, &channel->filter_work, 0);
1304 #endif
1305
1306                 /* There is no race here; although napi_disable() will
1307                  * only wait for napi_complete(), this isn't a problem
1308                  * since efx_nic_eventq_read_ack() will have no effect if
1309                  * interrupts have already been disabled.
1310                  */
1311                 if (napi_complete_done(napi, spent))
1312                         efx_nic_eventq_read_ack(channel);
1313         }
1314
1315         return spent;
1316 }
1317
1318 void efx_init_napi_channel(struct efx_channel *channel)
1319 {
1320         struct efx_nic *efx = channel->efx;
1321
1322         channel->napi_dev = efx->net_dev;
1323         netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll);
1324 }
1325
1326 void efx_init_napi(struct efx_nic *efx)
1327 {
1328         struct efx_channel *channel;
1329
1330         efx_for_each_channel(channel, efx)
1331                 efx_init_napi_channel(channel);
1332 }
1333
1334 void efx_fini_napi_channel(struct efx_channel *channel)
1335 {
1336         if (channel->napi_dev)
1337                 netif_napi_del(&channel->napi_str);
1338
1339         channel->napi_dev = NULL;
1340 }
1341
1342 void efx_fini_napi(struct efx_nic *efx)
1343 {
1344         struct efx_channel *channel;
1345
1346         efx_for_each_channel(channel, efx)
1347                 efx_fini_napi_channel(channel);
1348 }
1349
1350 /***************
1351  * Housekeeping
1352  ***************/
1353
1354 static int efx_channel_dummy_op_int(struct efx_channel *channel)
1355 {
1356         return 0;
1357 }
1358
1359 void efx_channel_dummy_op_void(struct efx_channel *channel)
1360 {
1361 }
1362
1363 static const struct efx_channel_type efx_default_channel_type = {
1364         .pre_probe              = efx_channel_dummy_op_int,
1365         .post_remove            = efx_channel_dummy_op_void,
1366         .get_name               = efx_get_channel_name,
1367         .copy                   = efx_copy_channel,
1368         .want_txqs              = efx_default_channel_want_txqs,
1369         .keep_eventq            = false,
1370         .want_pio               = true,
1371 };