drivers/net/ethernet/sfc/efx_channels.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2018 Solarflare Communications Inc.
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation, incorporated herein by reference.
   9  */
  10
  11 #include "net_driver.h"
  12 #include <linux/module.h>
  13 #include <linux/filter.h>
  14 #include "efx_channels.h"
  15 #include "efx.h"
  16 #include "efx_common.h"
  17 #include "tx_common.h"
  18 #include "rx_common.h"
  19 #include "nic.h"
  20 #include "sriov.h"
  21 #include "workarounds.h"
  22
  23 /* This is the first interrupt mode to try out of:
  24  * 0 => MSI-X
  25  * 1 => MSI
  26  * 2 => legacy
  27  */
  28 unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX;
  29
  30 /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS),
  31  * i.e. the number of CPUs among which we may distribute simultaneous
  32  * interrupt handling.
  33  *
  34  * Cards without MSI-X will only target one CPU via legacy or MSI interrupt.
  35  * The default (0) means to assign an interrupt to each core.
  36  */
  37 unsigned int rss_cpus;
  38
  39 static unsigned int irq_adapt_low_thresh = 8000;
  40 module_param(irq_adapt_low_thresh, uint, 0644);
  41 MODULE_PARM_DESC(irq_adapt_low_thresh,
  42                  "Threshold score for reducing IRQ moderation");
  43
  44 static unsigned int irq_adapt_high_thresh = 16000;
  45 module_param(irq_adapt_high_thresh, uint, 0644);
  46 MODULE_PARM_DESC(irq_adapt_high_thresh,
  47                  "Threshold score for increasing IRQ moderation");
  48
  49 static const struct efx_channel_type efx_default_channel_type;
  50
  51 /*************
  52  * INTERRUPTS
  53  *************/
  54
  55 static unsigned int count_online_cores(struct efx_nic *efx, bool local_node)
  56 {
  57         cpumask_var_t filter_mask;
  58         unsigned int count;
  59         int cpu;
  60
  61         if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) {
  62                 netif_warn(efx, probe, efx->net_dev,
  63                            "RSS disabled due to allocation failure\n");
  64                 return 1;
  65         }
  66
  67         cpumask_copy(filter_mask, cpu_online_mask);
  68         if (local_node)
  69                 cpumask_and(filter_mask, filter_mask,
  70                             cpumask_of_pcibus(efx->pci_dev->bus));
  71
  72         count = 0;
  73         for_each_cpu(cpu, filter_mask) {
  74                 ++count;
  75                 cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu));
  76         }
  77
  78         free_cpumask_var(filter_mask);
  79
  80         return count;
  81 }
  82
  83 static unsigned int efx_wanted_parallelism(struct efx_nic *efx)
  84 {
  85         unsigned int count;
  86
  87         if (rss_cpus) {
  88                 count = rss_cpus;
  89         } else {
  90                 count = count_online_cores(efx, true);
  91
  92                 /* If no online CPUs in local node, fallback to any online CPUs */
  93                 if (count == 0)
  94                         count = count_online_cores(efx, false);
  95         }
  96
  97         if (count > EFX_MAX_RX_QUEUES) {
  98                 netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn,
  99                                "Reducing number of rx queues from %u to %u.\n",
 100                                count, EFX_MAX_RX_QUEUES);
 101                 count = EFX_MAX_RX_QUEUES;
 102         }
 103
 104         /* If RSS is requested for the PF *and* VFs then we can't write RSS
 105          * table entries that are inaccessible to VFs
 106          */
 107 #ifdef CONFIG_SFC_SRIOV
 108         if (efx->type->sriov_wanted) {
 109                 if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 &&
 110                     count > efx_vf_size(efx)) {
 111                         netif_warn(efx, probe, efx->net_dev,
 112                                    "Reducing number of RSS channels from %u to %u for "
 113                                    "VF support. Increase vf-msix-limit to use more "
 114                                    "channels on the PF.\n",
 115                                    count, efx_vf_size(efx));
 116                         count = efx_vf_size(efx);
 117                 }
 118         }
 119 #endif
 120
 121         return count;
 122 }
 123
 124 static int efx_allocate_msix_channels(struct efx_nic *efx,
 125                                       unsigned int max_channels,
 126                                       unsigned int extra_channels,
 127                                       unsigned int parallelism)
 128 {
 129         unsigned int n_channels = parallelism;
 130         int vec_count;
 131         int tx_per_ev;
 132         int n_xdp_tx;
 133         int n_xdp_ev;
 134
 135         if (efx_separate_tx_channels)
 136                 n_channels *= 2;
 137         n_channels += extra_channels;
 138
 139         /* To allow XDP transmit to happen from arbitrary NAPI contexts
 140          * we allocate a TX queue per CPU. We share event queues across
 141          * multiple tx queues, assuming tx and ev queues are both
 142          * maximum size.
 143          */
 144         tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx);
 145         tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL);
 146         n_xdp_tx = num_possible_cpus();
 147         n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev);
 148
 149         vec_count = pci_msix_vec_count(efx->pci_dev);
 150         if (vec_count < 0)
 151                 return vec_count;
 152
 153         max_channels = min_t(unsigned int, vec_count, max_channels);
 154
 155         /* Check resources.
 156          * We need a channel per event queue, plus a VI per tx queue.
 157          * This may be more pessimistic than it needs to be.
 158          */
 159         if (n_channels >= max_channels) {
 160                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 161                 netif_warn(efx, drv, efx->net_dev,
 162                            "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
 163                            n_xdp_ev, n_channels, max_channels);
 164                 netif_warn(efx, drv, efx->net_dev,
 165                            "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
 166         } else if (n_channels + n_xdp_tx > efx->max_vis) {
 167                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 168                 netif_warn(efx, drv, efx->net_dev,
 169                            "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n",
 170                            n_xdp_tx, n_channels, efx->max_vis);
 171                 netif_warn(efx, drv, efx->net_dev,
 172                            "XDP_TX and XDP_REDIRECT might decrease device's performance\n");
 173         } else if (n_channels + n_xdp_ev > max_channels) {
 174                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED;
 175                 netif_warn(efx, drv, efx->net_dev,
 176                            "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n",
 177                            n_xdp_ev, n_channels, max_channels);
 178
 179                 n_xdp_ev = max_channels - n_channels;
 180                 netif_warn(efx, drv, efx->net_dev,
 181                            "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n",
 182                            DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev));
 183         } else {
 184                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED;
 185         }
 186
 187         if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) {
 188                 efx->n_xdp_channels = n_xdp_ev;
 189                 efx->xdp_tx_per_channel = tx_per_ev;
 190                 efx->xdp_tx_queue_count = n_xdp_tx;
 191                 n_channels += n_xdp_ev;
 192                 netif_dbg(efx, drv, efx->net_dev,
 193                           "Allocating %d TX and %d event queues for XDP\n",
 194                           n_xdp_ev * tx_per_ev, n_xdp_ev);
 195         } else {
 196                 efx->n_xdp_channels = 0;
 197                 efx->xdp_tx_per_channel = 0;
 198                 efx->xdp_tx_queue_count = n_xdp_tx;
 199         }
 200
 201         if (vec_count < n_channels) {
 202                 netif_err(efx, drv, efx->net_dev,
 203                           "WARNING: Insufficient MSI-X vectors available (%d < %u).\n",
 204                           vec_count, n_channels);
 205                 netif_err(efx, drv, efx->net_dev,
 206                           "WARNING: Performance may be reduced.\n");
 207                 n_channels = vec_count;
 208         }
 209
 210         n_channels = min(n_channels, max_channels);
 211
 212         efx->n_channels = n_channels;
 213
 214         /* Ignore XDP tx channels when creating rx channels. */
 215         n_channels -= efx->n_xdp_channels;
 216
 217         if (efx_separate_tx_channels) {
 218                 efx->n_tx_channels =
 219                         min(max(n_channels / 2, 1U),
 220                             efx->max_tx_channels);
 221                 efx->tx_channel_offset =
 222                         n_channels - efx->n_tx_channels;
 223                 efx->n_rx_channels =
 224                         max(n_channels -
 225                             efx->n_tx_channels, 1U);
 226         } else {
 227                 efx->n_tx_channels = min(n_channels, efx->max_tx_channels);
 228                 efx->tx_channel_offset = 0;
 229                 efx->n_rx_channels = n_channels;
 230         }
 231
 232         efx->n_rx_channels = min(efx->n_rx_channels, parallelism);
 233         efx->n_tx_channels = min(efx->n_tx_channels, parallelism);
 234
 235         efx->xdp_channel_offset = n_channels;
 236
 237         netif_dbg(efx, drv, efx->net_dev,
 238                   "Allocating %u RX channels\n",
 239                   efx->n_rx_channels);
 240
 241         return efx->n_channels;
 242 }
 243
 244 /* Probe the number and type of interrupts we are able to obtain, and
 245  * the resulting numbers of channels and RX queues.
 246  */
 247 int efx_probe_interrupts(struct efx_nic *efx)
 248 {
 249         unsigned int extra_channels = 0;
 250         unsigned int rss_spread;
 251         unsigned int i, j;
 252         int rc;
 253
 254         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++)
 255                 if (efx->extra_channel_type[i])
 256                         ++extra_channels;
 257
 258         if (efx->interrupt_mode == EFX_INT_MODE_MSIX) {
 259                 unsigned int parallelism = efx_wanted_parallelism(efx);
 260                 struct msix_entry xentries[EFX_MAX_CHANNELS];
 261                 unsigned int n_channels;
 262
 263                 rc = efx_allocate_msix_channels(efx, efx->max_channels,
 264                                                 extra_channels, parallelism);
 265                 if (rc >= 0) {
 266                         n_channels = rc;
 267                         for (i = 0; i < n_channels; i++)
 268                                 xentries[i].entry = i;
 269                         rc = pci_enable_msix_range(efx->pci_dev, xentries, 1,
 270                                                    n_channels);
 271                 }
 272                 if (rc < 0) {
 273                         /* Fall back to single channel MSI */
 274                         netif_err(efx, drv, efx->net_dev,
 275                                   "could not enable MSI-X\n");
 276                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI)
 277                                 efx->interrupt_mode = EFX_INT_MODE_MSI;
 278                         else
 279                                 return rc;
 280                 } else if (rc < n_channels) {
 281                         netif_err(efx, drv, efx->net_dev,
 282                                   "WARNING: Insufficient MSI-X vectors"
 283                                   " available (%d < %u).\n", rc, n_channels);
 284                         netif_err(efx, drv, efx->net_dev,
 285                                   "WARNING: Performance may be reduced.\n");
 286                         n_channels = rc;
 287                 }
 288
 289                 if (rc > 0) {
 290                         for (i = 0; i < efx->n_channels; i++)
 291                                 efx_get_channel(efx, i)->irq =
 292                                         xentries[i].vector;
 293                 }
 294         }
 295
 296         /* Try single interrupt MSI */
 297         if (efx->interrupt_mode == EFX_INT_MODE_MSI) {
 298                 efx->n_channels = 1;
 299                 efx->n_rx_channels = 1;
 300                 efx->n_tx_channels = 1;
 301                 efx->tx_channel_offset = 0;
 302                 efx->n_xdp_channels = 0;
 303                 efx->xdp_channel_offset = efx->n_channels;
 304                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 305                 rc = pci_enable_msi(efx->pci_dev);
 306                 if (rc == 0) {
 307                         efx_get_channel(efx, 0)->irq = efx->pci_dev->irq;
 308                 } else {
 309                         netif_err(efx, drv, efx->net_dev,
 310                                   "could not enable MSI\n");
 311                         if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY)
 312                                 efx->interrupt_mode = EFX_INT_MODE_LEGACY;
 313                         else
 314                                 return rc;
 315                 }
 316         }
 317
 318         /* Assume legacy interrupts */
 319         if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) {
 320                 efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0);
 321                 efx->n_rx_channels = 1;
 322                 efx->n_tx_channels = 1;
 323                 efx->tx_channel_offset = efx_separate_tx_channels ? 1 : 0;
 324                 efx->n_xdp_channels = 0;
 325                 efx->xdp_channel_offset = efx->n_channels;
 326                 efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED;
 327                 efx->legacy_irq = efx->pci_dev->irq;
 328         }
 329
 330         /* Assign extra channels if possible, before XDP channels */
 331         efx->n_extra_tx_channels = 0;
 332         j = efx->xdp_channel_offset;
 333         for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) {
 334                 if (!efx->extra_channel_type[i])
 335                         continue;
 336                 if (j <= efx->tx_channel_offset + efx->n_tx_channels) {
 337                         efx->extra_channel_type[i]->handle_no_channel(efx);
 338                 } else {
 339                         --j;
 340                         efx_get_channel(efx, j)->type =
 341                                 efx->extra_channel_type[i];
 342                         if (efx_channel_has_tx_queues(efx_get_channel(efx, j)))
 343                                 efx->n_extra_tx_channels++;
 344                 }
 345         }
 346
 347         rss_spread = efx->n_rx_channels;
 348         /* RSS might be usable on VFs even if it is disabled on the PF */
 349 #ifdef CONFIG_SFC_SRIOV
 350         if (efx->type->sriov_wanted) {
 351                 efx->rss_spread = ((rss_spread > 1 ||
 352                                     !efx->type->sriov_wanted(efx)) ?
 353                                    rss_spread : efx_vf_size(efx));
 354                 return 0;
 355         }
 356 #endif
 357         efx->rss_spread = rss_spread;
 358
 359         return 0;
 360 }
 361
 362 #if defined(CONFIG_SMP)
 363 void efx_set_interrupt_affinity(struct efx_nic *efx)
 364 {
 365         const struct cpumask *numa_mask = cpumask_of_pcibus(efx->pci_dev->bus);
 366         struct efx_channel *channel;
 367         unsigned int cpu;
 368
 369         /* If no online CPUs in local node, fallback to any online CPU */
 370         if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids)
 371                 numa_mask = cpu_online_mask;
 372
 373         cpu = -1;
 374         efx_for_each_channel(channel, efx) {
 375                 cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask);
 376                 if (cpu >= nr_cpu_ids)
 377                         cpu = cpumask_first_and(cpu_online_mask, numa_mask);
 378                 irq_set_affinity_hint(channel->irq, cpumask_of(cpu));
 379         }
 380 }
 381
 382 void efx_clear_interrupt_affinity(struct efx_nic *efx)
 383 {
 384         struct efx_channel *channel;
 385
 386         efx_for_each_channel(channel, efx)
 387                 irq_set_affinity_hint(channel->irq, NULL);
 388 }
 389 #else
 390 void
 391 efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 392 {
 393 }
 394
 395 void
 396 efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused)))
 397 {
 398 }
 399 #endif /* CONFIG_SMP */
 400
 401 void efx_remove_interrupts(struct efx_nic *efx)
 402 {
 403         struct efx_channel *channel;
 404
 405         /* Remove MSI/MSI-X interrupts */
 406         efx_for_each_channel(channel, efx)
 407                 channel->irq = 0;
 408         pci_disable_msi(efx->pci_dev);
 409         pci_disable_msix(efx->pci_dev);
 410
 411         /* Remove legacy interrupt */
 412         efx->legacy_irq = 0;
 413 }
 414
 415 /***************
 416  * EVENT QUEUES
 417  ***************/
 418
 419 /* Create event queue
 420  * Event queue memory allocations are done only once.  If the channel
 421  * is reset, the memory buffer will be reused; this guards against
 422  * errors during channel reset and also simplifies interrupt handling.
 423  */
 424 int efx_probe_eventq(struct efx_channel *channel)
 425 {
 426         struct efx_nic *efx = channel->efx;
 427         unsigned long entries;
 428
 429         netif_dbg(efx, probe, efx->net_dev,
 430                   "chan %d create event queue\n", channel->channel);
 431
 432         /* Build an event queue with room for one event per tx and rx buffer,
 433          * plus some extra for link state events and MCDI completions.
 434          */
 435         entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128);
 436         EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE);
 437         channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1;
 438
 439         return efx_nic_probe_eventq(channel);
 440 }
 441
 442 /* Prepare channel's event queue */
 443 int efx_init_eventq(struct efx_channel *channel)
 444 {
 445         struct efx_nic *efx = channel->efx;
 446         int rc;
 447
 448         EFX_WARN_ON_PARANOID(channel->eventq_init);
 449
 450         netif_dbg(efx, drv, efx->net_dev,
 451                   "chan %d init event queue\n", channel->channel);
 452
 453         rc = efx_nic_init_eventq(channel);
 454         if (rc == 0) {
 455                 efx->type->push_irq_moderation(channel);
 456                 channel->eventq_read_ptr = 0;
 457                 channel->eventq_init = true;
 458         }
 459         return rc;
 460 }
 461
 462 /* Enable event queue processing and NAPI */
 463 void efx_start_eventq(struct efx_channel *channel)
 464 {
 465         netif_dbg(channel->efx, ifup, channel->efx->net_dev,
 466                   "chan %d start event queue\n", channel->channel);
 467
 468         /* Make sure the NAPI handler sees the enabled flag set */
 469         channel->enabled = true;
 470         smp_wmb();
 471
 472         napi_enable(&channel->napi_str);
 473         efx_nic_eventq_read_ack(channel);
 474 }
 475
 476 /* Disable event queue processing and NAPI */
 477 void efx_stop_eventq(struct efx_channel *channel)
 478 {
 479         if (!channel->enabled)
 480                 return;
 481
 482         napi_disable(&channel->napi_str);
 483         channel->enabled = false;
 484 }
 485
 486 void efx_fini_eventq(struct efx_channel *channel)
 487 {
 488         if (!channel->eventq_init)
 489                 return;
 490
 491         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 492                   "chan %d fini event queue\n", channel->channel);
 493
 494         efx_nic_fini_eventq(channel);
 495         channel->eventq_init = false;
 496 }
 497
 498 void efx_remove_eventq(struct efx_channel *channel)
 499 {
 500         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 501                   "chan %d remove event queue\n", channel->channel);
 502
 503         efx_nic_remove_eventq(channel);
 504 }
 505
 506 /**************************************************************************
 507  *
 508  * Channel handling
 509  *
 510  *************************************************************************/
 511
 512 #ifdef CONFIG_RFS_ACCEL
 513 static void efx_filter_rfs_expire(struct work_struct *data)
 514 {
 515         struct delayed_work *dwork = to_delayed_work(data);
 516         struct efx_channel *channel;
 517         unsigned int time, quota;
 518
 519         channel = container_of(dwork, struct efx_channel, filter_work);
 520         time = jiffies - channel->rfs_last_expiry;
 521         quota = channel->rfs_filter_count * time / (30 * HZ);
 522         if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota)))
 523                 channel->rfs_last_expiry += time;
 524         /* Ensure we do more work eventually even if NAPI poll is not happening */
 525         schedule_delayed_work(dwork, 30 * HZ);
 526 }
 527 #endif
 528
 529 /* Allocate and initialise a channel structure. */
 530 static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i)
 531 {
 532         struct efx_rx_queue *rx_queue;
 533         struct efx_tx_queue *tx_queue;
 534         struct efx_channel *channel;
 535         int j;
 536
 537         channel = kzalloc(sizeof(*channel), GFP_KERNEL);
 538         if (!channel)
 539                 return NULL;
 540
 541         channel->efx = efx;
 542         channel->channel = i;
 543         channel->type = &efx_default_channel_type;
 544
 545         for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
 546                 tx_queue = &channel->tx_queue[j];
 547                 tx_queue->efx = efx;
 548                 tx_queue->queue = -1;
 549                 tx_queue->label = j;
 550                 tx_queue->channel = channel;
 551         }
 552
 553 #ifdef CONFIG_RFS_ACCEL
 554         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 555 #endif
 556
 557         rx_queue = &channel->rx_queue;
 558         rx_queue->efx = efx;
 559         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 560
 561         return channel;
 562 }
 563
 564 int efx_init_channels(struct efx_nic *efx)
 565 {
 566         unsigned int i;
 567
 568         for (i = 0; i < EFX_MAX_CHANNELS; i++) {
 569                 efx->channel[i] = efx_alloc_channel(efx, i);
 570                 if (!efx->channel[i])
 571                         return -ENOMEM;
 572                 efx->msi_context[i].efx = efx;
 573                 efx->msi_context[i].index = i;
 574         }
 575
 576         /* Higher numbered interrupt modes are less capable! */
 577         efx->interrupt_mode = min(efx->type->min_interrupt_mode,
 578                                   efx_interrupt_mode);
 579
 580         efx->max_channels = EFX_MAX_CHANNELS;
 581         efx->max_tx_channels = EFX_MAX_CHANNELS;
 582
 583         return 0;
 584 }
 585
 586 void efx_fini_channels(struct efx_nic *efx)
 587 {
 588         unsigned int i;
 589
 590         for (i = 0; i < EFX_MAX_CHANNELS; i++)
 591                 if (efx->channel[i]) {
 592                         kfree(efx->channel[i]);
 593                         efx->channel[i] = NULL;
 594                 }
 595 }
 596
 597 /* Allocate and initialise a channel structure, copying parameters
 598  * (but not resources) from an old channel structure.
 599  */
 600 struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel)
 601 {
 602         struct efx_rx_queue *rx_queue;
 603         struct efx_tx_queue *tx_queue;
 604         struct efx_channel *channel;
 605         int j;
 606
 607         channel = kmalloc(sizeof(*channel), GFP_KERNEL);
 608         if (!channel)
 609                 return NULL;
 610
 611         *channel = *old_channel;
 612
 613         channel->napi_dev = NULL;
 614         INIT_HLIST_NODE(&channel->napi_str.napi_hash_node);
 615         channel->napi_str.napi_id = 0;
 616         channel->napi_str.state = 0;
 617         memset(&channel->eventq, 0, sizeof(channel->eventq));
 618
 619         for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) {
 620                 tx_queue = &channel->tx_queue[j];
 621                 if (tx_queue->channel)
 622                         tx_queue->channel = channel;
 623                 tx_queue->buffer = NULL;
 624                 tx_queue->cb_page = NULL;
 625                 memset(&tx_queue->txd, 0, sizeof(tx_queue->txd));
 626         }
 627
 628         rx_queue = &channel->rx_queue;
 629         rx_queue->buffer = NULL;
 630         memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd));
 631         timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0);
 632 #ifdef CONFIG_RFS_ACCEL
 633         INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire);
 634 #endif
 635
 636         return channel;
 637 }
 638
 639 static int efx_probe_channel(struct efx_channel *channel)
 640 {
 641         struct efx_tx_queue *tx_queue;
 642         struct efx_rx_queue *rx_queue;
 643         int rc;
 644
 645         netif_dbg(channel->efx, probe, channel->efx->net_dev,
 646                   "creating channel %d\n", channel->channel);
 647
 648         rc = channel->type->pre_probe(channel);
 649         if (rc)
 650                 goto fail;
 651
 652         rc = efx_probe_eventq(channel);
 653         if (rc)
 654                 goto fail;
 655
 656         efx_for_each_channel_tx_queue(tx_queue, channel) {
 657                 rc = efx_probe_tx_queue(tx_queue);
 658                 if (rc)
 659                         goto fail;
 660         }
 661
 662         efx_for_each_channel_rx_queue(rx_queue, channel) {
 663                 rc = efx_probe_rx_queue(rx_queue);
 664                 if (rc)
 665                         goto fail;
 666         }
 667
 668         channel->rx_list = NULL;
 669
 670         return 0;
 671
 672 fail:
 673         efx_remove_channel(channel);
 674         return rc;
 675 }
 676
 677 static void efx_get_channel_name(struct efx_channel *channel, char *buf,
 678                                  size_t len)
 679 {
 680         struct efx_nic *efx = channel->efx;
 681         const char *type;
 682         int number;
 683
 684         number = channel->channel;
 685
 686         if (number >= efx->xdp_channel_offset &&
 687             !WARN_ON_ONCE(!efx->n_xdp_channels)) {
 688                 type = "-xdp";
 689                 number -= efx->xdp_channel_offset;
 690         } else if (efx->tx_channel_offset == 0) {
 691                 type = "";
 692         } else if (number < efx->tx_channel_offset) {
 693                 type = "-rx";
 694         } else {
 695                 type = "-tx";
 696                 number -= efx->tx_channel_offset;
 697         }
 698         snprintf(buf, len, "%s%s-%d", efx->name, type, number);
 699 }
 700
 701 void efx_set_channel_names(struct efx_nic *efx)
 702 {
 703         struct efx_channel *channel;
 704
 705         efx_for_each_channel(channel, efx)
 706                 channel->type->get_name(channel,
 707                                         efx->msi_context[channel->channel].name,
 708                                         sizeof(efx->msi_context[0].name));
 709 }
 710
 711 int efx_probe_channels(struct efx_nic *efx)
 712 {
 713         struct efx_channel *channel;
 714         int rc;
 715
 716         /* Restart special buffer allocation */
 717         efx->next_buffer_table = 0;
 718
 719         /* Probe channels in reverse, so that any 'extra' channels
 720          * use the start of the buffer table. This allows the traffic
 721          * channels to be resized without moving them or wasting the
 722          * entries before them.
 723          */
 724         efx_for_each_channel_rev(channel, efx) {
 725                 rc = efx_probe_channel(channel);
 726                 if (rc) {
 727                         netif_err(efx, probe, efx->net_dev,
 728                                   "failed to create channel %d\n",
 729                                   channel->channel);
 730                         goto fail;
 731                 }
 732         }
 733         efx_set_channel_names(efx);
 734
 735         return 0;
 736
 737 fail:
 738         efx_remove_channels(efx);
 739         return rc;
 740 }
 741
 742 void efx_remove_channel(struct efx_channel *channel)
 743 {
 744         struct efx_tx_queue *tx_queue;
 745         struct efx_rx_queue *rx_queue;
 746
 747         netif_dbg(channel->efx, drv, channel->efx->net_dev,
 748                   "destroy chan %d\n", channel->channel);
 749
 750         efx_for_each_channel_rx_queue(rx_queue, channel)
 751                 efx_remove_rx_queue(rx_queue);
 752         efx_for_each_channel_tx_queue(tx_queue, channel)
 753                 efx_remove_tx_queue(tx_queue);
 754         efx_remove_eventq(channel);
 755         channel->type->post_remove(channel);
 756 }
 757
 758 void efx_remove_channels(struct efx_nic *efx)
 759 {
 760         struct efx_channel *channel;
 761
 762         efx_for_each_channel(channel, efx)
 763                 efx_remove_channel(channel);
 764
 765         kfree(efx->xdp_tx_queues);
 766 }
 767
 768 static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number,
 769                                 struct efx_tx_queue *tx_queue)
 770 {
 771         if (xdp_queue_number >= efx->xdp_tx_queue_count)
 772                 return -EINVAL;
 773
 774         netif_dbg(efx, drv, efx->net_dev,
 775                   "Channel %u TXQ %u is XDP %u, HW %u\n",
 776                   tx_queue->channel->channel, tx_queue->label,
 777                   xdp_queue_number, tx_queue->queue);
 778         efx->xdp_tx_queues[xdp_queue_number] = tx_queue;
 779         return 0;
 780 }
 781
 782 static void efx_set_xdp_channels(struct efx_nic *efx)
 783 {
 784         struct efx_tx_queue *tx_queue;
 785         struct efx_channel *channel;
 786         unsigned int next_queue = 0;
 787         int xdp_queue_number = 0;
 788         int rc;
 789
 790         /* We need to mark which channels really have RX and TX
 791          * queues, and adjust the TX queue numbers if we have separate
 792          * RX-only and TX-only channels.
 793          */
 794         efx_for_each_channel(channel, efx) {
 795                 if (channel->channel < efx->tx_channel_offset)
 796                         continue;
 797
 798                 if (efx_channel_is_xdp_tx(channel)) {
 799                         efx_for_each_channel_tx_queue(tx_queue, channel) {
 800                                 tx_queue->queue = next_queue++;
 801                                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
 802                                                           tx_queue);
 803                                 if (rc == 0)
 804                                         xdp_queue_number++;
 805                         }
 806                 } else {
 807                         efx_for_each_channel_tx_queue(tx_queue, channel) {
 808                                 tx_queue->queue = next_queue++;
 809                                 netif_dbg(efx, drv, efx->net_dev,
 810                                           "Channel %u TXQ %u is HW %u\n",
 811                                           channel->channel, tx_queue->label,
 812                                           tx_queue->queue);
 813                         }
 814
 815                         /* If XDP is borrowing queues from net stack, it must
 816                          * use the queue with no csum offload, which is the
 817                          * first one of the channel
 818                          * (note: tx_queue_by_type is not initialized yet)
 819                          */
 820                         if (efx->xdp_txq_queues_mode ==
 821                             EFX_XDP_TX_QUEUES_BORROWED) {
 822                                 tx_queue = &channel->tx_queue[0];
 823                                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number,
 824                                                           tx_queue);
 825                                 if (rc == 0)
 826                                         xdp_queue_number++;
 827                         }
 828                 }
 829         }
 830         WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED &&
 831                 xdp_queue_number != efx->xdp_tx_queue_count);
 832         WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED &&
 833                 xdp_queue_number > efx->xdp_tx_queue_count);
 834
 835         /* If we have more CPUs than assigned XDP TX queues, assign the already
 836          * existing queues to the exceeding CPUs
 837          */
 838         next_queue = 0;
 839         while (xdp_queue_number < efx->xdp_tx_queue_count) {
 840                 tx_queue = efx->xdp_tx_queues[next_queue++];
 841                 rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue);
 842                 if (rc == 0)
 843                         xdp_queue_number++;
 844         }
 845 }
 846
 847 int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries)
 848 {
 849         struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel,
 850                            *ptp_channel = efx_ptp_channel(efx);
 851         struct efx_ptp_data *ptp_data = efx->ptp_data;
 852         unsigned int i, next_buffer_table = 0;
 853         u32 old_rxq_entries, old_txq_entries;
 854         int rc, rc2;
 855
 856         rc = efx_check_disabled(efx);
 857         if (rc)
 858                 return rc;
 859
 860         /* Not all channels should be reallocated. We must avoid
 861          * reallocating their buffer table entries.
 862          */
 863         efx_for_each_channel(channel, efx) {
 864                 struct efx_rx_queue *rx_queue;
 865                 struct efx_tx_queue *tx_queue;
 866
 867                 if (channel->type->copy)
 868                         continue;
 869                 next_buffer_table = max(next_buffer_table,
 870                                         channel->eventq.index +
 871                                         channel->eventq.entries);
 872                 efx_for_each_channel_rx_queue(rx_queue, channel)
 873                         next_buffer_table = max(next_buffer_table,
 874                                                 rx_queue->rxd.index +
 875                                                 rx_queue->rxd.entries);
 876                 efx_for_each_channel_tx_queue(tx_queue, channel)
 877                         next_buffer_table = max(next_buffer_table,
 878                                                 tx_queue->txd.index +
 879                                                 tx_queue->txd.entries);
 880         }
 881
 882         efx_device_detach_sync(efx);
 883         efx_stop_all(efx);
 884         efx_soft_disable_interrupts(efx);
 885
 886         /* Clone channels (where possible) */
 887         memset(other_channel, 0, sizeof(other_channel));
 888         for (i = 0; i < efx->n_channels; i++) {
 889                 channel = efx->channel[i];
 890                 if (channel->type->copy)
 891                         channel = channel->type->copy(channel);
 892                 if (!channel) {
 893                         rc = -ENOMEM;
 894                         goto out;
 895                 }
 896                 other_channel[i] = channel;
 897         }
 898
 899         /* Swap entry counts and channel pointers */
 900         old_rxq_entries = efx->rxq_entries;
 901         old_txq_entries = efx->txq_entries;
 902         efx->rxq_entries = rxq_entries;
 903         efx->txq_entries = txq_entries;
 904         for (i = 0; i < efx->n_channels; i++)
 905                 swap(efx->channel[i], other_channel[i]);
 906
 907         /* Restart buffer table allocation */
 908         efx->next_buffer_table = next_buffer_table;
 909
 910         for (i = 0; i < efx->n_channels; i++) {
 911                 channel = efx->channel[i];
 912                 if (!channel->type->copy)
 913                         continue;
 914                 rc = efx_probe_channel(channel);
 915                 if (rc)
 916                         goto rollback;
 917                 efx_init_napi_channel(efx->channel[i]);
 918         }
 919
 920         efx_set_xdp_channels(efx);
 921 out:
 922         efx->ptp_data = NULL;
 923         /* Destroy unused channel structures */
 924         for (i = 0; i < efx->n_channels; i++) {
 925                 channel = other_channel[i];
 926                 if (channel && channel->type->copy) {
 927                         efx_fini_napi_channel(channel);
 928                         efx_remove_channel(channel);
 929                         kfree(channel);
 930                 }
 931         }
 932
 933         efx->ptp_data = ptp_data;
 934         rc2 = efx_soft_enable_interrupts(efx);
 935         if (rc2) {
 936                 rc = rc ? rc : rc2;
 937                 netif_err(efx, drv, efx->net_dev,
 938                           "unable to restart interrupts on channel reallocation\n");
 939                 efx_schedule_reset(efx, RESET_TYPE_DISABLE);
 940         } else {
 941                 efx_start_all(efx);
 942                 efx_device_attach_if_not_resetting(efx);
 943         }
 944         return rc;
 945
 946 rollback:
 947         /* Swap back */
 948         efx->rxq_entries = old_rxq_entries;
 949         efx->txq_entries = old_txq_entries;
 950         for (i = 0; i < efx->n_channels; i++)
 951                 swap(efx->channel[i], other_channel[i]);
 952         efx_ptp_update_channel(efx, ptp_channel);
 953         goto out;
 954 }
 955
 956 int efx_set_channels(struct efx_nic *efx)
 957 {
 958         struct efx_channel *channel;
 959         int rc;
 960
 961         if (efx->xdp_tx_queue_count) {
 962                 EFX_WARN_ON_PARANOID(efx->xdp_tx_queues);
 963
 964                 /* Allocate array for XDP TX queue lookup. */
 965                 efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count,
 966                                              sizeof(*efx->xdp_tx_queues),
 967                                              GFP_KERNEL);
 968                 if (!efx->xdp_tx_queues)
 969                         return -ENOMEM;
 970         }
 971
 972         efx_for_each_channel(channel, efx) {
 973                 if (channel->channel < efx->n_rx_channels)
 974                         channel->rx_queue.core_index = channel->channel;
 975                 else
 976                         channel->rx_queue.core_index = -1;
 977         }
 978
 979         efx_set_xdp_channels(efx);
 980
 981         rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels);
 982         if (rc)
 983                 return rc;
 984         return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels);
 985 }
 986
 987 static bool efx_default_channel_want_txqs(struct efx_channel *channel)
 988 {
 989         return channel->channel - channel->efx->tx_channel_offset <
 990                 channel->efx->n_tx_channels;
 991 }
 992
 993 /*************
 994  * START/STOP
 995  *************/
 996
 997 int efx_soft_enable_interrupts(struct efx_nic *efx)
 998 {
 999         struct efx_channel *channel, *end_channel;
1000         int rc;
1001
1002         BUG_ON(efx->state == STATE_DISABLED);
1003
1004         efx->irq_soft_enabled = true;
1005         smp_wmb();
1006
1007         efx_for_each_channel(channel, efx) {
1008                 if (!channel->type->keep_eventq) {
1009                         rc = efx_init_eventq(channel);
1010                         if (rc)
1011                                 goto fail;
1012                 }
1013                 efx_start_eventq(channel);
1014         }
1015
1016         efx_mcdi_mode_event(efx);
1017
1018         return 0;
1019 fail:
1020         end_channel = channel;
1021         efx_for_each_channel(channel, efx) {
1022                 if (channel == end_channel)
1023                         break;
1024                 efx_stop_eventq(channel);
1025                 if (!channel->type->keep_eventq)
1026                         efx_fini_eventq(channel);
1027         }
1028
1029         return rc;
1030 }
1031
1032 void efx_soft_disable_interrupts(struct efx_nic *efx)
1033 {
1034         struct efx_channel *channel;
1035
1036         if (efx->state == STATE_DISABLED)
1037                 return;
1038
1039         efx_mcdi_mode_poll(efx);
1040
1041         efx->irq_soft_enabled = false;
1042         smp_wmb();
1043
1044         if (efx->legacy_irq)
1045                 synchronize_irq(efx->legacy_irq);
1046
1047         efx_for_each_channel(channel, efx) {
1048                 if (channel->irq)
1049                         synchronize_irq(channel->irq);
1050
1051                 efx_stop_eventq(channel);
1052                 if (!channel->type->keep_eventq)
1053                         efx_fini_eventq(channel);
1054         }
1055
1056         /* Flush the asynchronous MCDI request queue */
1057         efx_mcdi_flush_async(efx);
1058 }
1059
1060 int efx_enable_interrupts(struct efx_nic *efx)
1061 {
1062         struct efx_channel *channel, *end_channel;
1063         int rc;
1064
1065         /* TODO: Is this really a bug? */
1066         BUG_ON(efx->state == STATE_DISABLED);
1067
1068         if (efx->eeh_disabled_legacy_irq) {
1069                 enable_irq(efx->legacy_irq);
1070                 efx->eeh_disabled_legacy_irq = false;
1071         }
1072
1073         efx->type->irq_enable_master(efx);
1074
1075         efx_for_each_channel(channel, efx) {
1076                 if (channel->type->keep_eventq) {
1077                         rc = efx_init_eventq(channel);
1078                         if (rc)
1079                                 goto fail;
1080                 }
1081         }
1082
1083         rc = efx_soft_enable_interrupts(efx);
1084         if (rc)
1085                 goto fail;
1086
1087         return 0;
1088
1089 fail:
1090         end_channel = channel;
1091         efx_for_each_channel(channel, efx) {
1092                 if (channel == end_channel)
1093                         break;
1094                 if (channel->type->keep_eventq)
1095                         efx_fini_eventq(channel);
1096         }
1097
1098         efx->type->irq_disable_non_ev(efx);
1099
1100         return rc;
1101 }
1102
1103 void efx_disable_interrupts(struct efx_nic *efx)
1104 {
1105         struct efx_channel *channel;
1106
1107         efx_soft_disable_interrupts(efx);
1108
1109         efx_for_each_channel(channel, efx) {
1110                 if (channel->type->keep_eventq)
1111                         efx_fini_eventq(channel);
1112         }
1113
1114         efx->type->irq_disable_non_ev(efx);
1115 }
1116
1117 void efx_start_channels(struct efx_nic *efx)
1118 {
1119         struct efx_tx_queue *tx_queue;
1120         struct efx_rx_queue *rx_queue;
1121         struct efx_channel *channel;
1122
1123         efx_for_each_channel_rev(channel, efx) {
1124                 if (channel->type->start)
1125                         channel->type->start(channel);
1126                 efx_for_each_channel_tx_queue(tx_queue, channel) {
1127                         efx_init_tx_queue(tx_queue);
1128                         atomic_inc(&efx->active_queues);
1129                 }
1130
1131                 efx_for_each_channel_rx_queue(rx_queue, channel) {
1132                         efx_init_rx_queue(rx_queue);
1133                         atomic_inc(&efx->active_queues);
1134                         efx_stop_eventq(channel);
1135                         efx_fast_push_rx_descriptors(rx_queue, false);
1136                         efx_start_eventq(channel);
1137                 }
1138
1139                 WARN_ON(channel->rx_pkt_n_frags);
1140         }
1141 }
1142
1143 void efx_stop_channels(struct efx_nic *efx)
1144 {
1145         struct efx_tx_queue *tx_queue;
1146         struct efx_rx_queue *rx_queue;
1147         struct efx_channel *channel;
1148         int rc = 0;
1149
1150         /* Stop special channels and RX refill.
1151          * The channel's stop has to be called first, since it might wait
1152          * for a sentinel RX to indicate the channel has fully drained.
1153          */
1154         efx_for_each_channel(channel, efx) {
1155                 if (channel->type->stop)
1156                         channel->type->stop(channel);
1157                 efx_for_each_channel_rx_queue(rx_queue, channel)
1158                         rx_queue->refill_enabled = false;
1159         }
1160
1161         efx_for_each_channel(channel, efx) {
1162                 /* RX packet processing is pipelined, so wait for the
1163                  * NAPI handler to complete.  At least event queue 0
1164                  * might be kept active by non-data events, so don't
1165                  * use napi_synchronize() but actually disable NAPI
1166                  * temporarily.
1167                  */
1168                 if (efx_channel_has_rx_queue(channel)) {
1169                         efx_stop_eventq(channel);
1170                         efx_start_eventq(channel);
1171                 }
1172         }
1173
1174         if (efx->type->fini_dmaq)
1175                 rc = efx->type->fini_dmaq(efx);
1176
1177         if (rc) {
1178                 netif_err(efx, drv, efx->net_dev, "failed to flush queues\n");
1179         } else {
1180                 netif_dbg(efx, drv, efx->net_dev,
1181                           "successfully flushed all queues\n");
1182         }
1183
1184         efx_for_each_channel(channel, efx) {
1185                 efx_for_each_channel_rx_queue(rx_queue, channel)
1186                         efx_fini_rx_queue(rx_queue);
1187                 efx_for_each_channel_tx_queue(tx_queue, channel)
1188                         efx_fini_tx_queue(tx_queue);
1189         }
1190 }
1191
1192 /**************************************************************************
1193  *
1194  * NAPI interface
1195  *
1196  *************************************************************************/
1197
1198 /* Process channel's event queue
1199  *
1200  * This function is responsible for processing the event queue of a
1201  * single channel.  The caller must guarantee that this function will
1202  * never be concurrently called more than once on the same channel,
1203  * though different channels may be being processed concurrently.
1204  */
1205 static int efx_process_channel(struct efx_channel *channel, int budget)
1206 {
1207         struct efx_tx_queue *tx_queue;
1208         struct list_head rx_list;
1209         int spent;
1210
1211         if (unlikely(!channel->enabled))
1212                 return 0;
1213
1214         /* Prepare the batch receive list */
1215         EFX_WARN_ON_PARANOID(channel->rx_list != NULL);
1216         INIT_LIST_HEAD(&rx_list);
1217         channel->rx_list = &rx_list;
1218
1219         efx_for_each_channel_tx_queue(tx_queue, channel) {
1220                 tx_queue->pkts_compl = 0;
1221                 tx_queue->bytes_compl = 0;
1222         }
1223
1224         spent = efx_nic_process_eventq(channel, budget);
1225         if (spent && efx_channel_has_rx_queue(channel)) {
1226                 struct efx_rx_queue *rx_queue =
1227                         efx_channel_get_rx_queue(channel);
1228
1229                 efx_rx_flush_packet(channel);
1230                 efx_fast_push_rx_descriptors(rx_queue, true);
1231         }
1232
1233         /* Update BQL */
1234         efx_for_each_channel_tx_queue(tx_queue, channel) {
1235                 if (tx_queue->bytes_compl) {
1236                         netdev_tx_completed_queue(tx_queue->core_txq,
1237                                                   tx_queue->pkts_compl,
1238                                                   tx_queue->bytes_compl);
1239                 }
1240         }
1241
1242         /* Receive any packets we queued up */
1243         netif_receive_skb_list(channel->rx_list);
1244         channel->rx_list = NULL;
1245
1246         return spent;
1247 }
1248
1249 static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel)
1250 {
1251         int step = efx->irq_mod_step_us;
1252
1253         if (channel->irq_mod_score < irq_adapt_low_thresh) {
1254                 if (channel->irq_moderation_us > step) {
1255                         channel->irq_moderation_us -= step;
1256                         efx->type->push_irq_moderation(channel);
1257                 }
1258         } else if (channel->irq_mod_score > irq_adapt_high_thresh) {
1259                 if (channel->irq_moderation_us <
1260                     efx->irq_rx_moderation_us) {
1261                         channel->irq_moderation_us += step;
1262                         efx->type->push_irq_moderation(channel);
1263                 }
1264         }
1265
1266         channel->irq_count = 0;
1267         channel->irq_mod_score = 0;
1268 }
1269
1270 /* NAPI poll handler
1271  *
1272  * NAPI guarantees serialisation of polls of the same device, which
1273  * provides the guarantee required by efx_process_channel().
1274  */
1275 static int efx_poll(struct napi_struct *napi, int budget)
1276 {
1277         struct efx_channel *channel =
1278                 container_of(napi, struct efx_channel, napi_str);
1279         struct efx_nic *efx = channel->efx;
1280 #ifdef CONFIG_RFS_ACCEL
1281         unsigned int time;
1282 #endif
1283         int spent;
1284
1285         netif_vdbg(efx, intr, efx->net_dev,
1286                    "channel %d NAPI poll executing on CPU %d\n",
1287                    channel->channel, raw_smp_processor_id());
1288
1289         spent = efx_process_channel(channel, budget);
1290
1291         xdp_do_flush_map();
1292
1293         if (spent < budget) {
1294                 if (efx_channel_has_rx_queue(channel) &&
1295                     efx->irq_rx_adaptive &&
1296                     unlikely(++channel->irq_count == 1000)) {
1297                         efx_update_irq_mod(efx, channel);
1298                 }
1299
1300 #ifdef CONFIG_RFS_ACCEL
1301                 /* Perhaps expire some ARFS filters */
1302                 time = jiffies - channel->rfs_last_expiry;
1303                 /* Would our quota be >= 20? */
1304                 if (channel->rfs_filter_count * time >= 600 * HZ)
1305                         mod_delayed_work(system_wq, &channel->filter_work, 0);
1306 #endif
1307
1308                 /* There is no race here; although napi_disable() will
1309                  * only wait for napi_complete(), this isn't a problem
1310                  * since efx_nic_eventq_read_ack() will have no effect if
1311                  * interrupts have already been disabled.
1312                  */
1313                 if (napi_complete_done(napi, spent))
1314                         efx_nic_eventq_read_ack(channel);
1315         }
1316
1317         return spent;
1318 }
1319
1320 void efx_init_napi_channel(struct efx_channel *channel)
1321 {
1322         struct efx_nic *efx = channel->efx;
1323
1324         channel->napi_dev = efx->net_dev;
1325         netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll);
1326 }
1327
1328 void efx_init_napi(struct efx_nic *efx)
1329 {
1330         struct efx_channel *channel;
1331
1332         efx_for_each_channel(channel, efx)
1333                 efx_init_napi_channel(channel);
1334 }
1335
1336 void efx_fini_napi_channel(struct efx_channel *channel)
1337 {
1338         if (channel->napi_dev)
1339                 netif_napi_del(&channel->napi_str);
1340
1341         channel->napi_dev = NULL;
1342 }
1343
1344 void efx_fini_napi(struct efx_nic *efx)
1345 {
1346         struct efx_channel *channel;
1347
1348         efx_for_each_channel(channel, efx)
1349                 efx_fini_napi_channel(channel);
1350 }
1351
1352 /***************
1353  * Housekeeping
1354  ***************/
1355
1356 static int efx_channel_dummy_op_int(struct efx_channel *channel)
1357 {
1358         return 0;
1359 }
1360
1361 void efx_channel_dummy_op_void(struct efx_channel *channel)
1362 {
1363 }
1364
1365 static const struct efx_channel_type efx_default_channel_type = {
1366         .pre_probe              = efx_channel_dummy_op_int,
1367         .post_remove            = efx_channel_dummy_op_void,
1368         .get_name               = efx_get_channel_name,
1369         .copy                   = efx_copy_channel,
1370         .want_txqs              = efx_default_channel_want_txqs,
1371         .keep_eventq            = false,
1372         .want_pio               = true,
1373 };