Commit | Line | Data |
---|---|---|
768fd266 AM |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /**************************************************************************** | |
3 | * Driver for Solarflare network controllers and boards | |
4 | * Copyright 2018 Solarflare Communications Inc. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License version 2 as published | |
8 | * by the Free Software Foundation, incorporated herein by reference. | |
9 | */ | |
10 | ||
11 | #include "net_driver.h" | |
12 | #include <linux/module.h> | |
b6459415 | 13 | #include <linux/filter.h> |
768fd266 AM |
14 | #include "efx_channels.h" |
15 | #include "efx.h" | |
16 | #include "efx_common.h" | |
17 | #include "tx_common.h" | |
18 | #include "rx_common.h" | |
19 | #include "nic.h" | |
20 | #include "sriov.h" | |
e26ca4b5 | 21 | #include "workarounds.h" |
768fd266 | 22 | |
83975485 AM |
23 | /* This is the first interrupt mode to try out of: |
24 | * 0 => MSI-X | |
25 | * 1 => MSI | |
26 | * 2 => legacy | |
27 | */ | |
e4ff3232 | 28 | unsigned int efx_interrupt_mode = EFX_INT_MODE_MSIX; |
83975485 | 29 | |
37c45a4e AM |
30 | /* This is the requested number of CPUs to use for Receive-Side Scaling (RSS), |
31 | * i.e. the number of CPUs among which we may distribute simultaneous | |
32 | * interrupt handling. | |
33 | * | |
34 | * Cards without MSI-X will only target one CPU via legacy or MSI interrupt. | |
35 | * The default (0) means to assign an interrupt to each core. | |
36 | */ | |
67e6398e | 37 | unsigned int rss_cpus; |
37c45a4e | 38 | |
768fd266 AM |
39 | static unsigned int irq_adapt_low_thresh = 8000; |
40 | module_param(irq_adapt_low_thresh, uint, 0644); | |
41 | MODULE_PARM_DESC(irq_adapt_low_thresh, | |
42 | "Threshold score for reducing IRQ moderation"); | |
43 | ||
44 | static unsigned int irq_adapt_high_thresh = 16000; | |
45 | module_param(irq_adapt_high_thresh, uint, 0644); | |
46 | MODULE_PARM_DESC(irq_adapt_high_thresh, | |
47 | "Threshold score for increasing IRQ moderation"); | |
48 | ||
54fccfdd | 49 | static const struct efx_channel_type efx_default_channel_type; |
83975485 | 50 | |
37c45a4e AM |
51 | /************* |
52 | * INTERRUPTS | |
53 | *************/ | |
54 | ||
c265b569 | 55 | static unsigned int count_online_cores(struct efx_nic *efx, bool local_node) |
37c45a4e | 56 | { |
c265b569 | 57 | cpumask_var_t filter_mask; |
37c45a4e AM |
58 | unsigned int count; |
59 | int cpu; | |
60 | ||
c265b569 ÍH |
61 | if (unlikely(!zalloc_cpumask_var(&filter_mask, GFP_KERNEL))) { |
62 | netif_warn(efx, probe, efx->net_dev, | |
63 | "RSS disabled due to allocation failure\n"); | |
64 | return 1; | |
65 | } | |
66 | ||
67 | cpumask_copy(filter_mask, cpu_online_mask); | |
c9ad266b MH |
68 | if (local_node) |
69 | cpumask_and(filter_mask, filter_mask, | |
70 | cpumask_of_pcibus(efx->pci_dev->bus)); | |
c265b569 ÍH |
71 | |
72 | count = 0; | |
73 | for_each_cpu(cpu, filter_mask) { | |
74 | ++count; | |
75 | cpumask_andnot(filter_mask, filter_mask, topology_sibling_cpumask(cpu)); | |
76 | } | |
77 | ||
78 | free_cpumask_var(filter_mask); | |
79 | ||
80 | return count; | |
81 | } | |
82 | ||
83 | static unsigned int efx_wanted_parallelism(struct efx_nic *efx) | |
84 | { | |
85 | unsigned int count; | |
86 | ||
37c45a4e AM |
87 | if (rss_cpus) { |
88 | count = rss_cpus; | |
89 | } else { | |
c265b569 | 90 | count = count_online_cores(efx, true); |
37c45a4e | 91 | |
c265b569 ÍH |
92 | /* If no online CPUs in local node, fallback to any online CPUs */ |
93 | if (count == 0) | |
94 | count = count_online_cores(efx, false); | |
37c45a4e AM |
95 | } |
96 | ||
97 | if (count > EFX_MAX_RX_QUEUES) { | |
98 | netif_cond_dbg(efx, probe, efx->net_dev, !rss_cpus, warn, | |
99 | "Reducing number of rx queues from %u to %u.\n", | |
100 | count, EFX_MAX_RX_QUEUES); | |
101 | count = EFX_MAX_RX_QUEUES; | |
102 | } | |
103 | ||
104 | /* If RSS is requested for the PF *and* VFs then we can't write RSS | |
105 | * table entries that are inaccessible to VFs | |
106 | */ | |
107 | #ifdef CONFIG_SFC_SRIOV | |
108 | if (efx->type->sriov_wanted) { | |
109 | if (efx->type->sriov_wanted(efx) && efx_vf_size(efx) > 1 && | |
110 | count > efx_vf_size(efx)) { | |
111 | netif_warn(efx, probe, efx->net_dev, | |
112 | "Reducing number of RSS channels from %u to %u for " | |
113 | "VF support. Increase vf-msix-limit to use more " | |
114 | "channels on the PF.\n", | |
115 | count, efx_vf_size(efx)); | |
116 | count = efx_vf_size(efx); | |
117 | } | |
118 | } | |
119 | #endif | |
120 | ||
121 | return count; | |
122 | } | |
123 | ||
124 | static int efx_allocate_msix_channels(struct efx_nic *efx, | |
125 | unsigned int max_channels, | |
126 | unsigned int extra_channels, | |
127 | unsigned int parallelism) | |
128 | { | |
129 | unsigned int n_channels = parallelism; | |
130 | int vec_count; | |
e26ca4b5 | 131 | int tx_per_ev; |
37c45a4e AM |
132 | int n_xdp_tx; |
133 | int n_xdp_ev; | |
134 | ||
135 | if (efx_separate_tx_channels) | |
136 | n_channels *= 2; | |
137 | n_channels += extra_channels; | |
138 | ||
139 | /* To allow XDP transmit to happen from arbitrary NAPI contexts | |
140 | * we allocate a TX queue per CPU. We share event queues across | |
141 | * multiple tx queues, assuming tx and ev queues are both | |
142 | * maximum size. | |
143 | */ | |
e26ca4b5 | 144 | tx_per_ev = EFX_MAX_EVQ_SIZE / EFX_TXQ_MAX_ENT(efx); |
f28100cb | 145 | tx_per_ev = min(tx_per_ev, EFX_MAX_TXQ_PER_CHANNEL); |
37c45a4e | 146 | n_xdp_tx = num_possible_cpus(); |
e26ca4b5 | 147 | n_xdp_ev = DIV_ROUND_UP(n_xdp_tx, tx_per_ev); |
37c45a4e AM |
148 | |
149 | vec_count = pci_msix_vec_count(efx->pci_dev); | |
150 | if (vec_count < 0) | |
151 | return vec_count; | |
152 | ||
153 | max_channels = min_t(unsigned int, vec_count, max_channels); | |
154 | ||
155 | /* Check resources. | |
156 | * We need a channel per event queue, plus a VI per tx queue. | |
157 | * This may be more pessimistic than it needs to be. | |
158 | */ | |
41544618 | 159 | if (n_channels >= max_channels) { |
6215b608 ÍH |
160 | efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED; |
161 | netif_warn(efx, drv, efx->net_dev, | |
162 | "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n", | |
163 | n_xdp_ev, n_channels, max_channels); | |
164 | netif_warn(efx, drv, efx->net_dev, | |
165 | "XDP_TX and XDP_REDIRECT might decrease device's performance\n"); | |
de5f32e2 | 166 | } else if (n_channels + n_xdp_tx > efx->max_vis) { |
6215b608 ÍH |
167 | efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_BORROWED; |
168 | netif_warn(efx, drv, efx->net_dev, | |
169 | "Insufficient resources for %d XDP TX queues (%d other channels, max VIs %d)\n", | |
170 | n_xdp_tx, n_channels, efx->max_vis); | |
171 | netif_warn(efx, drv, efx->net_dev, | |
172 | "XDP_TX and XDP_REDIRECT might decrease device's performance\n"); | |
41544618 ÍH |
173 | } else if (n_channels + n_xdp_ev > max_channels) { |
174 | efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_SHARED; | |
175 | netif_warn(efx, drv, efx->net_dev, | |
176 | "Insufficient resources for %d XDP event queues (%d other channels, max %d)\n", | |
177 | n_xdp_ev, n_channels, max_channels); | |
178 | ||
179 | n_xdp_ev = max_channels - n_channels; | |
180 | netif_warn(efx, drv, efx->net_dev, | |
181 | "XDP_TX and XDP_REDIRECT will work with reduced performance (%d cpus/tx_queue)\n", | |
182 | DIV_ROUND_UP(n_xdp_tx, tx_per_ev * n_xdp_ev)); | |
37c45a4e | 183 | } else { |
41544618 ÍH |
184 | efx->xdp_txq_queues_mode = EFX_XDP_TX_QUEUES_DEDICATED; |
185 | } | |
186 | ||
6215b608 | 187 | if (efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_BORROWED) { |
37c45a4e | 188 | efx->n_xdp_channels = n_xdp_ev; |
f28100cb | 189 | efx->xdp_tx_per_channel = tx_per_ev; |
37c45a4e AM |
190 | efx->xdp_tx_queue_count = n_xdp_tx; |
191 | n_channels += n_xdp_ev; | |
192 | netif_dbg(efx, drv, efx->net_dev, | |
193 | "Allocating %d TX and %d event queues for XDP\n", | |
41544618 ÍH |
194 | n_xdp_ev * tx_per_ev, n_xdp_ev); |
195 | } else { | |
196 | efx->n_xdp_channels = 0; | |
197 | efx->xdp_tx_per_channel = 0; | |
6215b608 | 198 | efx->xdp_tx_queue_count = n_xdp_tx; |
37c45a4e AM |
199 | } |
200 | ||
201 | if (vec_count < n_channels) { | |
202 | netif_err(efx, drv, efx->net_dev, | |
203 | "WARNING: Insufficient MSI-X vectors available (%d < %u).\n", | |
204 | vec_count, n_channels); | |
205 | netif_err(efx, drv, efx->net_dev, | |
206 | "WARNING: Performance may be reduced.\n"); | |
207 | n_channels = vec_count; | |
208 | } | |
209 | ||
210 | n_channels = min(n_channels, max_channels); | |
211 | ||
212 | efx->n_channels = n_channels; | |
213 | ||
214 | /* Ignore XDP tx channels when creating rx channels. */ | |
215 | n_channels -= efx->n_xdp_channels; | |
216 | ||
217 | if (efx_separate_tx_channels) { | |
218 | efx->n_tx_channels = | |
219 | min(max(n_channels / 2, 1U), | |
220 | efx->max_tx_channels); | |
221 | efx->tx_channel_offset = | |
222 | n_channels - efx->n_tx_channels; | |
223 | efx->n_rx_channels = | |
224 | max(n_channels - | |
225 | efx->n_tx_channels, 1U); | |
226 | } else { | |
227 | efx->n_tx_channels = min(n_channels, efx->max_tx_channels); | |
228 | efx->tx_channel_offset = 0; | |
229 | efx->n_rx_channels = n_channels; | |
230 | } | |
231 | ||
232 | efx->n_rx_channels = min(efx->n_rx_channels, parallelism); | |
233 | efx->n_tx_channels = min(efx->n_tx_channels, parallelism); | |
234 | ||
235 | efx->xdp_channel_offset = n_channels; | |
236 | ||
237 | netif_dbg(efx, drv, efx->net_dev, | |
238 | "Allocating %u RX channels\n", | |
239 | efx->n_rx_channels); | |
240 | ||
241 | return efx->n_channels; | |
242 | } | |
243 | ||
244 | /* Probe the number and type of interrupts we are able to obtain, and | |
245 | * the resulting numbers of channels and RX queues. | |
246 | */ | |
247 | int efx_probe_interrupts(struct efx_nic *efx) | |
248 | { | |
249 | unsigned int extra_channels = 0; | |
250 | unsigned int rss_spread; | |
251 | unsigned int i, j; | |
252 | int rc; | |
253 | ||
254 | for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) | |
255 | if (efx->extra_channel_type[i]) | |
256 | ++extra_channels; | |
257 | ||
258 | if (efx->interrupt_mode == EFX_INT_MODE_MSIX) { | |
259 | unsigned int parallelism = efx_wanted_parallelism(efx); | |
260 | struct msix_entry xentries[EFX_MAX_CHANNELS]; | |
261 | unsigned int n_channels; | |
262 | ||
263 | rc = efx_allocate_msix_channels(efx, efx->max_channels, | |
264 | extra_channels, parallelism); | |
265 | if (rc >= 0) { | |
266 | n_channels = rc; | |
267 | for (i = 0; i < n_channels; i++) | |
268 | xentries[i].entry = i; | |
269 | rc = pci_enable_msix_range(efx->pci_dev, xentries, 1, | |
270 | n_channels); | |
271 | } | |
272 | if (rc < 0) { | |
273 | /* Fall back to single channel MSI */ | |
274 | netif_err(efx, drv, efx->net_dev, | |
275 | "could not enable MSI-X\n"); | |
276 | if (efx->type->min_interrupt_mode >= EFX_INT_MODE_MSI) | |
277 | efx->interrupt_mode = EFX_INT_MODE_MSI; | |
278 | else | |
279 | return rc; | |
280 | } else if (rc < n_channels) { | |
281 | netif_err(efx, drv, efx->net_dev, | |
282 | "WARNING: Insufficient MSI-X vectors" | |
283 | " available (%d < %u).\n", rc, n_channels); | |
284 | netif_err(efx, drv, efx->net_dev, | |
285 | "WARNING: Performance may be reduced.\n"); | |
286 | n_channels = rc; | |
287 | } | |
288 | ||
289 | if (rc > 0) { | |
290 | for (i = 0; i < efx->n_channels; i++) | |
291 | efx_get_channel(efx, i)->irq = | |
292 | xentries[i].vector; | |
293 | } | |
294 | } | |
295 | ||
296 | /* Try single interrupt MSI */ | |
297 | if (efx->interrupt_mode == EFX_INT_MODE_MSI) { | |
298 | efx->n_channels = 1; | |
299 | efx->n_rx_channels = 1; | |
300 | efx->n_tx_channels = 1; | |
c308dfd1 | 301 | efx->tx_channel_offset = 0; |
37c45a4e AM |
302 | efx->n_xdp_channels = 0; |
303 | efx->xdp_channel_offset = efx->n_channels; | |
304 | rc = pci_enable_msi(efx->pci_dev); | |
305 | if (rc == 0) { | |
306 | efx_get_channel(efx, 0)->irq = efx->pci_dev->irq; | |
307 | } else { | |
308 | netif_err(efx, drv, efx->net_dev, | |
309 | "could not enable MSI\n"); | |
310 | if (efx->type->min_interrupt_mode >= EFX_INT_MODE_LEGACY) | |
311 | efx->interrupt_mode = EFX_INT_MODE_LEGACY; | |
312 | else | |
313 | return rc; | |
314 | } | |
315 | } | |
316 | ||
317 | /* Assume legacy interrupts */ | |
318 | if (efx->interrupt_mode == EFX_INT_MODE_LEGACY) { | |
319 | efx->n_channels = 1 + (efx_separate_tx_channels ? 1 : 0); | |
320 | efx->n_rx_channels = 1; | |
321 | efx->n_tx_channels = 1; | |
f232af42 | 322 | efx->tx_channel_offset = efx_separate_tx_channels ? 1 : 0; |
37c45a4e AM |
323 | efx->n_xdp_channels = 0; |
324 | efx->xdp_channel_offset = efx->n_channels; | |
325 | efx->legacy_irq = efx->pci_dev->irq; | |
326 | } | |
327 | ||
328 | /* Assign extra channels if possible, before XDP channels */ | |
329 | efx->n_extra_tx_channels = 0; | |
330 | j = efx->xdp_channel_offset; | |
331 | for (i = 0; i < EFX_MAX_EXTRA_CHANNELS; i++) { | |
332 | if (!efx->extra_channel_type[i]) | |
333 | continue; | |
334 | if (j <= efx->tx_channel_offset + efx->n_tx_channels) { | |
335 | efx->extra_channel_type[i]->handle_no_channel(efx); | |
336 | } else { | |
337 | --j; | |
338 | efx_get_channel(efx, j)->type = | |
339 | efx->extra_channel_type[i]; | |
340 | if (efx_channel_has_tx_queues(efx_get_channel(efx, j))) | |
341 | efx->n_extra_tx_channels++; | |
342 | } | |
343 | } | |
344 | ||
345 | rss_spread = efx->n_rx_channels; | |
346 | /* RSS might be usable on VFs even if it is disabled on the PF */ | |
347 | #ifdef CONFIG_SFC_SRIOV | |
348 | if (efx->type->sriov_wanted) { | |
349 | efx->rss_spread = ((rss_spread > 1 || | |
350 | !efx->type->sriov_wanted(efx)) ? | |
351 | rss_spread : efx_vf_size(efx)); | |
352 | return 0; | |
353 | } | |
354 | #endif | |
355 | efx->rss_spread = rss_spread; | |
356 | ||
357 | return 0; | |
358 | } | |
359 | ||
360 | #if defined(CONFIG_SMP) | |
361 | void efx_set_interrupt_affinity(struct efx_nic *efx) | |
362 | { | |
c9ad266b | 363 | const struct cpumask *numa_mask = cpumask_of_pcibus(efx->pci_dev->bus); |
37c45a4e AM |
364 | struct efx_channel *channel; |
365 | unsigned int cpu; | |
366 | ||
09a99ab1 ÍH |
367 | /* If no online CPUs in local node, fallback to any online CPU */ |
368 | if (cpumask_first_and(cpu_online_mask, numa_mask) >= nr_cpu_ids) | |
369 | numa_mask = cpu_online_mask; | |
370 | ||
371 | cpu = -1; | |
37c45a4e | 372 | efx_for_each_channel(channel, efx) { |
09a99ab1 ÍH |
373 | cpu = cpumask_next_and(cpu, cpu_online_mask, numa_mask); |
374 | if (cpu >= nr_cpu_ids) | |
375 | cpu = cpumask_first_and(cpu_online_mask, numa_mask); | |
37c45a4e AM |
376 | irq_set_affinity_hint(channel->irq, cpumask_of(cpu)); |
377 | } | |
378 | } | |
379 | ||
380 | void efx_clear_interrupt_affinity(struct efx_nic *efx) | |
381 | { | |
382 | struct efx_channel *channel; | |
383 | ||
384 | efx_for_each_channel(channel, efx) | |
385 | irq_set_affinity_hint(channel->irq, NULL); | |
386 | } | |
387 | #else | |
388 | void | |
389 | efx_set_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) | |
390 | { | |
391 | } | |
392 | ||
393 | void | |
394 | efx_clear_interrupt_affinity(struct efx_nic *efx __attribute__ ((unused))) | |
395 | { | |
396 | } | |
397 | #endif /* CONFIG_SMP */ | |
398 | ||
399 | void efx_remove_interrupts(struct efx_nic *efx) | |
400 | { | |
401 | struct efx_channel *channel; | |
402 | ||
403 | /* Remove MSI/MSI-X interrupts */ | |
404 | efx_for_each_channel(channel, efx) | |
405 | channel->irq = 0; | |
406 | pci_disable_msi(efx->pci_dev); | |
407 | pci_disable_msix(efx->pci_dev); | |
408 | ||
409 | /* Remove legacy interrupt */ | |
410 | efx->legacy_irq = 0; | |
411 | } | |
412 | ||
5f999256 AM |
413 | /*************** |
414 | * EVENT QUEUES | |
415 | ***************/ | |
416 | ||
417 | /* Create event queue | |
418 | * Event queue memory allocations are done only once. If the channel | |
419 | * is reset, the memory buffer will be reused; this guards against | |
420 | * errors during channel reset and also simplifies interrupt handling. | |
421 | */ | |
422 | int efx_probe_eventq(struct efx_channel *channel) | |
423 | { | |
424 | struct efx_nic *efx = channel->efx; | |
425 | unsigned long entries; | |
426 | ||
427 | netif_dbg(efx, probe, efx->net_dev, | |
428 | "chan %d create event queue\n", channel->channel); | |
429 | ||
430 | /* Build an event queue with room for one event per tx and rx buffer, | |
431 | * plus some extra for link state events and MCDI completions. | |
432 | */ | |
433 | entries = roundup_pow_of_two(efx->rxq_entries + efx->txq_entries + 128); | |
434 | EFX_WARN_ON_PARANOID(entries > EFX_MAX_EVQ_SIZE); | |
435 | channel->eventq_mask = max(entries, EFX_MIN_EVQ_SIZE) - 1; | |
436 | ||
437 | return efx_nic_probe_eventq(channel); | |
438 | } | |
439 | ||
440 | /* Prepare channel's event queue */ | |
441 | int efx_init_eventq(struct efx_channel *channel) | |
442 | { | |
443 | struct efx_nic *efx = channel->efx; | |
444 | int rc; | |
445 | ||
446 | EFX_WARN_ON_PARANOID(channel->eventq_init); | |
447 | ||
448 | netif_dbg(efx, drv, efx->net_dev, | |
449 | "chan %d init event queue\n", channel->channel); | |
450 | ||
451 | rc = efx_nic_init_eventq(channel); | |
452 | if (rc == 0) { | |
453 | efx->type->push_irq_moderation(channel); | |
454 | channel->eventq_read_ptr = 0; | |
455 | channel->eventq_init = true; | |
456 | } | |
457 | return rc; | |
458 | } | |
459 | ||
460 | /* Enable event queue processing and NAPI */ | |
461 | void efx_start_eventq(struct efx_channel *channel) | |
462 | { | |
463 | netif_dbg(channel->efx, ifup, channel->efx->net_dev, | |
464 | "chan %d start event queue\n", channel->channel); | |
465 | ||
466 | /* Make sure the NAPI handler sees the enabled flag set */ | |
467 | channel->enabled = true; | |
468 | smp_wmb(); | |
469 | ||
470 | napi_enable(&channel->napi_str); | |
471 | efx_nic_eventq_read_ack(channel); | |
472 | } | |
473 | ||
474 | /* Disable event queue processing and NAPI */ | |
475 | void efx_stop_eventq(struct efx_channel *channel) | |
476 | { | |
477 | if (!channel->enabled) | |
478 | return; | |
479 | ||
480 | napi_disable(&channel->napi_str); | |
481 | channel->enabled = false; | |
482 | } | |
483 | ||
484 | void efx_fini_eventq(struct efx_channel *channel) | |
485 | { | |
486 | if (!channel->eventq_init) | |
487 | return; | |
488 | ||
489 | netif_dbg(channel->efx, drv, channel->efx->net_dev, | |
490 | "chan %d fini event queue\n", channel->channel); | |
491 | ||
492 | efx_nic_fini_eventq(channel); | |
493 | channel->eventq_init = false; | |
494 | } | |
495 | ||
496 | void efx_remove_eventq(struct efx_channel *channel) | |
497 | { | |
498 | netif_dbg(channel->efx, drv, channel->efx->net_dev, | |
499 | "chan %d remove event queue\n", channel->channel); | |
500 | ||
501 | efx_nic_remove_eventq(channel); | |
502 | } | |
503 | ||
83975485 AM |
504 | /************************************************************************** |
505 | * | |
506 | * Channel handling | |
507 | * | |
508 | *************************************************************************/ | |
509 | ||
025c5a0b EC |
510 | #ifdef CONFIG_RFS_ACCEL |
511 | static void efx_filter_rfs_expire(struct work_struct *data) | |
512 | { | |
513 | struct delayed_work *dwork = to_delayed_work(data); | |
514 | struct efx_channel *channel; | |
515 | unsigned int time, quota; | |
516 | ||
517 | channel = container_of(dwork, struct efx_channel, filter_work); | |
518 | time = jiffies - channel->rfs_last_expiry; | |
519 | quota = channel->rfs_filter_count * time / (30 * HZ); | |
520 | if (quota >= 20 && __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, quota))) | |
521 | channel->rfs_last_expiry += time; | |
522 | /* Ensure we do more work eventually even if NAPI poll is not happening */ | |
523 | schedule_delayed_work(dwork, 30 * HZ); | |
524 | } | |
525 | #endif | |
526 | ||
83975485 | 527 | /* Allocate and initialise a channel structure. */ |
4da24fa6 | 528 | static struct efx_channel *efx_alloc_channel(struct efx_nic *efx, int i) |
83975485 AM |
529 | { |
530 | struct efx_rx_queue *rx_queue; | |
531 | struct efx_tx_queue *tx_queue; | |
532 | struct efx_channel *channel; | |
533 | int j; | |
534 | ||
535 | channel = kzalloc(sizeof(*channel), GFP_KERNEL); | |
536 | if (!channel) | |
537 | return NULL; | |
538 | ||
539 | channel->efx = efx; | |
540 | channel->channel = i; | |
541 | channel->type = &efx_default_channel_type; | |
542 | ||
12804793 | 543 | for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) { |
83975485 AM |
544 | tx_queue = &channel->tx_queue[j]; |
545 | tx_queue->efx = efx; | |
a81dcd85 EC |
546 | tx_queue->queue = -1; |
547 | tx_queue->label = j; | |
83975485 AM |
548 | tx_queue->channel = channel; |
549 | } | |
550 | ||
551 | #ifdef CONFIG_RFS_ACCEL | |
552 | INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); | |
553 | #endif | |
554 | ||
555 | rx_queue = &channel->rx_queue; | |
556 | rx_queue->efx = efx; | |
557 | timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); | |
558 | ||
559 | return channel; | |
560 | } | |
561 | ||
562 | int efx_init_channels(struct efx_nic *efx) | |
563 | { | |
564 | unsigned int i; | |
565 | ||
566 | for (i = 0; i < EFX_MAX_CHANNELS; i++) { | |
4da24fa6 | 567 | efx->channel[i] = efx_alloc_channel(efx, i); |
83975485 AM |
568 | if (!efx->channel[i]) |
569 | return -ENOMEM; | |
570 | efx->msi_context[i].efx = efx; | |
571 | efx->msi_context[i].index = i; | |
572 | } | |
573 | ||
574 | /* Higher numbered interrupt modes are less capable! */ | |
83975485 | 575 | efx->interrupt_mode = min(efx->type->min_interrupt_mode, |
e4ff3232 | 576 | efx_interrupt_mode); |
83975485 | 577 | |
937aa3ae EC |
578 | efx->max_channels = EFX_MAX_CHANNELS; |
579 | efx->max_tx_channels = EFX_MAX_CHANNELS; | |
580 | ||
83975485 AM |
581 | return 0; |
582 | } | |
583 | ||
584 | void efx_fini_channels(struct efx_nic *efx) | |
585 | { | |
586 | unsigned int i; | |
587 | ||
588 | for (i = 0; i < EFX_MAX_CHANNELS; i++) | |
589 | if (efx->channel[i]) { | |
590 | kfree(efx->channel[i]); | |
591 | efx->channel[i] = NULL; | |
592 | } | |
593 | } | |
594 | ||
595 | /* Allocate and initialise a channel structure, copying parameters | |
596 | * (but not resources) from an old channel structure. | |
597 | */ | |
598 | struct efx_channel *efx_copy_channel(const struct efx_channel *old_channel) | |
599 | { | |
600 | struct efx_rx_queue *rx_queue; | |
601 | struct efx_tx_queue *tx_queue; | |
602 | struct efx_channel *channel; | |
603 | int j; | |
604 | ||
605 | channel = kmalloc(sizeof(*channel), GFP_KERNEL); | |
606 | if (!channel) | |
607 | return NULL; | |
608 | ||
609 | *channel = *old_channel; | |
610 | ||
611 | channel->napi_dev = NULL; | |
612 | INIT_HLIST_NODE(&channel->napi_str.napi_hash_node); | |
613 | channel->napi_str.napi_id = 0; | |
614 | channel->napi_str.state = 0; | |
615 | memset(&channel->eventq, 0, sizeof(channel->eventq)); | |
616 | ||
12804793 | 617 | for (j = 0; j < EFX_MAX_TXQ_PER_CHANNEL; j++) { |
83975485 AM |
618 | tx_queue = &channel->tx_queue[j]; |
619 | if (tx_queue->channel) | |
620 | tx_queue->channel = channel; | |
621 | tx_queue->buffer = NULL; | |
4b1bd9db | 622 | tx_queue->cb_page = NULL; |
83975485 AM |
623 | memset(&tx_queue->txd, 0, sizeof(tx_queue->txd)); |
624 | } | |
625 | ||
626 | rx_queue = &channel->rx_queue; | |
627 | rx_queue->buffer = NULL; | |
628 | memset(&rx_queue->rxd, 0, sizeof(rx_queue->rxd)); | |
629 | timer_setup(&rx_queue->slow_fill, efx_rx_slow_fill, 0); | |
630 | #ifdef CONFIG_RFS_ACCEL | |
631 | INIT_DELAYED_WORK(&channel->filter_work, efx_filter_rfs_expire); | |
632 | #endif | |
633 | ||
634 | return channel; | |
635 | } | |
636 | ||
637 | static int efx_probe_channel(struct efx_channel *channel) | |
638 | { | |
639 | struct efx_tx_queue *tx_queue; | |
640 | struct efx_rx_queue *rx_queue; | |
641 | int rc; | |
642 | ||
643 | netif_dbg(channel->efx, probe, channel->efx->net_dev, | |
644 | "creating channel %d\n", channel->channel); | |
645 | ||
646 | rc = channel->type->pre_probe(channel); | |
647 | if (rc) | |
648 | goto fail; | |
649 | ||
650 | rc = efx_probe_eventq(channel); | |
651 | if (rc) | |
652 | goto fail; | |
653 | ||
654 | efx_for_each_channel_tx_queue(tx_queue, channel) { | |
655 | rc = efx_probe_tx_queue(tx_queue); | |
656 | if (rc) | |
657 | goto fail; | |
658 | } | |
659 | ||
660 | efx_for_each_channel_rx_queue(rx_queue, channel) { | |
661 | rc = efx_probe_rx_queue(rx_queue); | |
662 | if (rc) | |
663 | goto fail; | |
664 | } | |
665 | ||
666 | channel->rx_list = NULL; | |
667 | ||
668 | return 0; | |
669 | ||
670 | fail: | |
671 | efx_remove_channel(channel); | |
672 | return rc; | |
673 | } | |
674 | ||
54fccfdd MH |
675 | static void efx_get_channel_name(struct efx_channel *channel, char *buf, |
676 | size_t len) | |
83975485 AM |
677 | { |
678 | struct efx_nic *efx = channel->efx; | |
679 | const char *type; | |
680 | int number; | |
681 | ||
682 | number = channel->channel; | |
683 | ||
684 | if (number >= efx->xdp_channel_offset && | |
685 | !WARN_ON_ONCE(!efx->n_xdp_channels)) { | |
686 | type = "-xdp"; | |
687 | number -= efx->xdp_channel_offset; | |
688 | } else if (efx->tx_channel_offset == 0) { | |
689 | type = ""; | |
690 | } else if (number < efx->tx_channel_offset) { | |
691 | type = "-rx"; | |
692 | } else { | |
693 | type = "-tx"; | |
694 | number -= efx->tx_channel_offset; | |
695 | } | |
696 | snprintf(buf, len, "%s%s-%d", efx->name, type, number); | |
697 | } | |
698 | ||
699 | void efx_set_channel_names(struct efx_nic *efx) | |
700 | { | |
701 | struct efx_channel *channel; | |
702 | ||
703 | efx_for_each_channel(channel, efx) | |
704 | channel->type->get_name(channel, | |
705 | efx->msi_context[channel->channel].name, | |
706 | sizeof(efx->msi_context[0].name)); | |
707 | } | |
708 | ||
709 | int efx_probe_channels(struct efx_nic *efx) | |
710 | { | |
711 | struct efx_channel *channel; | |
712 | int rc; | |
713 | ||
714 | /* Restart special buffer allocation */ | |
715 | efx->next_buffer_table = 0; | |
716 | ||
717 | /* Probe channels in reverse, so that any 'extra' channels | |
718 | * use the start of the buffer table. This allows the traffic | |
719 | * channels to be resized without moving them or wasting the | |
720 | * entries before them. | |
721 | */ | |
722 | efx_for_each_channel_rev(channel, efx) { | |
723 | rc = efx_probe_channel(channel); | |
724 | if (rc) { | |
725 | netif_err(efx, probe, efx->net_dev, | |
726 | "failed to create channel %d\n", | |
727 | channel->channel); | |
728 | goto fail; | |
729 | } | |
730 | } | |
731 | efx_set_channel_names(efx); | |
732 | ||
733 | return 0; | |
734 | ||
735 | fail: | |
736 | efx_remove_channels(efx); | |
737 | return rc; | |
738 | } | |
739 | ||
740 | void efx_remove_channel(struct efx_channel *channel) | |
741 | { | |
742 | struct efx_tx_queue *tx_queue; | |
743 | struct efx_rx_queue *rx_queue; | |
744 | ||
745 | netif_dbg(channel->efx, drv, channel->efx->net_dev, | |
746 | "destroy chan %d\n", channel->channel); | |
747 | ||
748 | efx_for_each_channel_rx_queue(rx_queue, channel) | |
749 | efx_remove_rx_queue(rx_queue); | |
f9cac93e | 750 | efx_for_each_channel_tx_queue(tx_queue, channel) |
83975485 AM |
751 | efx_remove_tx_queue(tx_queue); |
752 | efx_remove_eventq(channel); | |
753 | channel->type->post_remove(channel); | |
754 | } | |
755 | ||
756 | void efx_remove_channels(struct efx_nic *efx) | |
757 | { | |
758 | struct efx_channel *channel; | |
759 | ||
760 | efx_for_each_channel(channel, efx) | |
761 | efx_remove_channel(channel); | |
762 | ||
763 | kfree(efx->xdp_tx_queues); | |
764 | } | |
765 | ||
059a47f1 TY |
766 | static int efx_set_xdp_tx_queue(struct efx_nic *efx, int xdp_queue_number, |
767 | struct efx_tx_queue *tx_queue) | |
768 | { | |
769 | if (xdp_queue_number >= efx->xdp_tx_queue_count) | |
770 | return -EINVAL; | |
771 | ||
772 | netif_dbg(efx, drv, efx->net_dev, | |
773 | "Channel %u TXQ %u is XDP %u, HW %u\n", | |
774 | tx_queue->channel->channel, tx_queue->label, | |
775 | xdp_queue_number, tx_queue->queue); | |
776 | efx->xdp_tx_queues[xdp_queue_number] = tx_queue; | |
777 | return 0; | |
778 | } | |
779 | ||
780 | static void efx_set_xdp_channels(struct efx_nic *efx) | |
781 | { | |
782 | struct efx_tx_queue *tx_queue; | |
783 | struct efx_channel *channel; | |
784 | unsigned int next_queue = 0; | |
785 | int xdp_queue_number = 0; | |
786 | int rc; | |
787 | ||
788 | /* We need to mark which channels really have RX and TX | |
789 | * queues, and adjust the TX queue numbers if we have separate | |
790 | * RX-only and TX-only channels. | |
791 | */ | |
792 | efx_for_each_channel(channel, efx) { | |
793 | if (channel->channel < efx->tx_channel_offset) | |
794 | continue; | |
795 | ||
796 | if (efx_channel_is_xdp_tx(channel)) { | |
797 | efx_for_each_channel_tx_queue(tx_queue, channel) { | |
798 | tx_queue->queue = next_queue++; | |
799 | rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, | |
800 | tx_queue); | |
801 | if (rc == 0) | |
802 | xdp_queue_number++; | |
803 | } | |
804 | } else { | |
805 | efx_for_each_channel_tx_queue(tx_queue, channel) { | |
806 | tx_queue->queue = next_queue++; | |
807 | netif_dbg(efx, drv, efx->net_dev, | |
808 | "Channel %u TXQ %u is HW %u\n", | |
809 | channel->channel, tx_queue->label, | |
810 | tx_queue->queue); | |
811 | } | |
812 | ||
813 | /* If XDP is borrowing queues from net stack, it must | |
814 | * use the queue with no csum offload, which is the | |
815 | * first one of the channel | |
816 | * (note: tx_queue_by_type is not initialized yet) | |
817 | */ | |
818 | if (efx->xdp_txq_queues_mode == | |
819 | EFX_XDP_TX_QUEUES_BORROWED) { | |
820 | tx_queue = &channel->tx_queue[0]; | |
821 | rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, | |
822 | tx_queue); | |
823 | if (rc == 0) | |
824 | xdp_queue_number++; | |
825 | } | |
826 | } | |
827 | } | |
828 | WARN_ON(efx->xdp_txq_queues_mode == EFX_XDP_TX_QUEUES_DEDICATED && | |
829 | xdp_queue_number != efx->xdp_tx_queue_count); | |
830 | WARN_ON(efx->xdp_txq_queues_mode != EFX_XDP_TX_QUEUES_DEDICATED && | |
831 | xdp_queue_number > efx->xdp_tx_queue_count); | |
832 | ||
833 | /* If we have more CPUs than assigned XDP TX queues, assign the already | |
834 | * existing queues to the exceeding CPUs | |
835 | */ | |
836 | next_queue = 0; | |
837 | while (xdp_queue_number < efx->xdp_tx_queue_count) { | |
838 | tx_queue = efx->xdp_tx_queues[next_queue++]; | |
839 | rc = efx_set_xdp_tx_queue(efx, xdp_queue_number, tx_queue); | |
840 | if (rc == 0) | |
841 | xdp_queue_number++; | |
842 | } | |
843 | } | |
844 | ||
83975485 AM |
845 | int efx_realloc_channels(struct efx_nic *efx, u32 rxq_entries, u32 txq_entries) |
846 | { | |
49e6123c TY |
847 | struct efx_channel *other_channel[EFX_MAX_CHANNELS], *channel, |
848 | *ptp_channel = efx_ptp_channel(efx); | |
849 | struct efx_ptp_data *ptp_data = efx->ptp_data; | |
83975485 AM |
850 | unsigned int i, next_buffer_table = 0; |
851 | u32 old_rxq_entries, old_txq_entries; | |
852 | int rc, rc2; | |
853 | ||
854 | rc = efx_check_disabled(efx); | |
855 | if (rc) | |
856 | return rc; | |
857 | ||
858 | /* Not all channels should be reallocated. We must avoid | |
859 | * reallocating their buffer table entries. | |
860 | */ | |
861 | efx_for_each_channel(channel, efx) { | |
862 | struct efx_rx_queue *rx_queue; | |
863 | struct efx_tx_queue *tx_queue; | |
864 | ||
865 | if (channel->type->copy) | |
866 | continue; | |
867 | next_buffer_table = max(next_buffer_table, | |
868 | channel->eventq.index + | |
869 | channel->eventq.entries); | |
870 | efx_for_each_channel_rx_queue(rx_queue, channel) | |
871 | next_buffer_table = max(next_buffer_table, | |
872 | rx_queue->rxd.index + | |
873 | rx_queue->rxd.entries); | |
874 | efx_for_each_channel_tx_queue(tx_queue, channel) | |
875 | next_buffer_table = max(next_buffer_table, | |
876 | tx_queue->txd.index + | |
877 | tx_queue->txd.entries); | |
878 | } | |
879 | ||
880 | efx_device_detach_sync(efx); | |
881 | efx_stop_all(efx); | |
882 | efx_soft_disable_interrupts(efx); | |
883 | ||
884 | /* Clone channels (where possible) */ | |
885 | memset(other_channel, 0, sizeof(other_channel)); | |
886 | for (i = 0; i < efx->n_channels; i++) { | |
887 | channel = efx->channel[i]; | |
888 | if (channel->type->copy) | |
889 | channel = channel->type->copy(channel); | |
890 | if (!channel) { | |
891 | rc = -ENOMEM; | |
892 | goto out; | |
893 | } | |
894 | other_channel[i] = channel; | |
895 | } | |
896 | ||
897 | /* Swap entry counts and channel pointers */ | |
898 | old_rxq_entries = efx->rxq_entries; | |
899 | old_txq_entries = efx->txq_entries; | |
900 | efx->rxq_entries = rxq_entries; | |
901 | efx->txq_entries = txq_entries; | |
0cf765fb JC |
902 | for (i = 0; i < efx->n_channels; i++) |
903 | swap(efx->channel[i], other_channel[i]); | |
83975485 AM |
904 | |
905 | /* Restart buffer table allocation */ | |
906 | efx->next_buffer_table = next_buffer_table; | |
907 | ||
908 | for (i = 0; i < efx->n_channels; i++) { | |
909 | channel = efx->channel[i]; | |
910 | if (!channel->type->copy) | |
911 | continue; | |
912 | rc = efx_probe_channel(channel); | |
913 | if (rc) | |
914 | goto rollback; | |
915 | efx_init_napi_channel(efx->channel[i]); | |
916 | } | |
917 | ||
059a47f1 | 918 | efx_set_xdp_channels(efx); |
83975485 | 919 | out: |
49e6123c | 920 | efx->ptp_data = NULL; |
83975485 AM |
921 | /* Destroy unused channel structures */ |
922 | for (i = 0; i < efx->n_channels; i++) { | |
923 | channel = other_channel[i]; | |
924 | if (channel && channel->type->copy) { | |
925 | efx_fini_napi_channel(channel); | |
926 | efx_remove_channel(channel); | |
927 | kfree(channel); | |
928 | } | |
929 | } | |
930 | ||
49e6123c | 931 | efx->ptp_data = ptp_data; |
83975485 AM |
932 | rc2 = efx_soft_enable_interrupts(efx); |
933 | if (rc2) { | |
934 | rc = rc ? rc : rc2; | |
935 | netif_err(efx, drv, efx->net_dev, | |
936 | "unable to restart interrupts on channel reallocation\n"); | |
937 | efx_schedule_reset(efx, RESET_TYPE_DISABLE); | |
938 | } else { | |
939 | efx_start_all(efx); | |
940 | efx_device_attach_if_not_resetting(efx); | |
941 | } | |
942 | return rc; | |
943 | ||
944 | rollback: | |
945 | /* Swap back */ | |
946 | efx->rxq_entries = old_rxq_entries; | |
947 | efx->txq_entries = old_txq_entries; | |
0cf765fb JC |
948 | for (i = 0; i < efx->n_channels; i++) |
949 | swap(efx->channel[i], other_channel[i]); | |
49e6123c | 950 | efx_ptp_update_channel(efx, ptp_channel); |
83975485 AM |
951 | goto out; |
952 | } | |
953 | ||
954 | int efx_set_channels(struct efx_nic *efx) | |
955 | { | |
a81dcd85 | 956 | struct efx_channel *channel; |
69a70496 | 957 | int rc; |
83975485 | 958 | |
83975485 AM |
959 | if (efx->xdp_tx_queue_count) { |
960 | EFX_WARN_ON_PARANOID(efx->xdp_tx_queues); | |
961 | ||
962 | /* Allocate array for XDP TX queue lookup. */ | |
963 | efx->xdp_tx_queues = kcalloc(efx->xdp_tx_queue_count, | |
964 | sizeof(*efx->xdp_tx_queues), | |
965 | GFP_KERNEL); | |
966 | if (!efx->xdp_tx_queues) | |
967 | return -ENOMEM; | |
968 | } | |
969 | ||
83975485 AM |
970 | efx_for_each_channel(channel, efx) { |
971 | if (channel->channel < efx->n_rx_channels) | |
972 | channel->rx_queue.core_index = channel->channel; | |
973 | else | |
974 | channel->rx_queue.core_index = -1; | |
83975485 | 975 | } |
41544618 | 976 | |
059a47f1 | 977 | efx_set_xdp_channels(efx); |
69a70496 EC |
978 | |
979 | rc = netif_set_real_num_tx_queues(efx->net_dev, efx->n_tx_channels); | |
980 | if (rc) | |
981 | return rc; | |
982 | return netif_set_real_num_rx_queues(efx->net_dev, efx->n_rx_channels); | |
83975485 AM |
983 | } |
984 | ||
54fccfdd | 985 | static bool efx_default_channel_want_txqs(struct efx_channel *channel) |
83975485 AM |
986 | { |
987 | return channel->channel - channel->efx->tx_channel_offset < | |
988 | channel->efx->n_tx_channels; | |
989 | } | |
990 | ||
e20ba5b1 AM |
991 | /************* |
992 | * START/STOP | |
993 | *************/ | |
994 | ||
995 | int efx_soft_enable_interrupts(struct efx_nic *efx) | |
996 | { | |
997 | struct efx_channel *channel, *end_channel; | |
998 | int rc; | |
999 | ||
1000 | BUG_ON(efx->state == STATE_DISABLED); | |
1001 | ||
1002 | efx->irq_soft_enabled = true; | |
1003 | smp_wmb(); | |
1004 | ||
1005 | efx_for_each_channel(channel, efx) { | |
1006 | if (!channel->type->keep_eventq) { | |
1007 | rc = efx_init_eventq(channel); | |
1008 | if (rc) | |
1009 | goto fail; | |
1010 | } | |
1011 | efx_start_eventq(channel); | |
1012 | } | |
1013 | ||
1014 | efx_mcdi_mode_event(efx); | |
1015 | ||
1016 | return 0; | |
1017 | fail: | |
1018 | end_channel = channel; | |
1019 | efx_for_each_channel(channel, efx) { | |
1020 | if (channel == end_channel) | |
1021 | break; | |
1022 | efx_stop_eventq(channel); | |
1023 | if (!channel->type->keep_eventq) | |
1024 | efx_fini_eventq(channel); | |
1025 | } | |
1026 | ||
1027 | return rc; | |
1028 | } | |
1029 | ||
1030 | void efx_soft_disable_interrupts(struct efx_nic *efx) | |
1031 | { | |
1032 | struct efx_channel *channel; | |
1033 | ||
1034 | if (efx->state == STATE_DISABLED) | |
1035 | return; | |
1036 | ||
1037 | efx_mcdi_mode_poll(efx); | |
1038 | ||
1039 | efx->irq_soft_enabled = false; | |
1040 | smp_wmb(); | |
1041 | ||
1042 | if (efx->legacy_irq) | |
1043 | synchronize_irq(efx->legacy_irq); | |
1044 | ||
1045 | efx_for_each_channel(channel, efx) { | |
1046 | if (channel->irq) | |
1047 | synchronize_irq(channel->irq); | |
1048 | ||
1049 | efx_stop_eventq(channel); | |
1050 | if (!channel->type->keep_eventq) | |
1051 | efx_fini_eventq(channel); | |
1052 | } | |
1053 | ||
1054 | /* Flush the asynchronous MCDI request queue */ | |
1055 | efx_mcdi_flush_async(efx); | |
1056 | } | |
1057 | ||
1058 | int efx_enable_interrupts(struct efx_nic *efx) | |
1059 | { | |
1060 | struct efx_channel *channel, *end_channel; | |
1061 | int rc; | |
1062 | ||
1063 | /* TODO: Is this really a bug? */ | |
1064 | BUG_ON(efx->state == STATE_DISABLED); | |
1065 | ||
1066 | if (efx->eeh_disabled_legacy_irq) { | |
1067 | enable_irq(efx->legacy_irq); | |
1068 | efx->eeh_disabled_legacy_irq = false; | |
1069 | } | |
1070 | ||
1071 | efx->type->irq_enable_master(efx); | |
1072 | ||
1073 | efx_for_each_channel(channel, efx) { | |
1074 | if (channel->type->keep_eventq) { | |
1075 | rc = efx_init_eventq(channel); | |
1076 | if (rc) | |
1077 | goto fail; | |
1078 | } | |
1079 | } | |
1080 | ||
1081 | rc = efx_soft_enable_interrupts(efx); | |
1082 | if (rc) | |
1083 | goto fail; | |
1084 | ||
1085 | return 0; | |
1086 | ||
1087 | fail: | |
1088 | end_channel = channel; | |
1089 | efx_for_each_channel(channel, efx) { | |
1090 | if (channel == end_channel) | |
1091 | break; | |
1092 | if (channel->type->keep_eventq) | |
1093 | efx_fini_eventq(channel); | |
1094 | } | |
1095 | ||
1096 | efx->type->irq_disable_non_ev(efx); | |
1097 | ||
1098 | return rc; | |
1099 | } | |
1100 | ||
1101 | void efx_disable_interrupts(struct efx_nic *efx) | |
1102 | { | |
1103 | struct efx_channel *channel; | |
1104 | ||
1105 | efx_soft_disable_interrupts(efx); | |
1106 | ||
1107 | efx_for_each_channel(channel, efx) { | |
1108 | if (channel->type->keep_eventq) | |
1109 | efx_fini_eventq(channel); | |
1110 | } | |
1111 | ||
1112 | efx->type->irq_disable_non_ev(efx); | |
1113 | } | |
1114 | ||
1115 | void efx_start_channels(struct efx_nic *efx) | |
1116 | { | |
1117 | struct efx_tx_queue *tx_queue; | |
1118 | struct efx_rx_queue *rx_queue; | |
1119 | struct efx_channel *channel; | |
1120 | ||
fb5833d8 | 1121 | efx_for_each_channel_rev(channel, efx) { |
e20ba5b1 AM |
1122 | efx_for_each_channel_tx_queue(tx_queue, channel) { |
1123 | efx_init_tx_queue(tx_queue); | |
1124 | atomic_inc(&efx->active_queues); | |
1125 | } | |
1126 | ||
1127 | efx_for_each_channel_rx_queue(rx_queue, channel) { | |
1128 | efx_init_rx_queue(rx_queue); | |
1129 | atomic_inc(&efx->active_queues); | |
1130 | efx_stop_eventq(channel); | |
1131 | efx_fast_push_rx_descriptors(rx_queue, false); | |
1132 | efx_start_eventq(channel); | |
1133 | } | |
1134 | ||
1135 | WARN_ON(channel->rx_pkt_n_frags); | |
1136 | } | |
1137 | } | |
1138 | ||
1139 | void efx_stop_channels(struct efx_nic *efx) | |
1140 | { | |
1141 | struct efx_tx_queue *tx_queue; | |
1142 | struct efx_rx_queue *rx_queue; | |
1143 | struct efx_channel *channel; | |
b5775b47 | 1144 | int rc = 0; |
e20ba5b1 AM |
1145 | |
1146 | /* Stop RX refill */ | |
1147 | efx_for_each_channel(channel, efx) { | |
1148 | efx_for_each_channel_rx_queue(rx_queue, channel) | |
1149 | rx_queue->refill_enabled = false; | |
1150 | } | |
1151 | ||
1152 | efx_for_each_channel(channel, efx) { | |
1153 | /* RX packet processing is pipelined, so wait for the | |
1154 | * NAPI handler to complete. At least event queue 0 | |
1155 | * might be kept active by non-data events, so don't | |
1156 | * use napi_synchronize() but actually disable NAPI | |
1157 | * temporarily. | |
1158 | */ | |
1159 | if (efx_channel_has_rx_queue(channel)) { | |
1160 | efx_stop_eventq(channel); | |
1161 | efx_start_eventq(channel); | |
1162 | } | |
1163 | } | |
1164 | ||
b5775b47 AM |
1165 | if (efx->type->fini_dmaq) |
1166 | rc = efx->type->fini_dmaq(efx); | |
1167 | ||
e20ba5b1 AM |
1168 | if (rc) { |
1169 | netif_err(efx, drv, efx->net_dev, "failed to flush queues\n"); | |
1170 | } else { | |
1171 | netif_dbg(efx, drv, efx->net_dev, | |
1172 | "successfully flushed all queues\n"); | |
1173 | } | |
1174 | ||
1175 | efx_for_each_channel(channel, efx) { | |
1176 | efx_for_each_channel_rx_queue(rx_queue, channel) | |
1177 | efx_fini_rx_queue(rx_queue); | |
f9cac93e | 1178 | efx_for_each_channel_tx_queue(tx_queue, channel) |
e20ba5b1 AM |
1179 | efx_fini_tx_queue(tx_queue); |
1180 | } | |
1181 | } | |
1182 | ||
768fd266 AM |
1183 | /************************************************************************** |
1184 | * | |
1185 | * NAPI interface | |
1186 | * | |
1187 | *************************************************************************/ | |
1188 | ||
1189 | /* Process channel's event queue | |
1190 | * | |
1191 | * This function is responsible for processing the event queue of a | |
1192 | * single channel. The caller must guarantee that this function will | |
1193 | * never be concurrently called more than once on the same channel, | |
1194 | * though different channels may be being processed concurrently. | |
1195 | */ | |
1196 | static int efx_process_channel(struct efx_channel *channel, int budget) | |
1197 | { | |
1198 | struct efx_tx_queue *tx_queue; | |
1199 | struct list_head rx_list; | |
1200 | int spent; | |
1201 | ||
1202 | if (unlikely(!channel->enabled)) | |
1203 | return 0; | |
1204 | ||
1205 | /* Prepare the batch receive list */ | |
1206 | EFX_WARN_ON_PARANOID(channel->rx_list != NULL); | |
1207 | INIT_LIST_HEAD(&rx_list); | |
1208 | channel->rx_list = &rx_list; | |
1209 | ||
1210 | efx_for_each_channel_tx_queue(tx_queue, channel) { | |
1211 | tx_queue->pkts_compl = 0; | |
1212 | tx_queue->bytes_compl = 0; | |
1213 | } | |
1214 | ||
1215 | spent = efx_nic_process_eventq(channel, budget); | |
1216 | if (spent && efx_channel_has_rx_queue(channel)) { | |
1217 | struct efx_rx_queue *rx_queue = | |
1218 | efx_channel_get_rx_queue(channel); | |
1219 | ||
1220 | efx_rx_flush_packet(channel); | |
1221 | efx_fast_push_rx_descriptors(rx_queue, true); | |
1222 | } | |
1223 | ||
1224 | /* Update BQL */ | |
1225 | efx_for_each_channel_tx_queue(tx_queue, channel) { | |
1226 | if (tx_queue->bytes_compl) { | |
1227 | netdev_tx_completed_queue(tx_queue->core_txq, | |
1228 | tx_queue->pkts_compl, | |
1229 | tx_queue->bytes_compl); | |
1230 | } | |
1231 | } | |
1232 | ||
1233 | /* Receive any packets we queued up */ | |
1234 | netif_receive_skb_list(channel->rx_list); | |
1235 | channel->rx_list = NULL; | |
1236 | ||
1237 | return spent; | |
1238 | } | |
1239 | ||
1240 | static void efx_update_irq_mod(struct efx_nic *efx, struct efx_channel *channel) | |
1241 | { | |
1242 | int step = efx->irq_mod_step_us; | |
1243 | ||
1244 | if (channel->irq_mod_score < irq_adapt_low_thresh) { | |
1245 | if (channel->irq_moderation_us > step) { | |
1246 | channel->irq_moderation_us -= step; | |
1247 | efx->type->push_irq_moderation(channel); | |
1248 | } | |
1249 | } else if (channel->irq_mod_score > irq_adapt_high_thresh) { | |
1250 | if (channel->irq_moderation_us < | |
1251 | efx->irq_rx_moderation_us) { | |
1252 | channel->irq_moderation_us += step; | |
1253 | efx->type->push_irq_moderation(channel); | |
1254 | } | |
1255 | } | |
1256 | ||
1257 | channel->irq_count = 0; | |
1258 | channel->irq_mod_score = 0; | |
1259 | } | |
1260 | ||
1261 | /* NAPI poll handler | |
1262 | * | |
1263 | * NAPI guarantees serialisation of polls of the same device, which | |
1264 | * provides the guarantee required by efx_process_channel(). | |
1265 | */ | |
1266 | static int efx_poll(struct napi_struct *napi, int budget) | |
1267 | { | |
1268 | struct efx_channel *channel = | |
1269 | container_of(napi, struct efx_channel, napi_str); | |
1270 | struct efx_nic *efx = channel->efx; | |
b7683155 EC |
1271 | #ifdef CONFIG_RFS_ACCEL |
1272 | unsigned int time; | |
1273 | #endif | |
768fd266 AM |
1274 | int spent; |
1275 | ||
1276 | netif_vdbg(efx, intr, efx->net_dev, | |
1277 | "channel %d NAPI poll executing on CPU %d\n", | |
1278 | channel->channel, raw_smp_processor_id()); | |
1279 | ||
1280 | spent = efx_process_channel(channel, budget); | |
1281 | ||
1282 | xdp_do_flush_map(); | |
1283 | ||
1284 | if (spent < budget) { | |
1285 | if (efx_channel_has_rx_queue(channel) && | |
1286 | efx->irq_rx_adaptive && | |
1287 | unlikely(++channel->irq_count == 1000)) { | |
1288 | efx_update_irq_mod(efx, channel); | |
1289 | } | |
1290 | ||
1291 | #ifdef CONFIG_RFS_ACCEL | |
1292 | /* Perhaps expire some ARFS filters */ | |
b7683155 EC |
1293 | time = jiffies - channel->rfs_last_expiry; |
1294 | /* Would our quota be >= 20? */ | |
1295 | if (channel->rfs_filter_count * time >= 600 * HZ) | |
1296 | mod_delayed_work(system_wq, &channel->filter_work, 0); | |
768fd266 AM |
1297 | #endif |
1298 | ||
1299 | /* There is no race here; although napi_disable() will | |
1300 | * only wait for napi_complete(), this isn't a problem | |
1301 | * since efx_nic_eventq_read_ack() will have no effect if | |
1302 | * interrupts have already been disabled. | |
1303 | */ | |
1304 | if (napi_complete_done(napi, spent)) | |
1305 | efx_nic_eventq_read_ack(channel); | |
1306 | } | |
1307 | ||
1308 | return spent; | |
1309 | } | |
1310 | ||
1311 | void efx_init_napi_channel(struct efx_channel *channel) | |
1312 | { | |
1313 | struct efx_nic *efx = channel->efx; | |
1314 | ||
1315 | channel->napi_dev = efx->net_dev; | |
b48b89f9 | 1316 | netif_napi_add(channel->napi_dev, &channel->napi_str, efx_poll); |
768fd266 AM |
1317 | } |
1318 | ||
1319 | void efx_init_napi(struct efx_nic *efx) | |
1320 | { | |
1321 | struct efx_channel *channel; | |
1322 | ||
1323 | efx_for_each_channel(channel, efx) | |
1324 | efx_init_napi_channel(channel); | |
1325 | } | |
1326 | ||
1327 | void efx_fini_napi_channel(struct efx_channel *channel) | |
1328 | { | |
1329 | if (channel->napi_dev) | |
1330 | netif_napi_del(&channel->napi_str); | |
1331 | ||
1332 | channel->napi_dev = NULL; | |
1333 | } | |
1334 | ||
1335 | void efx_fini_napi(struct efx_nic *efx) | |
1336 | { | |
1337 | struct efx_channel *channel; | |
1338 | ||
1339 | efx_for_each_channel(channel, efx) | |
1340 | efx_fini_napi_channel(channel); | |
1341 | } | |
54fccfdd MH |
1342 | |
1343 | /*************** | |
1344 | * Housekeeping | |
1345 | ***************/ | |
1346 | ||
1347 | static int efx_channel_dummy_op_int(struct efx_channel *channel) | |
1348 | { | |
1349 | return 0; | |
1350 | } | |
1351 | ||
1352 | void efx_channel_dummy_op_void(struct efx_channel *channel) | |
1353 | { | |
1354 | } | |
1355 | ||
1356 | static const struct efx_channel_type efx_default_channel_type = { | |
1357 | .pre_probe = efx_channel_dummy_op_int, | |
1358 | .post_remove = efx_channel_dummy_op_void, | |
1359 | .get_name = efx_get_channel_name, | |
1360 | .copy = efx_copy_channel, | |
1361 | .want_txqs = efx_default_channel_want_txqs, | |
1362 | .keep_eventq = false, | |
1363 | .want_pio = true, | |
1364 | }; |