Commit | Line | Data |
---|---|---|
1751cc36 AM |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /**************************************************************************** | |
3 | * Driver for Solarflare network controllers and boards | |
4 | * Copyright 2018 Solarflare Communications Inc. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms of the GNU General Public License version 2 as published | |
8 | * by the Free Software Foundation, incorporated herein by reference. | |
9 | */ | |
10 | ||
11 | #include "net_driver.h" | |
12 | #include <linux/module.h> | |
3d95b884 | 13 | #include <linux/iommu.h> |
1751cc36 AM |
14 | #include "efx.h" |
15 | #include "nic.h" | |
16 | #include "rx_common.h" | |
17 | ||
18 | /* This is the percentage fill level below which new RX descriptors | |
19 | * will be added to the RX descriptor ring. | |
20 | */ | |
21 | static unsigned int rx_refill_threshold; | |
22 | module_param(rx_refill_threshold, uint, 0444); | |
23 | MODULE_PARM_DESC(rx_refill_threshold, | |
24 | "RX descriptor ring refill threshold (%)"); | |
25 | ||
3d95b884 AM |
26 | /* Number of RX buffers to recycle pages for. When creating the RX page recycle |
27 | * ring, this number is divided by the number of buffers per page to calculate | |
28 | * the number of pages to store in the RX page recycle ring. | |
29 | */ | |
30 | #define EFX_RECYCLE_RING_SIZE_IOMMU 4096 | |
31 | #define EFX_RECYCLE_RING_SIZE_NOIOMMU (2 * EFX_RX_PREFERRED_BATCH) | |
32 | ||
1751cc36 AM |
33 | /* RX maximum head room required. |
34 | * | |
35 | * This must be at least 1 to prevent overflow, plus one packet-worth | |
36 | * to allow pipelined receives. | |
37 | */ | |
38 | #define EFX_RXD_HEAD_ROOM (1 + EFX_RX_MAX_FRAGS) | |
39 | ||
3d95b884 AM |
40 | /* Check the RX page recycle ring for a page that can be reused. */ |
41 | static struct page *efx_reuse_page(struct efx_rx_queue *rx_queue) | |
42 | { | |
43 | struct efx_nic *efx = rx_queue->efx; | |
44 | struct efx_rx_page_state *state; | |
45 | unsigned int index; | |
46 | struct page *page; | |
47 | ||
48 | index = rx_queue->page_remove & rx_queue->page_ptr_mask; | |
49 | page = rx_queue->page_ring[index]; | |
50 | if (page == NULL) | |
51 | return NULL; | |
52 | ||
53 | rx_queue->page_ring[index] = NULL; | |
54 | /* page_remove cannot exceed page_add. */ | |
55 | if (rx_queue->page_remove != rx_queue->page_add) | |
56 | ++rx_queue->page_remove; | |
57 | ||
58 | /* If page_count is 1 then we hold the only reference to this page. */ | |
59 | if (page_count(page) == 1) { | |
60 | ++rx_queue->page_recycle_count; | |
61 | return page; | |
62 | } else { | |
63 | state = page_address(page); | |
64 | dma_unmap_page(&efx->pci_dev->dev, state->dma_addr, | |
65 | PAGE_SIZE << efx->rx_buffer_order, | |
66 | DMA_FROM_DEVICE); | |
67 | put_page(page); | |
68 | ++rx_queue->page_recycle_failed; | |
69 | } | |
70 | ||
71 | return NULL; | |
72 | } | |
73 | ||
74 | /* Attempt to recycle the page if there is an RX recycle ring; the page can | |
75 | * only be added if this is the final RX buffer, to prevent pages being used in | |
76 | * the descriptor ring and appearing in the recycle ring simultaneously. | |
77 | */ | |
78 | static void efx_recycle_rx_page(struct efx_channel *channel, | |
79 | struct efx_rx_buffer *rx_buf) | |
80 | { | |
81 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); | |
82 | struct efx_nic *efx = rx_queue->efx; | |
83 | struct page *page = rx_buf->page; | |
84 | unsigned int index; | |
85 | ||
86 | /* Only recycle the page after processing the final buffer. */ | |
87 | if (!(rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE)) | |
88 | return; | |
89 | ||
90 | index = rx_queue->page_add & rx_queue->page_ptr_mask; | |
91 | if (rx_queue->page_ring[index] == NULL) { | |
92 | unsigned int read_index = rx_queue->page_remove & | |
93 | rx_queue->page_ptr_mask; | |
94 | ||
95 | /* The next slot in the recycle ring is available, but | |
96 | * increment page_remove if the read pointer currently | |
97 | * points here. | |
98 | */ | |
99 | if (read_index == index) | |
100 | ++rx_queue->page_remove; | |
101 | rx_queue->page_ring[index] = page; | |
102 | ++rx_queue->page_add; | |
103 | return; | |
104 | } | |
105 | ++rx_queue->page_recycle_full; | |
106 | efx_unmap_rx_buffer(efx, rx_buf); | |
107 | put_page(rx_buf->page); | |
108 | } | |
109 | ||
110 | /* Recycle the pages that are used by buffers that have just been received. */ | |
111 | void efx_recycle_rx_pages(struct efx_channel *channel, | |
112 | struct efx_rx_buffer *rx_buf, | |
113 | unsigned int n_frags) | |
114 | { | |
115 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); | |
116 | ||
117 | do { | |
118 | efx_recycle_rx_page(channel, rx_buf); | |
119 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); | |
120 | } while (--n_frags); | |
121 | } | |
122 | ||
123 | void efx_discard_rx_packet(struct efx_channel *channel, | |
124 | struct efx_rx_buffer *rx_buf, | |
125 | unsigned int n_frags) | |
126 | { | |
127 | struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); | |
128 | ||
129 | efx_recycle_rx_pages(channel, rx_buf, n_frags); | |
130 | ||
131 | efx_free_rx_buffers(rx_queue, rx_buf, n_frags); | |
132 | } | |
133 | ||
134 | static void efx_init_rx_recycle_ring(struct efx_rx_queue *rx_queue) | |
135 | { | |
136 | unsigned int bufs_in_recycle_ring, page_ring_size; | |
137 | struct efx_nic *efx = rx_queue->efx; | |
138 | ||
139 | /* Set the RX recycle ring size */ | |
140 | #ifdef CONFIG_PPC64 | |
141 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU; | |
142 | #else | |
143 | if (iommu_present(&pci_bus_type)) | |
144 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_IOMMU; | |
145 | else | |
146 | bufs_in_recycle_ring = EFX_RECYCLE_RING_SIZE_NOIOMMU; | |
147 | #endif /* CONFIG_PPC64 */ | |
148 | ||
149 | page_ring_size = roundup_pow_of_two(bufs_in_recycle_ring / | |
150 | efx->rx_bufs_per_page); | |
151 | rx_queue->page_ring = kcalloc(page_ring_size, | |
152 | sizeof(*rx_queue->page_ring), GFP_KERNEL); | |
153 | rx_queue->page_ptr_mask = page_ring_size - 1; | |
154 | } | |
155 | ||
156 | static void efx_fini_rx_recycle_ring(struct efx_rx_queue *rx_queue) | |
157 | { | |
158 | struct efx_nic *efx = rx_queue->efx; | |
159 | int i; | |
160 | ||
161 | /* Unmap and release the pages in the recycle ring. Remove the ring. */ | |
162 | for (i = 0; i <= rx_queue->page_ptr_mask; i++) { | |
163 | struct page *page = rx_queue->page_ring[i]; | |
164 | struct efx_rx_page_state *state; | |
165 | ||
166 | if (page == NULL) | |
167 | continue; | |
168 | ||
169 | state = page_address(page); | |
170 | dma_unmap_page(&efx->pci_dev->dev, state->dma_addr, | |
171 | PAGE_SIZE << efx->rx_buffer_order, | |
172 | DMA_FROM_DEVICE); | |
173 | put_page(page); | |
174 | } | |
175 | kfree(rx_queue->page_ring); | |
176 | rx_queue->page_ring = NULL; | |
177 | } | |
178 | ||
1751cc36 AM |
179 | static void efx_fini_rx_buffer(struct efx_rx_queue *rx_queue, |
180 | struct efx_rx_buffer *rx_buf) | |
181 | { | |
182 | /* Release the page reference we hold for the buffer. */ | |
183 | if (rx_buf->page) | |
184 | put_page(rx_buf->page); | |
185 | ||
186 | /* If this is the last buffer in a page, unmap and free it. */ | |
187 | if (rx_buf->flags & EFX_RX_BUF_LAST_IN_PAGE) { | |
188 | efx_unmap_rx_buffer(rx_queue->efx, rx_buf); | |
189 | efx_free_rx_buffers(rx_queue, rx_buf, 1); | |
190 | } | |
191 | rx_buf->page = NULL; | |
192 | } | |
193 | ||
194 | int efx_probe_rx_queue(struct efx_rx_queue *rx_queue) | |
195 | { | |
196 | struct efx_nic *efx = rx_queue->efx; | |
197 | unsigned int entries; | |
198 | int rc; | |
199 | ||
200 | /* Create the smallest power-of-two aligned ring */ | |
201 | entries = max(roundup_pow_of_two(efx->rxq_entries), EFX_MIN_DMAQ_SIZE); | |
202 | EFX_WARN_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE); | |
203 | rx_queue->ptr_mask = entries - 1; | |
204 | ||
205 | netif_dbg(efx, probe, efx->net_dev, | |
206 | "creating RX queue %d size %#x mask %#x\n", | |
207 | efx_rx_queue_index(rx_queue), efx->rxq_entries, | |
208 | rx_queue->ptr_mask); | |
209 | ||
210 | /* Allocate RX buffers */ | |
211 | rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer), | |
212 | GFP_KERNEL); | |
213 | if (!rx_queue->buffer) | |
214 | return -ENOMEM; | |
215 | ||
216 | rc = efx_nic_probe_rx(rx_queue); | |
217 | if (rc) { | |
218 | kfree(rx_queue->buffer); | |
219 | rx_queue->buffer = NULL; | |
220 | } | |
221 | ||
222 | return rc; | |
223 | } | |
224 | ||
225 | void efx_init_rx_queue(struct efx_rx_queue *rx_queue) | |
226 | { | |
227 | unsigned int max_fill, trigger, max_trigger; | |
228 | struct efx_nic *efx = rx_queue->efx; | |
229 | int rc = 0; | |
230 | ||
231 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, | |
232 | "initialising RX queue %d\n", efx_rx_queue_index(rx_queue)); | |
233 | ||
234 | /* Initialise ptr fields */ | |
235 | rx_queue->added_count = 0; | |
236 | rx_queue->notified_count = 0; | |
237 | rx_queue->removed_count = 0; | |
238 | rx_queue->min_fill = -1U; | |
239 | efx_init_rx_recycle_ring(rx_queue); | |
240 | ||
241 | rx_queue->page_remove = 0; | |
242 | rx_queue->page_add = rx_queue->page_ptr_mask + 1; | |
243 | rx_queue->page_recycle_count = 0; | |
244 | rx_queue->page_recycle_failed = 0; | |
245 | rx_queue->page_recycle_full = 0; | |
246 | ||
247 | /* Initialise limit fields */ | |
248 | max_fill = efx->rxq_entries - EFX_RXD_HEAD_ROOM; | |
249 | max_trigger = | |
250 | max_fill - efx->rx_pages_per_batch * efx->rx_bufs_per_page; | |
251 | if (rx_refill_threshold != 0) { | |
252 | trigger = max_fill * min(rx_refill_threshold, 100U) / 100U; | |
253 | if (trigger > max_trigger) | |
254 | trigger = max_trigger; | |
255 | } else { | |
256 | trigger = max_trigger; | |
257 | } | |
258 | ||
259 | rx_queue->max_fill = max_fill; | |
260 | rx_queue->fast_fill_trigger = trigger; | |
261 | rx_queue->refill_enabled = true; | |
262 | ||
263 | /* Initialise XDP queue information */ | |
264 | rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev, | |
265 | rx_queue->core_index); | |
266 | ||
267 | if (rc) { | |
268 | netif_err(efx, rx_err, efx->net_dev, | |
269 | "Failure to initialise XDP queue information rc=%d\n", | |
270 | rc); | |
271 | efx->xdp_rxq_info_failed = true; | |
272 | } else { | |
273 | rx_queue->xdp_rxq_info_valid = true; | |
274 | } | |
275 | ||
276 | /* Set up RX descriptor ring */ | |
277 | efx_nic_init_rx(rx_queue); | |
278 | } | |
279 | ||
280 | void efx_fini_rx_queue(struct efx_rx_queue *rx_queue) | |
281 | { | |
1751cc36 AM |
282 | struct efx_rx_buffer *rx_buf; |
283 | int i; | |
284 | ||
285 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, | |
286 | "shutting down RX queue %d\n", efx_rx_queue_index(rx_queue)); | |
287 | ||
288 | del_timer_sync(&rx_queue->slow_fill); | |
289 | ||
290 | /* Release RX buffers from the current read ptr to the write ptr */ | |
291 | if (rx_queue->buffer) { | |
292 | for (i = rx_queue->removed_count; i < rx_queue->added_count; | |
293 | i++) { | |
294 | unsigned int index = i & rx_queue->ptr_mask; | |
295 | ||
296 | rx_buf = efx_rx_buffer(rx_queue, index); | |
297 | efx_fini_rx_buffer(rx_queue, rx_buf); | |
298 | } | |
299 | } | |
300 | ||
3d95b884 | 301 | efx_fini_rx_recycle_ring(rx_queue); |
1751cc36 AM |
302 | |
303 | if (rx_queue->xdp_rxq_info_valid) | |
304 | xdp_rxq_info_unreg(&rx_queue->xdp_rxq_info); | |
305 | ||
306 | rx_queue->xdp_rxq_info_valid = false; | |
307 | } | |
308 | ||
309 | void efx_remove_rx_queue(struct efx_rx_queue *rx_queue) | |
310 | { | |
311 | netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, | |
312 | "destroying RX queue %d\n", efx_rx_queue_index(rx_queue)); | |
313 | ||
314 | efx_nic_remove_rx(rx_queue); | |
315 | ||
316 | kfree(rx_queue->buffer); | |
317 | rx_queue->buffer = NULL; | |
318 | } | |
319 | ||
320 | /* Unmap a DMA-mapped page. This function is only called for the final RX | |
321 | * buffer in a page. | |
322 | */ | |
323 | void efx_unmap_rx_buffer(struct efx_nic *efx, | |
324 | struct efx_rx_buffer *rx_buf) | |
325 | { | |
326 | struct page *page = rx_buf->page; | |
327 | ||
328 | if (page) { | |
329 | struct efx_rx_page_state *state = page_address(page); | |
330 | ||
331 | dma_unmap_page(&efx->pci_dev->dev, | |
332 | state->dma_addr, | |
333 | PAGE_SIZE << efx->rx_buffer_order, | |
334 | DMA_FROM_DEVICE); | |
335 | } | |
336 | } | |
337 | ||
338 | void efx_free_rx_buffers(struct efx_rx_queue *rx_queue, | |
339 | struct efx_rx_buffer *rx_buf, | |
340 | unsigned int num_bufs) | |
341 | { | |
342 | do { | |
343 | if (rx_buf->page) { | |
344 | put_page(rx_buf->page); | |
345 | rx_buf->page = NULL; | |
346 | } | |
347 | rx_buf = efx_rx_buf_next(rx_queue, rx_buf); | |
348 | } while (--num_bufs); | |
349 | } | |
350 | ||
351 | void efx_rx_slow_fill(struct timer_list *t) | |
352 | { | |
353 | struct efx_rx_queue *rx_queue = from_timer(rx_queue, t, slow_fill); | |
354 | ||
355 | /* Post an event to cause NAPI to run and refill the queue */ | |
356 | efx_nic_generate_fill_event(rx_queue); | |
357 | ++rx_queue->slow_fill_count; | |
358 | } | |
359 | ||
360 | void efx_schedule_slow_fill(struct efx_rx_queue *rx_queue) | |
361 | { | |
362 | mod_timer(&rx_queue->slow_fill, jiffies + msecs_to_jiffies(10)); | |
363 | } | |
364 | ||
365 | /* efx_init_rx_buffers - create EFX_RX_BATCH page-based RX buffers | |
366 | * | |
367 | * @rx_queue: Efx RX queue | |
368 | * | |
369 | * This allocates a batch of pages, maps them for DMA, and populates | |
370 | * struct efx_rx_buffers for each one. Return a negative error code or | |
371 | * 0 on success. If a single page can be used for multiple buffers, | |
372 | * then the page will either be inserted fully, or not at all. | |
373 | */ | |
374 | static int efx_init_rx_buffers(struct efx_rx_queue *rx_queue, bool atomic) | |
375 | { | |
376 | unsigned int page_offset, index, count; | |
377 | struct efx_nic *efx = rx_queue->efx; | |
378 | struct efx_rx_page_state *state; | |
379 | struct efx_rx_buffer *rx_buf; | |
380 | dma_addr_t dma_addr; | |
381 | struct page *page; | |
382 | ||
383 | count = 0; | |
384 | do { | |
385 | page = efx_reuse_page(rx_queue); | |
386 | if (page == NULL) { | |
387 | page = alloc_pages(__GFP_COMP | | |
388 | (atomic ? GFP_ATOMIC : GFP_KERNEL), | |
389 | efx->rx_buffer_order); | |
390 | if (unlikely(page == NULL)) | |
391 | return -ENOMEM; | |
392 | dma_addr = | |
393 | dma_map_page(&efx->pci_dev->dev, page, 0, | |
394 | PAGE_SIZE << efx->rx_buffer_order, | |
395 | DMA_FROM_DEVICE); | |
396 | if (unlikely(dma_mapping_error(&efx->pci_dev->dev, | |
397 | dma_addr))) { | |
398 | __free_pages(page, efx->rx_buffer_order); | |
399 | return -EIO; | |
400 | } | |
401 | state = page_address(page); | |
402 | state->dma_addr = dma_addr; | |
403 | } else { | |
404 | state = page_address(page); | |
405 | dma_addr = state->dma_addr; | |
406 | } | |
407 | ||
408 | dma_addr += sizeof(struct efx_rx_page_state); | |
409 | page_offset = sizeof(struct efx_rx_page_state); | |
410 | ||
411 | do { | |
412 | index = rx_queue->added_count & rx_queue->ptr_mask; | |
413 | rx_buf = efx_rx_buffer(rx_queue, index); | |
414 | rx_buf->dma_addr = dma_addr + efx->rx_ip_align + | |
86e85bf6 | 415 | EFX_XDP_HEADROOM; |
1751cc36 AM |
416 | rx_buf->page = page; |
417 | rx_buf->page_offset = page_offset + efx->rx_ip_align + | |
86e85bf6 | 418 | EFX_XDP_HEADROOM; |
1751cc36 AM |
419 | rx_buf->len = efx->rx_dma_len; |
420 | rx_buf->flags = 0; | |
421 | ++rx_queue->added_count; | |
422 | get_page(page); | |
423 | dma_addr += efx->rx_page_buf_step; | |
424 | page_offset += efx->rx_page_buf_step; | |
425 | } while (page_offset + efx->rx_page_buf_step <= PAGE_SIZE); | |
426 | ||
427 | rx_buf->flags = EFX_RX_BUF_LAST_IN_PAGE; | |
428 | } while (++count < efx->rx_pages_per_batch); | |
429 | ||
430 | return 0; | |
431 | } | |
432 | ||
433 | void efx_rx_config_page_split(struct efx_nic *efx) | |
434 | { | |
435 | efx->rx_page_buf_step = ALIGN(efx->rx_dma_len + efx->rx_ip_align + | |
86e85bf6 | 436 | EFX_XDP_HEADROOM + EFX_XDP_TAILROOM, |
1751cc36 AM |
437 | EFX_RX_BUF_ALIGNMENT); |
438 | efx->rx_bufs_per_page = efx->rx_buffer_order ? 1 : | |
439 | ((PAGE_SIZE - sizeof(struct efx_rx_page_state)) / | |
440 | efx->rx_page_buf_step); | |
441 | efx->rx_buffer_truesize = (PAGE_SIZE << efx->rx_buffer_order) / | |
442 | efx->rx_bufs_per_page; | |
443 | efx->rx_pages_per_batch = DIV_ROUND_UP(EFX_RX_PREFERRED_BATCH, | |
444 | efx->rx_bufs_per_page); | |
445 | } | |
446 | ||
447 | /* efx_fast_push_rx_descriptors - push new RX descriptors quickly | |
448 | * @rx_queue: RX descriptor queue | |
449 | * | |
450 | * This will aim to fill the RX descriptor queue up to | |
451 | * @rx_queue->@max_fill. If there is insufficient atomic | |
452 | * memory to do so, a slow fill will be scheduled. | |
453 | * | |
454 | * The caller must provide serialisation (none is used here). In practise, | |
455 | * this means this function must run from the NAPI handler, or be called | |
456 | * when NAPI is disabled. | |
457 | */ | |
458 | void efx_fast_push_rx_descriptors(struct efx_rx_queue *rx_queue, bool atomic) | |
459 | { | |
460 | struct efx_nic *efx = rx_queue->efx; | |
461 | unsigned int fill_level, batch_size; | |
462 | int space, rc = 0; | |
463 | ||
464 | if (!rx_queue->refill_enabled) | |
465 | return; | |
466 | ||
467 | /* Calculate current fill level, and exit if we don't need to fill */ | |
468 | fill_level = (rx_queue->added_count - rx_queue->removed_count); | |
469 | EFX_WARN_ON_ONCE_PARANOID(fill_level > rx_queue->efx->rxq_entries); | |
470 | if (fill_level >= rx_queue->fast_fill_trigger) | |
471 | goto out; | |
472 | ||
473 | /* Record minimum fill level */ | |
474 | if (unlikely(fill_level < rx_queue->min_fill)) { | |
475 | if (fill_level) | |
476 | rx_queue->min_fill = fill_level; | |
477 | } | |
478 | ||
479 | batch_size = efx->rx_pages_per_batch * efx->rx_bufs_per_page; | |
480 | space = rx_queue->max_fill - fill_level; | |
481 | EFX_WARN_ON_ONCE_PARANOID(space < batch_size); | |
482 | ||
483 | netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, | |
484 | "RX queue %d fast-filling descriptor ring from" | |
485 | " level %d to level %d\n", | |
486 | efx_rx_queue_index(rx_queue), fill_level, | |
487 | rx_queue->max_fill); | |
488 | ||
489 | do { | |
490 | rc = efx_init_rx_buffers(rx_queue, atomic); | |
491 | if (unlikely(rc)) { | |
492 | /* Ensure that we don't leave the rx queue empty */ | |
493 | efx_schedule_slow_fill(rx_queue); | |
494 | goto out; | |
495 | } | |
496 | } while ((space -= batch_size) >= batch_size); | |
497 | ||
498 | netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, | |
499 | "RX queue %d fast-filled descriptor ring " | |
500 | "to level %d\n", efx_rx_queue_index(rx_queue), | |
501 | rx_queue->added_count - rx_queue->removed_count); | |
502 | ||
503 | out: | |
504 | if (rx_queue->notified_count != rx_queue->added_count) | |
505 | efx_nic_notify_rx_desc(rx_queue); | |
506 | } | |
3d95b884 AM |
507 | |
508 | /* Pass a received packet up through GRO. GRO can handle pages | |
509 | * regardless of checksum state and skbs with a good checksum. | |
510 | */ | |
511 | void | |
512 | efx_rx_packet_gro(struct efx_channel *channel, struct efx_rx_buffer *rx_buf, | |
513 | unsigned int n_frags, u8 *eh) | |
514 | { | |
515 | struct napi_struct *napi = &channel->napi_str; | |
516 | struct efx_nic *efx = channel->efx; | |
517 | struct sk_buff *skb; | |
518 | ||
519 | skb = napi_get_frags(napi); | |
520 | if (unlikely(!skb)) { | |
521 | struct efx_rx_queue *rx_queue; | |
522 | ||
523 | rx_queue = efx_channel_get_rx_queue(channel); | |
524 | efx_free_rx_buffers(rx_queue, rx_buf, n_frags); | |
525 | return; | |
526 | } | |
527 | ||
528 | if (efx->net_dev->features & NETIF_F_RXHASH) | |
529 | skb_set_hash(skb, efx_rx_buf_hash(efx, eh), | |
530 | PKT_HASH_TYPE_L3); | |
531 | skb->ip_summed = ((rx_buf->flags & EFX_RX_PKT_CSUMMED) ? | |
532 | CHECKSUM_UNNECESSARY : CHECKSUM_NONE); | |
533 | skb->csum_level = !!(rx_buf->flags & EFX_RX_PKT_CSUM_LEVEL); | |
534 | ||
535 | for (;;) { | |
536 | skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, | |
537 | rx_buf->page, rx_buf->page_offset, | |
538 | rx_buf->len); | |
539 | rx_buf->page = NULL; | |
540 | skb->len += rx_buf->len; | |
541 | if (skb_shinfo(skb)->nr_frags == n_frags) | |
542 | break; | |
543 | ||
544 | rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); | |
545 | } | |
546 | ||
547 | skb->data_len = skb->len; | |
548 | skb->truesize += n_frags * efx->rx_buffer_truesize; | |
549 | ||
550 | skb_record_rx_queue(skb, channel->rx_queue.core_index); | |
551 | ||
552 | napi_gro_frags(napi); | |
553 | } | |
960f1627 AM |
554 | |
555 | /* RSS contexts. We're using linked lists and crappy O(n) algorithms, because | |
556 | * (a) this is an infrequent control-plane operation and (b) n is small (max 64) | |
557 | */ | |
558 | struct efx_rss_context *efx_alloc_rss_context_entry(struct efx_nic *efx) | |
559 | { | |
560 | struct list_head *head = &efx->rss_context.list; | |
561 | struct efx_rss_context *ctx, *new; | |
562 | u32 id = 1; /* Don't use zero, that refers to the master RSS context */ | |
563 | ||
564 | WARN_ON(!mutex_is_locked(&efx->rss_lock)); | |
565 | ||
566 | /* Search for first gap in the numbering */ | |
567 | list_for_each_entry(ctx, head, list) { | |
568 | if (ctx->user_id != id) | |
569 | break; | |
570 | id++; | |
571 | /* Check for wrap. If this happens, we have nearly 2^32 | |
572 | * allocated RSS contexts, which seems unlikely. | |
573 | */ | |
574 | if (WARN_ON_ONCE(!id)) | |
575 | return NULL; | |
576 | } | |
577 | ||
578 | /* Create the new entry */ | |
579 | new = kmalloc(sizeof(*new), GFP_KERNEL); | |
580 | if (!new) | |
581 | return NULL; | |
f7226e0f | 582 | new->context_id = EFX_MCDI_RSS_CONTEXT_INVALID; |
960f1627 AM |
583 | new->rx_hash_udp_4tuple = false; |
584 | ||
585 | /* Insert the new entry into the gap */ | |
586 | new->user_id = id; | |
587 | list_add_tail(&new->list, &ctx->list); | |
588 | return new; | |
589 | } | |
590 | ||
591 | struct efx_rss_context *efx_find_rss_context_entry(struct efx_nic *efx, u32 id) | |
592 | { | |
593 | struct list_head *head = &efx->rss_context.list; | |
594 | struct efx_rss_context *ctx; | |
595 | ||
596 | WARN_ON(!mutex_is_locked(&efx->rss_lock)); | |
597 | ||
598 | list_for_each_entry(ctx, head, list) | |
599 | if (ctx->user_id == id) | |
600 | return ctx; | |
601 | return NULL; | |
602 | } | |
603 | ||
604 | void efx_free_rss_context_entry(struct efx_rss_context *ctx) | |
605 | { | |
606 | list_del(&ctx->list); | |
607 | kfree(ctx); | |
608 | } | |
609 | ||
610 | void efx_set_default_rx_indir_table(struct efx_nic *efx, | |
611 | struct efx_rss_context *ctx) | |
612 | { | |
613 | size_t i; | |
614 | ||
615 | for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) | |
616 | ctx->rx_indir_table[i] = | |
617 | ethtool_rxfh_indir_default(i, efx->rss_spread); | |
618 | } | |
f7226e0f AM |
619 | |
620 | /** | |
621 | * efx_filter_is_mc_recipient - test whether spec is a multicast recipient | |
622 | * @spec: Specification to test | |
623 | * | |
624 | * Return: %true if the specification is a non-drop RX filter that | |
625 | * matches a local MAC address I/G bit value of 1 or matches a local | |
626 | * IPv4 or IPv6 address value in the respective multicast address | |
627 | * range. Otherwise %false. | |
628 | */ | |
629 | bool efx_filter_is_mc_recipient(const struct efx_filter_spec *spec) | |
630 | { | |
631 | if (!(spec->flags & EFX_FILTER_FLAG_RX) || | |
632 | spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP) | |
633 | return false; | |
634 | ||
635 | if (spec->match_flags & | |
636 | (EFX_FILTER_MATCH_LOC_MAC | EFX_FILTER_MATCH_LOC_MAC_IG) && | |
637 | is_multicast_ether_addr(spec->loc_mac)) | |
638 | return true; | |
639 | ||
640 | if ((spec->match_flags & | |
641 | (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) == | |
642 | (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) { | |
643 | if (spec->ether_type == htons(ETH_P_IP) && | |
644 | ipv4_is_multicast(spec->loc_host[0])) | |
645 | return true; | |
646 | if (spec->ether_type == htons(ETH_P_IPV6) && | |
647 | ((const u8 *)spec->loc_host)[0] == 0xff) | |
648 | return true; | |
649 | } | |
650 | ||
651 | return false; | |
652 | } | |
653 | ||
654 | bool efx_filter_spec_equal(const struct efx_filter_spec *left, | |
655 | const struct efx_filter_spec *right) | |
656 | { | |
657 | if ((left->match_flags ^ right->match_flags) | | |
658 | ((left->flags ^ right->flags) & | |
659 | (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX))) | |
660 | return false; | |
661 | ||
662 | return memcmp(&left->outer_vid, &right->outer_vid, | |
663 | sizeof(struct efx_filter_spec) - | |
664 | offsetof(struct efx_filter_spec, outer_vid)) == 0; | |
665 | } | |
666 | ||
667 | u32 efx_filter_spec_hash(const struct efx_filter_spec *spec) | |
668 | { | |
669 | BUILD_BUG_ON(offsetof(struct efx_filter_spec, outer_vid) & 3); | |
670 | return jhash2((const u32 *)&spec->outer_vid, | |
671 | (sizeof(struct efx_filter_spec) - | |
672 | offsetof(struct efx_filter_spec, outer_vid)) / 4, | |
673 | 0); | |
674 | } | |
675 | ||
676 | #ifdef CONFIG_RFS_ACCEL | |
677 | bool efx_rps_check_rule(struct efx_arfs_rule *rule, unsigned int filter_idx, | |
678 | bool *force) | |
679 | { | |
680 | if (rule->filter_id == EFX_ARFS_FILTER_ID_PENDING) { | |
681 | /* ARFS is currently updating this entry, leave it */ | |
682 | return false; | |
683 | } | |
684 | if (rule->filter_id == EFX_ARFS_FILTER_ID_ERROR) { | |
685 | /* ARFS tried and failed to update this, so it's probably out | |
686 | * of date. Remove the filter and the ARFS rule entry. | |
687 | */ | |
688 | rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING; | |
689 | *force = true; | |
690 | return true; | |
691 | } else if (WARN_ON(rule->filter_id != filter_idx)) { /* can't happen */ | |
692 | /* ARFS has moved on, so old filter is not needed. Since we did | |
693 | * not mark the rule with EFX_ARFS_FILTER_ID_REMOVING, it will | |
694 | * not be removed by efx_rps_hash_del() subsequently. | |
695 | */ | |
696 | *force = true; | |
697 | return true; | |
698 | } | |
699 | /* Remove it iff ARFS wants to. */ | |
700 | return true; | |
701 | } | |
702 | ||
703 | static | |
704 | struct hlist_head *efx_rps_hash_bucket(struct efx_nic *efx, | |
705 | const struct efx_filter_spec *spec) | |
706 | { | |
707 | u32 hash = efx_filter_spec_hash(spec); | |
708 | ||
709 | lockdep_assert_held(&efx->rps_hash_lock); | |
710 | if (!efx->rps_hash_table) | |
711 | return NULL; | |
712 | return &efx->rps_hash_table[hash % EFX_ARFS_HASH_TABLE_SIZE]; | |
713 | } | |
714 | ||
715 | struct efx_arfs_rule *efx_rps_hash_find(struct efx_nic *efx, | |
716 | const struct efx_filter_spec *spec) | |
717 | { | |
718 | struct efx_arfs_rule *rule; | |
719 | struct hlist_head *head; | |
720 | struct hlist_node *node; | |
721 | ||
722 | head = efx_rps_hash_bucket(efx, spec); | |
723 | if (!head) | |
724 | return NULL; | |
725 | hlist_for_each(node, head) { | |
726 | rule = container_of(node, struct efx_arfs_rule, node); | |
727 | if (efx_filter_spec_equal(spec, &rule->spec)) | |
728 | return rule; | |
729 | } | |
730 | return NULL; | |
731 | } | |
732 | ||
733 | struct efx_arfs_rule *efx_rps_hash_add(struct efx_nic *efx, | |
734 | const struct efx_filter_spec *spec, | |
735 | bool *new) | |
736 | { | |
737 | struct efx_arfs_rule *rule; | |
738 | struct hlist_head *head; | |
739 | struct hlist_node *node; | |
740 | ||
741 | head = efx_rps_hash_bucket(efx, spec); | |
742 | if (!head) | |
743 | return NULL; | |
744 | hlist_for_each(node, head) { | |
745 | rule = container_of(node, struct efx_arfs_rule, node); | |
746 | if (efx_filter_spec_equal(spec, &rule->spec)) { | |
747 | *new = false; | |
748 | return rule; | |
749 | } | |
750 | } | |
751 | rule = kmalloc(sizeof(*rule), GFP_ATOMIC); | |
752 | *new = true; | |
753 | if (rule) { | |
754 | memcpy(&rule->spec, spec, sizeof(rule->spec)); | |
755 | hlist_add_head(&rule->node, head); | |
756 | } | |
757 | return rule; | |
758 | } | |
759 | ||
760 | void efx_rps_hash_del(struct efx_nic *efx, const struct efx_filter_spec *spec) | |
761 | { | |
762 | struct efx_arfs_rule *rule; | |
763 | struct hlist_head *head; | |
764 | struct hlist_node *node; | |
765 | ||
766 | head = efx_rps_hash_bucket(efx, spec); | |
767 | if (WARN_ON(!head)) | |
768 | return; | |
769 | hlist_for_each(node, head) { | |
770 | rule = container_of(node, struct efx_arfs_rule, node); | |
771 | if (efx_filter_spec_equal(spec, &rule->spec)) { | |
772 | /* Someone already reused the entry. We know that if | |
773 | * this check doesn't fire (i.e. filter_id == REMOVING) | |
774 | * then the REMOVING mark was put there by our caller, | |
775 | * because caller is holding a lock on filter table and | |
776 | * only holders of that lock set REMOVING. | |
777 | */ | |
778 | if (rule->filter_id != EFX_ARFS_FILTER_ID_REMOVING) | |
779 | return; | |
780 | hlist_del(node); | |
781 | kfree(rule); | |
782 | return; | |
783 | } | |
784 | } | |
785 | /* We didn't find it. */ | |
786 | WARN_ON(1); | |
787 | } | |
788 | #endif | |
789 | ||
790 | int efx_probe_filters(struct efx_nic *efx) | |
791 | { | |
792 | int rc; | |
793 | ||
794 | init_rwsem(&efx->filter_sem); | |
795 | mutex_lock(&efx->mac_lock); | |
796 | down_write(&efx->filter_sem); | |
797 | rc = efx->type->filter_table_probe(efx); | |
798 | if (rc) | |
799 | goto out_unlock; | |
800 | ||
801 | #ifdef CONFIG_RFS_ACCEL | |
802 | if (efx->type->offload_features & NETIF_F_NTUPLE) { | |
803 | struct efx_channel *channel; | |
804 | int i, success = 1; | |
805 | ||
806 | efx_for_each_channel(channel, efx) { | |
807 | channel->rps_flow_id = | |
808 | kcalloc(efx->type->max_rx_ip_filters, | |
809 | sizeof(*channel->rps_flow_id), | |
810 | GFP_KERNEL); | |
811 | if (!channel->rps_flow_id) | |
812 | success = 0; | |
813 | else | |
814 | for (i = 0; | |
815 | i < efx->type->max_rx_ip_filters; | |
816 | ++i) | |
817 | channel->rps_flow_id[i] = | |
818 | RPS_FLOW_ID_INVALID; | |
819 | channel->rfs_expire_index = 0; | |
820 | channel->rfs_filter_count = 0; | |
821 | } | |
822 | ||
823 | if (!success) { | |
824 | efx_for_each_channel(channel, efx) | |
825 | kfree(channel->rps_flow_id); | |
826 | efx->type->filter_table_remove(efx); | |
827 | rc = -ENOMEM; | |
828 | goto out_unlock; | |
829 | } | |
830 | } | |
831 | #endif | |
832 | out_unlock: | |
833 | up_write(&efx->filter_sem); | |
834 | mutex_unlock(&efx->mac_lock); | |
835 | return rc; | |
836 | } | |
837 | ||
838 | void efx_remove_filters(struct efx_nic *efx) | |
839 | { | |
840 | #ifdef CONFIG_RFS_ACCEL | |
841 | struct efx_channel *channel; | |
842 | ||
843 | efx_for_each_channel(channel, efx) { | |
844 | cancel_delayed_work_sync(&channel->filter_work); | |
845 | kfree(channel->rps_flow_id); | |
846 | } | |
847 | #endif | |
848 | down_write(&efx->filter_sem); | |
849 | efx->type->filter_table_remove(efx); | |
850 | up_write(&efx->filter_sem); | |
851 | } |