Commit | Line | Data |
---|---|---|
0a714186 BT |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2018 Intel Corporation. */ | |
3 | ||
4 | #include <linux/bpf_trace.h> | |
5 | #include <net/xdp_sock.h> | |
6 | #include <net/xdp.h> | |
7 | ||
8 | #include "i40e.h" | |
9 | #include "i40e_txrx_common.h" | |
10 | #include "i40e_xsk.h" | |
11 | ||
12 | /** | |
13 | * i40e_alloc_xsk_umems - Allocate an array to store per ring UMEMs | |
14 | * @vsi: Current VSI | |
15 | * | |
16 | * Returns 0 on success, <0 on failure | |
17 | **/ | |
18 | static int i40e_alloc_xsk_umems(struct i40e_vsi *vsi) | |
19 | { | |
20 | if (vsi->xsk_umems) | |
21 | return 0; | |
22 | ||
23 | vsi->num_xsk_umems_used = 0; | |
24 | vsi->num_xsk_umems = vsi->alloc_queue_pairs; | |
25 | vsi->xsk_umems = kcalloc(vsi->num_xsk_umems, sizeof(*vsi->xsk_umems), | |
26 | GFP_KERNEL); | |
27 | if (!vsi->xsk_umems) { | |
28 | vsi->num_xsk_umems = 0; | |
29 | return -ENOMEM; | |
30 | } | |
31 | ||
32 | return 0; | |
33 | } | |
34 | ||
35 | /** | |
529eb362 | 36 | * i40e_add_xsk_umem - Store a UMEM for a certain ring/qid |
0a714186 BT |
37 | * @vsi: Current VSI |
38 | * @umem: UMEM to store | |
39 | * @qid: Ring/qid to associate with the UMEM | |
40 | * | |
41 | * Returns 0 on success, <0 on failure | |
42 | **/ | |
43 | static int i40e_add_xsk_umem(struct i40e_vsi *vsi, struct xdp_umem *umem, | |
44 | u16 qid) | |
45 | { | |
46 | int err; | |
47 | ||
48 | err = i40e_alloc_xsk_umems(vsi); | |
49 | if (err) | |
50 | return err; | |
51 | ||
52 | vsi->xsk_umems[qid] = umem; | |
53 | vsi->num_xsk_umems_used++; | |
54 | ||
55 | return 0; | |
56 | } | |
57 | ||
58 | /** | |
529eb362 | 59 | * i40e_remove_xsk_umem - Remove a UMEM for a certain ring/qid |
0a714186 BT |
60 | * @vsi: Current VSI |
61 | * @qid: Ring/qid associated with the UMEM | |
62 | **/ | |
63 | static void i40e_remove_xsk_umem(struct i40e_vsi *vsi, u16 qid) | |
64 | { | |
65 | vsi->xsk_umems[qid] = NULL; | |
66 | vsi->num_xsk_umems_used--; | |
67 | ||
68 | if (vsi->num_xsk_umems == 0) { | |
69 | kfree(vsi->xsk_umems); | |
70 | vsi->xsk_umems = NULL; | |
71 | vsi->num_xsk_umems = 0; | |
72 | } | |
73 | } | |
74 | ||
75 | /** | |
76 | * i40e_xsk_umem_dma_map - DMA maps all UMEM memory for the netdev | |
77 | * @vsi: Current VSI | |
78 | * @umem: UMEM to DMA map | |
79 | * | |
80 | * Returns 0 on success, <0 on failure | |
81 | **/ | |
82 | static int i40e_xsk_umem_dma_map(struct i40e_vsi *vsi, struct xdp_umem *umem) | |
83 | { | |
84 | struct i40e_pf *pf = vsi->back; | |
85 | struct device *dev; | |
86 | unsigned int i, j; | |
87 | dma_addr_t dma; | |
88 | ||
89 | dev = &pf->pdev->dev; | |
90 | for (i = 0; i < umem->npgs; i++) { | |
91 | dma = dma_map_page_attrs(dev, umem->pgs[i], 0, PAGE_SIZE, | |
92 | DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR); | |
93 | if (dma_mapping_error(dev, dma)) | |
94 | goto out_unmap; | |
95 | ||
96 | umem->pages[i].dma = dma; | |
97 | } | |
98 | ||
99 | return 0; | |
100 | ||
101 | out_unmap: | |
102 | for (j = 0; j < i; j++) { | |
103 | dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE, | |
104 | DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR); | |
105 | umem->pages[i].dma = 0; | |
106 | } | |
107 | ||
108 | return -1; | |
109 | } | |
110 | ||
111 | /** | |
112 | * i40e_xsk_umem_dma_unmap - DMA unmaps all UMEM memory for the netdev | |
113 | * @vsi: Current VSI | |
114 | * @umem: UMEM to DMA map | |
115 | **/ | |
116 | static void i40e_xsk_umem_dma_unmap(struct i40e_vsi *vsi, struct xdp_umem *umem) | |
117 | { | |
118 | struct i40e_pf *pf = vsi->back; | |
119 | struct device *dev; | |
120 | unsigned int i; | |
121 | ||
122 | dev = &pf->pdev->dev; | |
123 | ||
124 | for (i = 0; i < umem->npgs; i++) { | |
125 | dma_unmap_page_attrs(dev, umem->pages[i].dma, PAGE_SIZE, | |
126 | DMA_BIDIRECTIONAL, I40E_RX_DMA_ATTR); | |
127 | ||
128 | umem->pages[i].dma = 0; | |
129 | } | |
130 | } | |
131 | ||
132 | /** | |
529eb362 | 133 | * i40e_xsk_umem_enable - Enable/associate a UMEM to a certain ring/qid |
0a714186 BT |
134 | * @vsi: Current VSI |
135 | * @umem: UMEM | |
136 | * @qid: Rx ring to associate UMEM to | |
137 | * | |
138 | * Returns 0 on success, <0 on failure | |
139 | **/ | |
140 | static int i40e_xsk_umem_enable(struct i40e_vsi *vsi, struct xdp_umem *umem, | |
141 | u16 qid) | |
142 | { | |
411dc16f | 143 | struct xdp_umem_fq_reuse *reuseq; |
0a714186 BT |
144 | bool if_running; |
145 | int err; | |
146 | ||
147 | if (vsi->type != I40E_VSI_MAIN) | |
148 | return -EINVAL; | |
149 | ||
150 | if (qid >= vsi->num_queue_pairs) | |
151 | return -EINVAL; | |
152 | ||
153 | if (vsi->xsk_umems) { | |
154 | if (qid >= vsi->num_xsk_umems) | |
155 | return -EINVAL; | |
156 | if (vsi->xsk_umems[qid]) | |
157 | return -EBUSY; | |
158 | } | |
159 | ||
411dc16f BT |
160 | reuseq = xsk_reuseq_prepare(vsi->rx_rings[0]->count); |
161 | if (!reuseq) | |
162 | return -ENOMEM; | |
163 | ||
164 | xsk_reuseq_free(xsk_reuseq_swap(umem, reuseq)); | |
165 | ||
0a714186 BT |
166 | err = i40e_xsk_umem_dma_map(vsi, umem); |
167 | if (err) | |
168 | return err; | |
169 | ||
170 | if_running = netif_running(vsi->netdev) && i40e_enabled_xdp_vsi(vsi); | |
171 | ||
172 | if (if_running) { | |
173 | err = i40e_queue_pair_disable(vsi, qid); | |
174 | if (err) | |
175 | return err; | |
176 | } | |
177 | ||
178 | err = i40e_add_xsk_umem(vsi, umem, qid); | |
179 | if (err) | |
180 | return err; | |
181 | ||
182 | if (if_running) { | |
183 | err = i40e_queue_pair_enable(vsi, qid); | |
184 | if (err) | |
185 | return err; | |
14ffeb52 MK |
186 | |
187 | /* Kick start the NAPI context so that receiving will start */ | |
188 | err = i40e_xsk_async_xmit(vsi->netdev, qid); | |
189 | if (err) | |
190 | return err; | |
0a714186 BT |
191 | } |
192 | ||
193 | return 0; | |
194 | } | |
195 | ||
196 | /** | |
529eb362 | 197 | * i40e_xsk_umem_disable - Disassociate a UMEM from a certain ring/qid |
0a714186 BT |
198 | * @vsi: Current VSI |
199 | * @qid: Rx ring to associate UMEM to | |
200 | * | |
201 | * Returns 0 on success, <0 on failure | |
202 | **/ | |
203 | static int i40e_xsk_umem_disable(struct i40e_vsi *vsi, u16 qid) | |
204 | { | |
205 | bool if_running; | |
206 | int err; | |
207 | ||
208 | if (!vsi->xsk_umems || qid >= vsi->num_xsk_umems || | |
209 | !vsi->xsk_umems[qid]) | |
210 | return -EINVAL; | |
211 | ||
212 | if_running = netif_running(vsi->netdev) && i40e_enabled_xdp_vsi(vsi); | |
213 | ||
214 | if (if_running) { | |
215 | err = i40e_queue_pair_disable(vsi, qid); | |
216 | if (err) | |
217 | return err; | |
218 | } | |
219 | ||
220 | i40e_xsk_umem_dma_unmap(vsi, vsi->xsk_umems[qid]); | |
221 | i40e_remove_xsk_umem(vsi, qid); | |
222 | ||
223 | if (if_running) { | |
224 | err = i40e_queue_pair_enable(vsi, qid); | |
225 | if (err) | |
226 | return err; | |
227 | } | |
228 | ||
229 | return 0; | |
230 | } | |
231 | ||
232 | /** | |
233 | * i40e_xsk_umem_query - Queries a certain ring/qid for its UMEM | |
234 | * @vsi: Current VSI | |
235 | * @umem: UMEM associated to the ring, if any | |
236 | * @qid: Rx ring to associate UMEM to | |
237 | * | |
238 | * This function will store, if any, the UMEM associated to certain ring. | |
239 | * | |
240 | * Returns 0 on success, <0 on failure | |
241 | **/ | |
242 | int i40e_xsk_umem_query(struct i40e_vsi *vsi, struct xdp_umem **umem, | |
243 | u16 qid) | |
244 | { | |
245 | if (vsi->type != I40E_VSI_MAIN) | |
246 | return -EINVAL; | |
247 | ||
248 | if (qid >= vsi->num_queue_pairs) | |
249 | return -EINVAL; | |
250 | ||
251 | if (vsi->xsk_umems) { | |
252 | if (qid >= vsi->num_xsk_umems) | |
253 | return -EINVAL; | |
254 | *umem = vsi->xsk_umems[qid]; | |
255 | return 0; | |
256 | } | |
257 | ||
258 | *umem = NULL; | |
259 | return 0; | |
260 | } | |
261 | ||
262 | /** | |
529eb362 | 263 | * i40e_xsk_umem_setup - Enable/disassociate a UMEM to/from a ring/qid |
0a714186 BT |
264 | * @vsi: Current VSI |
265 | * @umem: UMEM to enable/associate to a ring, or NULL to disable | |
266 | * @qid: Rx ring to (dis)associate UMEM (from)to | |
267 | * | |
529eb362 | 268 | * This function enables or disables a UMEM to a certain ring. |
0a714186 BT |
269 | * |
270 | * Returns 0 on success, <0 on failure | |
271 | **/ | |
272 | int i40e_xsk_umem_setup(struct i40e_vsi *vsi, struct xdp_umem *umem, | |
273 | u16 qid) | |
274 | { | |
275 | return umem ? i40e_xsk_umem_enable(vsi, umem, qid) : | |
276 | i40e_xsk_umem_disable(vsi, qid); | |
277 | } | |
278 | ||
279 | /** | |
280 | * i40e_run_xdp_zc - Executes an XDP program on an xdp_buff | |
281 | * @rx_ring: Rx ring | |
282 | * @xdp: xdp_buff used as input to the XDP program | |
283 | * | |
529eb362 | 284 | * This function enables or disables a UMEM to a certain ring. |
0a714186 BT |
285 | * |
286 | * Returns any of I40E_XDP_{PASS, CONSUMED, TX, REDIR} | |
287 | **/ | |
288 | static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) | |
289 | { | |
290 | int err, result = I40E_XDP_PASS; | |
291 | struct i40e_ring *xdp_ring; | |
292 | struct bpf_prog *xdp_prog; | |
293 | u32 act; | |
294 | ||
295 | rcu_read_lock(); | |
296 | /* NB! xdp_prog will always be !NULL, due to the fact that | |
297 | * this path is enabled by setting an XDP program. | |
298 | */ | |
299 | xdp_prog = READ_ONCE(rx_ring->xdp_prog); | |
300 | act = bpf_prog_run_xdp(xdp_prog, xdp); | |
301 | xdp->handle += xdp->data - xdp->data_hard_start; | |
302 | switch (act) { | |
303 | case XDP_PASS: | |
304 | break; | |
305 | case XDP_TX: | |
306 | xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; | |
307 | result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring); | |
308 | break; | |
309 | case XDP_REDIRECT: | |
310 | err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); | |
311 | result = !err ? I40E_XDP_REDIR : I40E_XDP_CONSUMED; | |
312 | break; | |
313 | default: | |
314 | bpf_warn_invalid_xdp_action(act); | |
315 | case XDP_ABORTED: | |
316 | trace_xdp_exception(rx_ring->netdev, xdp_prog, act); | |
317 | /* fallthrough -- handle aborts by dropping packet */ | |
318 | case XDP_DROP: | |
319 | result = I40E_XDP_CONSUMED; | |
320 | break; | |
321 | } | |
322 | rcu_read_unlock(); | |
323 | return result; | |
324 | } | |
325 | ||
326 | /** | |
327 | * i40e_alloc_buffer_zc - Allocates an i40e_rx_buffer | |
328 | * @rx_ring: Rx ring | |
329 | * @bi: Rx buffer to populate | |
330 | * | |
331 | * This function allocates an Rx buffer. The buffer can come from fill | |
332 | * queue, or via the recycle queue (next_to_alloc). | |
333 | * | |
334 | * Returns true for a successful allocation, false otherwise | |
335 | **/ | |
336 | static bool i40e_alloc_buffer_zc(struct i40e_ring *rx_ring, | |
337 | struct i40e_rx_buffer *bi) | |
338 | { | |
339 | struct xdp_umem *umem = rx_ring->xsk_umem; | |
340 | void *addr = bi->addr; | |
341 | u64 handle, hr; | |
342 | ||
343 | if (addr) { | |
344 | rx_ring->rx_stats.page_reuse_count++; | |
345 | return true; | |
346 | } | |
347 | ||
348 | if (!xsk_umem_peek_addr(umem, &handle)) { | |
349 | rx_ring->rx_stats.alloc_page_failed++; | |
350 | return false; | |
351 | } | |
352 | ||
353 | hr = umem->headroom + XDP_PACKET_HEADROOM; | |
354 | ||
355 | bi->dma = xdp_umem_get_dma(umem, handle); | |
356 | bi->dma += hr; | |
357 | ||
358 | bi->addr = xdp_umem_get_data(umem, handle); | |
359 | bi->addr += hr; | |
360 | ||
361 | bi->handle = handle + umem->headroom; | |
362 | ||
363 | xsk_umem_discard_addr(umem); | |
364 | return true; | |
365 | } | |
366 | ||
367 | /** | |
411dc16f | 368 | * i40e_alloc_buffer_slow_zc - Allocates an i40e_rx_buffer |
0a714186 | 369 | * @rx_ring: Rx ring |
411dc16f | 370 | * @bi: Rx buffer to populate |
0a714186 | 371 | * |
411dc16f BT |
372 | * This function allocates an Rx buffer. The buffer can come from fill |
373 | * queue, or via the reuse queue. | |
0a714186 BT |
374 | * |
375 | * Returns true for a successful allocation, false otherwise | |
376 | **/ | |
411dc16f BT |
377 | static bool i40e_alloc_buffer_slow_zc(struct i40e_ring *rx_ring, |
378 | struct i40e_rx_buffer *bi) | |
379 | { | |
380 | struct xdp_umem *umem = rx_ring->xsk_umem; | |
381 | u64 handle, hr; | |
382 | ||
383 | if (!xsk_umem_peek_addr_rq(umem, &handle)) { | |
384 | rx_ring->rx_stats.alloc_page_failed++; | |
385 | return false; | |
386 | } | |
387 | ||
388 | handle &= rx_ring->xsk_umem->chunk_mask; | |
389 | ||
390 | hr = umem->headroom + XDP_PACKET_HEADROOM; | |
391 | ||
392 | bi->dma = xdp_umem_get_dma(umem, handle); | |
393 | bi->dma += hr; | |
394 | ||
395 | bi->addr = xdp_umem_get_data(umem, handle); | |
396 | bi->addr += hr; | |
397 | ||
398 | bi->handle = handle + umem->headroom; | |
399 | ||
400 | xsk_umem_discard_addr_rq(umem); | |
401 | return true; | |
402 | } | |
403 | ||
404 | static __always_inline bool | |
405 | __i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count, | |
406 | bool alloc(struct i40e_ring *rx_ring, | |
407 | struct i40e_rx_buffer *bi)) | |
0a714186 BT |
408 | { |
409 | u16 ntu = rx_ring->next_to_use; | |
410 | union i40e_rx_desc *rx_desc; | |
411 | struct i40e_rx_buffer *bi; | |
412 | bool ok = true; | |
413 | ||
414 | rx_desc = I40E_RX_DESC(rx_ring, ntu); | |
415 | bi = &rx_ring->rx_bi[ntu]; | |
416 | do { | |
411dc16f | 417 | if (!alloc(rx_ring, bi)) { |
0a714186 BT |
418 | ok = false; |
419 | goto no_buffers; | |
420 | } | |
421 | ||
422 | dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 0, | |
423 | rx_ring->rx_buf_len, | |
424 | DMA_BIDIRECTIONAL); | |
425 | ||
426 | rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); | |
427 | ||
428 | rx_desc++; | |
429 | bi++; | |
430 | ntu++; | |
431 | ||
432 | if (unlikely(ntu == rx_ring->count)) { | |
433 | rx_desc = I40E_RX_DESC(rx_ring, 0); | |
434 | bi = rx_ring->rx_bi; | |
435 | ntu = 0; | |
436 | } | |
437 | ||
438 | rx_desc->wb.qword1.status_error_len = 0; | |
439 | count--; | |
440 | } while (count); | |
441 | ||
442 | no_buffers: | |
443 | if (rx_ring->next_to_use != ntu) | |
444 | i40e_release_rx_desc(rx_ring, ntu); | |
445 | ||
446 | return ok; | |
447 | } | |
448 | ||
411dc16f BT |
449 | /** |
450 | * i40e_alloc_rx_buffers_zc - Allocates a number of Rx buffers | |
451 | * @rx_ring: Rx ring | |
452 | * @count: The number of buffers to allocate | |
453 | * | |
454 | * This function allocates a number of Rx buffers from the reuse queue | |
455 | * or fill ring and places them on the Rx ring. | |
456 | * | |
457 | * Returns true for a successful allocation, false otherwise | |
458 | **/ | |
459 | bool i40e_alloc_rx_buffers_zc(struct i40e_ring *rx_ring, u16 count) | |
460 | { | |
461 | return __i40e_alloc_rx_buffers_zc(rx_ring, count, | |
462 | i40e_alloc_buffer_slow_zc); | |
463 | } | |
464 | ||
465 | /** | |
466 | * i40e_alloc_rx_buffers_fast_zc - Allocates a number of Rx buffers | |
467 | * @rx_ring: Rx ring | |
468 | * @count: The number of buffers to allocate | |
469 | * | |
470 | * This function allocates a number of Rx buffers from the fill ring | |
471 | * or the internal recycle mechanism and places them on the Rx ring. | |
472 | * | |
473 | * Returns true for a successful allocation, false otherwise | |
474 | **/ | |
475 | static bool i40e_alloc_rx_buffers_fast_zc(struct i40e_ring *rx_ring, u16 count) | |
476 | { | |
477 | return __i40e_alloc_rx_buffers_zc(rx_ring, count, | |
478 | i40e_alloc_buffer_zc); | |
479 | } | |
480 | ||
0a714186 BT |
481 | /** |
482 | * i40e_get_rx_buffer_zc - Return the current Rx buffer | |
483 | * @rx_ring: Rx ring | |
484 | * @size: The size of the rx buffer (read from descriptor) | |
485 | * | |
486 | * This function returns the current, received Rx buffer, and also | |
487 | * does DMA synchronization. the Rx ring. | |
488 | * | |
489 | * Returns the received Rx buffer | |
490 | **/ | |
491 | static struct i40e_rx_buffer *i40e_get_rx_buffer_zc(struct i40e_ring *rx_ring, | |
492 | const unsigned int size) | |
493 | { | |
494 | struct i40e_rx_buffer *bi; | |
495 | ||
496 | bi = &rx_ring->rx_bi[rx_ring->next_to_clean]; | |
497 | ||
498 | /* we are reusing so sync this buffer for CPU use */ | |
499 | dma_sync_single_range_for_cpu(rx_ring->dev, | |
500 | bi->dma, 0, | |
501 | size, | |
502 | DMA_BIDIRECTIONAL); | |
503 | ||
504 | return bi; | |
505 | } | |
506 | ||
507 | /** | |
508 | * i40e_reuse_rx_buffer_zc - Recycle an Rx buffer | |
509 | * @rx_ring: Rx ring | |
510 | * @old_bi: The Rx buffer to recycle | |
511 | * | |
512 | * This function recycles a finished Rx buffer, and places it on the | |
513 | * recycle queue (next_to_alloc). | |
514 | **/ | |
515 | static void i40e_reuse_rx_buffer_zc(struct i40e_ring *rx_ring, | |
516 | struct i40e_rx_buffer *old_bi) | |
517 | { | |
518 | struct i40e_rx_buffer *new_bi = &rx_ring->rx_bi[rx_ring->next_to_alloc]; | |
93ee30f3 | 519 | unsigned long mask = (unsigned long)rx_ring->xsk_umem->chunk_mask; |
0a714186 BT |
520 | u64 hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM; |
521 | u16 nta = rx_ring->next_to_alloc; | |
522 | ||
523 | /* update, and store next to alloc */ | |
524 | nta++; | |
525 | rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; | |
526 | ||
527 | /* transfer page from old buffer to new buffer */ | |
528 | new_bi->dma = old_bi->dma & mask; | |
529 | new_bi->dma += hr; | |
530 | ||
531 | new_bi->addr = (void *)((unsigned long)old_bi->addr & mask); | |
532 | new_bi->addr += hr; | |
533 | ||
534 | new_bi->handle = old_bi->handle & mask; | |
535 | new_bi->handle += rx_ring->xsk_umem->headroom; | |
536 | ||
537 | old_bi->addr = NULL; | |
538 | } | |
539 | ||
540 | /** | |
541 | * i40e_zca_free - Free callback for MEM_TYPE_ZERO_COPY allocations | |
542 | * @alloc: Zero-copy allocator | |
543 | * @handle: Buffer handle | |
544 | **/ | |
545 | void i40e_zca_free(struct zero_copy_allocator *alloc, unsigned long handle) | |
546 | { | |
547 | struct i40e_rx_buffer *bi; | |
548 | struct i40e_ring *rx_ring; | |
549 | u64 hr, mask; | |
550 | u16 nta; | |
551 | ||
552 | rx_ring = container_of(alloc, struct i40e_ring, zca); | |
553 | hr = rx_ring->xsk_umem->headroom + XDP_PACKET_HEADROOM; | |
93ee30f3 | 554 | mask = rx_ring->xsk_umem->chunk_mask; |
0a714186 BT |
555 | |
556 | nta = rx_ring->next_to_alloc; | |
557 | bi = &rx_ring->rx_bi[nta]; | |
558 | ||
559 | nta++; | |
560 | rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; | |
561 | ||
562 | handle &= mask; | |
563 | ||
564 | bi->dma = xdp_umem_get_dma(rx_ring->xsk_umem, handle); | |
565 | bi->dma += hr; | |
566 | ||
567 | bi->addr = xdp_umem_get_data(rx_ring->xsk_umem, handle); | |
568 | bi->addr += hr; | |
569 | ||
570 | bi->handle = (u64)handle + rx_ring->xsk_umem->headroom; | |
571 | } | |
572 | ||
573 | /** | |
574 | * i40e_construct_skb_zc - Create skbufff from zero-copy Rx buffer | |
575 | * @rx_ring: Rx ring | |
576 | * @bi: Rx buffer | |
577 | * @xdp: xdp_buff | |
578 | * | |
579 | * This functions allocates a new skb from a zero-copy Rx buffer. | |
580 | * | |
581 | * Returns the skb, or NULL on failure. | |
582 | **/ | |
583 | static struct sk_buff *i40e_construct_skb_zc(struct i40e_ring *rx_ring, | |
584 | struct i40e_rx_buffer *bi, | |
585 | struct xdp_buff *xdp) | |
586 | { | |
587 | unsigned int metasize = xdp->data - xdp->data_meta; | |
588 | unsigned int datasize = xdp->data_end - xdp->data; | |
589 | struct sk_buff *skb; | |
590 | ||
591 | /* allocate a skb to store the frags */ | |
592 | skb = __napi_alloc_skb(&rx_ring->q_vector->napi, | |
593 | xdp->data_end - xdp->data_hard_start, | |
594 | GFP_ATOMIC | __GFP_NOWARN); | |
595 | if (unlikely(!skb)) | |
596 | return NULL; | |
597 | ||
598 | skb_reserve(skb, xdp->data - xdp->data_hard_start); | |
599 | memcpy(__skb_put(skb, datasize), xdp->data, datasize); | |
600 | if (metasize) | |
601 | skb_metadata_set(skb, metasize); | |
602 | ||
603 | i40e_reuse_rx_buffer_zc(rx_ring, bi); | |
604 | return skb; | |
605 | } | |
606 | ||
607 | /** | |
608 | * i40e_inc_ntc: Advance the next_to_clean index | |
609 | * @rx_ring: Rx ring | |
610 | **/ | |
611 | static void i40e_inc_ntc(struct i40e_ring *rx_ring) | |
612 | { | |
613 | u32 ntc = rx_ring->next_to_clean + 1; | |
614 | ||
615 | ntc = (ntc < rx_ring->count) ? ntc : 0; | |
616 | rx_ring->next_to_clean = ntc; | |
617 | prefetch(I40E_RX_DESC(rx_ring, ntc)); | |
618 | } | |
619 | ||
620 | /** | |
621 | * i40e_clean_rx_irq_zc - Consumes Rx packets from the hardware ring | |
622 | * @rx_ring: Rx ring | |
623 | * @budget: NAPI budget | |
624 | * | |
625 | * Returns amount of work completed | |
626 | **/ | |
627 | int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) | |
628 | { | |
629 | unsigned int total_rx_bytes = 0, total_rx_packets = 0; | |
630 | u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); | |
631 | unsigned int xdp_res, xdp_xmit = 0; | |
632 | bool failure = false; | |
633 | struct sk_buff *skb; | |
634 | struct xdp_buff xdp; | |
635 | ||
636 | xdp.rxq = &rx_ring->xdp_rxq; | |
637 | ||
638 | while (likely(total_rx_packets < (unsigned int)budget)) { | |
639 | struct i40e_rx_buffer *bi; | |
640 | union i40e_rx_desc *rx_desc; | |
641 | unsigned int size; | |
0a714186 BT |
642 | u64 qword; |
643 | ||
644 | if (cleaned_count >= I40E_RX_BUFFER_WRITE) { | |
645 | failure = failure || | |
411dc16f BT |
646 | !i40e_alloc_rx_buffers_fast_zc(rx_ring, |
647 | cleaned_count); | |
0a714186 BT |
648 | cleaned_count = 0; |
649 | } | |
650 | ||
651 | rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean); | |
652 | qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); | |
653 | ||
654 | /* This memory barrier is needed to keep us from reading | |
655 | * any other fields out of the rx_desc until we have | |
656 | * verified the descriptor has been written back. | |
657 | */ | |
658 | dma_rmb(); | |
659 | ||
660 | bi = i40e_clean_programming_status(rx_ring, rx_desc, | |
661 | qword); | |
662 | if (unlikely(bi)) { | |
663 | i40e_reuse_rx_buffer_zc(rx_ring, bi); | |
664 | cleaned_count++; | |
665 | continue; | |
666 | } | |
667 | ||
668 | size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> | |
669 | I40E_RXD_QW1_LENGTH_PBUF_SHIFT; | |
670 | if (!size) | |
671 | break; | |
672 | ||
673 | bi = i40e_get_rx_buffer_zc(rx_ring, size); | |
674 | xdp.data = bi->addr; | |
675 | xdp.data_meta = xdp.data; | |
676 | xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM; | |
677 | xdp.data_end = xdp.data + size; | |
678 | xdp.handle = bi->handle; | |
679 | ||
680 | xdp_res = i40e_run_xdp_zc(rx_ring, &xdp); | |
681 | if (xdp_res) { | |
682 | if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) { | |
683 | xdp_xmit |= xdp_res; | |
684 | bi->addr = NULL; | |
685 | } else { | |
686 | i40e_reuse_rx_buffer_zc(rx_ring, bi); | |
687 | } | |
688 | ||
689 | total_rx_bytes += size; | |
690 | total_rx_packets++; | |
691 | ||
692 | cleaned_count++; | |
693 | i40e_inc_ntc(rx_ring); | |
694 | continue; | |
695 | } | |
696 | ||
697 | /* XDP_PASS path */ | |
698 | ||
699 | /* NB! We are not checking for errors using | |
700 | * i40e_test_staterr with | |
701 | * BIT(I40E_RXD_QW1_ERROR_SHIFT). This is due to that | |
702 | * SBP is *not* set in PRT_SBPVSI (default not set). | |
703 | */ | |
704 | skb = i40e_construct_skb_zc(rx_ring, bi, &xdp); | |
705 | if (!skb) { | |
706 | rx_ring->rx_stats.alloc_buff_failed++; | |
707 | break; | |
708 | } | |
709 | ||
710 | cleaned_count++; | |
711 | i40e_inc_ntc(rx_ring); | |
712 | ||
713 | if (eth_skb_pad(skb)) | |
714 | continue; | |
715 | ||
716 | total_rx_bytes += skb->len; | |
717 | total_rx_packets++; | |
718 | ||
800b8f63 | 719 | i40e_process_skb_fields(rx_ring, rx_desc, skb); |
2a508c64 | 720 | napi_gro_receive(&rx_ring->q_vector->napi, skb); |
0a714186 BT |
721 | } |
722 | ||
723 | i40e_finalize_xdp_rx(rx_ring, xdp_xmit); | |
724 | i40e_update_rx_stats(rx_ring, total_rx_bytes, total_rx_packets); | |
725 | return failure ? budget : (int)total_rx_packets; | |
726 | } | |
727 | ||
1328dcdd MK |
728 | /** |
729 | * i40e_xmit_zc - Performs zero-copy Tx AF_XDP | |
730 | * @xdp_ring: XDP Tx ring | |
731 | * @budget: NAPI budget | |
732 | * | |
733 | * Returns true if the work is finished. | |
734 | **/ | |
735 | static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget) | |
736 | { | |
cf484f9f | 737 | struct i40e_tx_desc *tx_desc = NULL; |
1328dcdd | 738 | struct i40e_tx_buffer *tx_bi; |
1328dcdd MK |
739 | bool work_done = true; |
740 | dma_addr_t dma; | |
741 | u32 len; | |
742 | ||
743 | while (budget-- > 0) { | |
744 | if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) { | |
745 | xdp_ring->tx_stats.tx_busy++; | |
746 | work_done = false; | |
747 | break; | |
748 | } | |
749 | ||
750 | if (!xsk_umem_consume_tx(xdp_ring->xsk_umem, &dma, &len)) | |
751 | break; | |
752 | ||
753 | dma_sync_single_for_device(xdp_ring->dev, dma, len, | |
754 | DMA_BIDIRECTIONAL); | |
755 | ||
756 | tx_bi = &xdp_ring->tx_bi[xdp_ring->next_to_use]; | |
757 | tx_bi->bytecount = len; | |
758 | ||
759 | tx_desc = I40E_TX_DESC(xdp_ring, xdp_ring->next_to_use); | |
760 | tx_desc->buffer_addr = cpu_to_le64(dma); | |
761 | tx_desc->cmd_type_offset_bsz = | |
762 | build_ctob(I40E_TX_DESC_CMD_ICRC | |
763 | | I40E_TX_DESC_CMD_EOP, | |
764 | 0, len, 0); | |
1328dcdd MK |
765 | |
766 | xdp_ring->next_to_use++; | |
767 | if (xdp_ring->next_to_use == xdp_ring->count) | |
768 | xdp_ring->next_to_use = 0; | |
769 | } | |
770 | ||
cf484f9f | 771 | if (tx_desc) { |
1328dcdd MK |
772 | /* Request an interrupt for the last frame and bump tail ptr. */ |
773 | tx_desc->cmd_type_offset_bsz |= (I40E_TX_DESC_CMD_RS << | |
774 | I40E_TXD_QW1_CMD_SHIFT); | |
775 | i40e_xdp_ring_update_tail(xdp_ring); | |
776 | ||
777 | xsk_umem_consume_tx_done(xdp_ring->xsk_umem); | |
778 | } | |
779 | ||
780 | return !!budget && work_done; | |
781 | } | |
782 | ||
783 | /** | |
784 | * i40e_clean_xdp_tx_buffer - Frees and unmaps an XDP Tx entry | |
785 | * @tx_ring: XDP Tx ring | |
786 | * @tx_bi: Tx buffer info to clean | |
787 | **/ | |
788 | static void i40e_clean_xdp_tx_buffer(struct i40e_ring *tx_ring, | |
789 | struct i40e_tx_buffer *tx_bi) | |
790 | { | |
791 | xdp_return_frame(tx_bi->xdpf); | |
792 | dma_unmap_single(tx_ring->dev, | |
793 | dma_unmap_addr(tx_bi, dma), | |
794 | dma_unmap_len(tx_bi, len), DMA_TO_DEVICE); | |
795 | dma_unmap_len_set(tx_bi, len, 0); | |
796 | } | |
797 | ||
798 | /** | |
799 | * i40e_clean_xdp_tx_irq - Completes AF_XDP entries, and cleans XDP entries | |
800 | * @tx_ring: XDP Tx ring | |
801 | * @tx_bi: Tx buffer info to clean | |
802 | * | |
803 | * Returns true if cleanup/tranmission is done. | |
804 | **/ | |
805 | bool i40e_clean_xdp_tx_irq(struct i40e_vsi *vsi, | |
806 | struct i40e_ring *tx_ring, int napi_budget) | |
807 | { | |
808 | unsigned int ntc, total_bytes = 0, budget = vsi->work_limit; | |
809 | u32 i, completed_frames, frames_ready, xsk_frames = 0; | |
810 | struct xdp_umem *umem = tx_ring->xsk_umem; | |
811 | u32 head_idx = i40e_get_head(tx_ring); | |
812 | bool work_done = true, xmit_done; | |
813 | struct i40e_tx_buffer *tx_bi; | |
814 | ||
815 | if (head_idx < tx_ring->next_to_clean) | |
816 | head_idx += tx_ring->count; | |
817 | frames_ready = head_idx - tx_ring->next_to_clean; | |
818 | ||
819 | if (frames_ready == 0) { | |
820 | goto out_xmit; | |
821 | } else if (frames_ready > budget) { | |
822 | completed_frames = budget; | |
823 | work_done = false; | |
824 | } else { | |
825 | completed_frames = frames_ready; | |
826 | } | |
827 | ||
828 | ntc = tx_ring->next_to_clean; | |
829 | ||
830 | for (i = 0; i < completed_frames; i++) { | |
831 | tx_bi = &tx_ring->tx_bi[ntc]; | |
832 | ||
833 | if (tx_bi->xdpf) | |
834 | i40e_clean_xdp_tx_buffer(tx_ring, tx_bi); | |
835 | else | |
836 | xsk_frames++; | |
837 | ||
838 | tx_bi->xdpf = NULL; | |
839 | total_bytes += tx_bi->bytecount; | |
840 | ||
841 | if (++ntc >= tx_ring->count) | |
842 | ntc = 0; | |
843 | } | |
844 | ||
845 | tx_ring->next_to_clean += completed_frames; | |
846 | if (unlikely(tx_ring->next_to_clean >= tx_ring->count)) | |
847 | tx_ring->next_to_clean -= tx_ring->count; | |
848 | ||
849 | if (xsk_frames) | |
850 | xsk_umem_complete_tx(umem, xsk_frames); | |
851 | ||
852 | i40e_arm_wb(tx_ring, vsi, budget); | |
853 | i40e_update_tx_stats(tx_ring, completed_frames, total_bytes); | |
854 | ||
855 | out_xmit: | |
856 | xmit_done = i40e_xmit_zc(tx_ring, budget); | |
857 | ||
858 | return work_done && xmit_done; | |
859 | } | |
860 | ||
861 | /** | |
862 | * i40e_xsk_async_xmit - Implements the ndo_xsk_async_xmit | |
863 | * @dev: the netdevice | |
864 | * @queue_id: queue id to wake up | |
865 | * | |
866 | * Returns <0 for errors, 0 otherwise. | |
867 | **/ | |
868 | int i40e_xsk_async_xmit(struct net_device *dev, u32 queue_id) | |
869 | { | |
870 | struct i40e_netdev_priv *np = netdev_priv(dev); | |
871 | struct i40e_vsi *vsi = np->vsi; | |
872 | struct i40e_ring *ring; | |
873 | ||
874 | if (test_bit(__I40E_VSI_DOWN, vsi->state)) | |
875 | return -ENETDOWN; | |
876 | ||
877 | if (!i40e_enabled_xdp_vsi(vsi)) | |
878 | return -ENXIO; | |
879 | ||
880 | if (queue_id >= vsi->num_queue_pairs) | |
881 | return -ENXIO; | |
882 | ||
883 | if (!vsi->xdp_rings[queue_id]->xsk_umem) | |
884 | return -ENXIO; | |
885 | ||
886 | ring = vsi->xdp_rings[queue_id]; | |
887 | ||
888 | /* The idea here is that if NAPI is running, mark a miss, so | |
889 | * it will run again. If not, trigger an interrupt and | |
890 | * schedule the NAPI from interrupt context. If NAPI would be | |
891 | * scheduled here, the interrupt affinity would not be | |
892 | * honored. | |
893 | */ | |
894 | if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) | |
895 | i40e_force_wb(vsi, ring->q_vector); | |
896 | ||
897 | return 0; | |
898 | } | |
9dbb1370 | 899 | |
411dc16f BT |
900 | void i40e_xsk_clean_rx_ring(struct i40e_ring *rx_ring) |
901 | { | |
902 | u16 i; | |
903 | ||
904 | for (i = 0; i < rx_ring->count; i++) { | |
905 | struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i]; | |
906 | ||
907 | if (!rx_bi->addr) | |
908 | continue; | |
909 | ||
910 | xsk_umem_fq_reuse(rx_ring->xsk_umem, rx_bi->handle); | |
911 | rx_bi->addr = NULL; | |
912 | } | |
913 | } | |
914 | ||
9dbb1370 BT |
915 | /** |
916 | * i40e_xsk_clean_xdp_ring - Clean the XDP Tx ring on shutdown | |
917 | * @xdp_ring: XDP Tx ring | |
918 | **/ | |
919 | void i40e_xsk_clean_tx_ring(struct i40e_ring *tx_ring) | |
920 | { | |
921 | u16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use; | |
922 | struct xdp_umem *umem = tx_ring->xsk_umem; | |
923 | struct i40e_tx_buffer *tx_bi; | |
924 | u32 xsk_frames = 0; | |
925 | ||
926 | while (ntc != ntu) { | |
927 | tx_bi = &tx_ring->tx_bi[ntc]; | |
928 | ||
929 | if (tx_bi->xdpf) | |
930 | i40e_clean_xdp_tx_buffer(tx_ring, tx_bi); | |
931 | else | |
932 | xsk_frames++; | |
933 | ||
934 | tx_bi->xdpf = NULL; | |
935 | ||
936 | ntc++; | |
937 | if (ntc >= tx_ring->count) | |
938 | ntc = 0; | |
939 | } | |
940 | ||
941 | if (xsk_frames) | |
942 | xsk_umem_complete_tx(umem, xsk_frames); | |
943 | } | |
3ab52af5 BT |
944 | |
945 | /** | |
946 | * i40e_xsk_any_rx_ring_enabled - Checks if Rx rings have AF_XDP UMEM attached | |
947 | * @vsi: vsi | |
948 | * | |
949 | * Returns true if any of the Rx rings has an AF_XDP UMEM attached | |
950 | **/ | |
951 | bool i40e_xsk_any_rx_ring_enabled(struct i40e_vsi *vsi) | |
952 | { | |
953 | int i; | |
954 | ||
955 | if (!vsi->xsk_umems) | |
956 | return false; | |
957 | ||
958 | for (i = 0; i < vsi->num_queue_pairs; i++) { | |
959 | if (vsi->xsk_umems[i]) | |
960 | return true; | |
961 | } | |
962 | ||
963 | return false; | |
964 | } |