projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ionic: use fewer firmware doorbells on rx fill
[linux-block.git] / drivers / net / ethernet / pensando / ionic / ionic_txrx.c
CommitLineData
0f3154e6
SN		 1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3
4#include <linux/ip.h>
5#include <linux/ipv6.h>
6#include <linux/if_vlan.h>
7#include <net/ip6_checksum.h>
8
9#include "ionic.h"
10#include "ionic_lif.h"
11#include "ionic_txrx.h"
12
5b3f3f2a
SN		 13static void ionic_rx_clean(struct ionic_queue *q,
14 struct ionic_desc_info *desc_info,
15 struct ionic_cq_info *cq_info,
16 void *cb_arg);
0f3154e6
SN		 17
18static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
19 ionic_desc_cb cb_func, void *cb_arg)
20{
21 DEBUG_STATS_TXQ_POST(q_to_qcq(q), q->head->desc, ring_dbell);
22
23 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
24}
25
26static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
27 ionic_desc_cb cb_func, void *cb_arg)
28{
29 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
30
31 DEBUG_STATS_RX_BUFF_CNT(q_to_qcq(q));
32}
33
34static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
35{
36 return netdev_get_tx_queue(q->lif->netdev, q->index);
37}
38
08f2e4b2
SN		 39static struct sk_buff *ionic_rx_skb_alloc(struct ionic_queue *q,
40 unsigned int len, bool frags)
0f3154e6 41{
08f2e4b2
SN		 42 struct ionic_lif *lif = q->lif;
43 struct ionic_rx_stats *stats;
44 struct net_device *netdev;
45 struct sk_buff *skb;
0f3154e6 46
08f2e4b2
SN		 47 netdev = lif->netdev;
48 stats = q_to_rx_stats(q);
0f3154e6 49
08f2e4b2
SN		 50 if (frags)
51 skb = napi_get_frags(&q_to_qcq(q)->napi);
52 else
53 skb = netdev_alloc_skb_ip_align(netdev, len);
54
55 if (unlikely(!skb)) {
56 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
57 netdev->name, q->name);
58 stats->alloc_err++;
59 return NULL;
60 }
61
62 return skb;
0f3154e6
SN		 63}
64
08f2e4b2
SN		 65static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
66 struct ionic_desc_info *desc_info,
67 struct ionic_cq_info *cq_info)
0f3154e6
SN		 68{
69 struct ionic_rxq_comp *comp = cq_info->cq_desc;
0f3154e6 70 struct device *dev = q->lif->ionic->dev;
08f2e4b2
SN		 71 struct ionic_page_info *page_info;
72 struct sk_buff *skb;
73 unsigned int i;
74 u16 frag_len;
75 u16 len;
0f3154e6 76
08f2e4b2
SN		 77 page_info = &desc_info->pages[0];
78 len = le16_to_cpu(comp->len);
0f3154e6 79
08f2e4b2 80 prefetch(page_address(page_info->page) + NET_IP_ALIGN);
0f3154e6 81
08f2e4b2
SN		 82 skb = ionic_rx_skb_alloc(q, len, true);
83 if (unlikely(!skb))
84 return NULL;
0f3154e6 85
08f2e4b2
SN		 86 i = comp->num_sg_elems + 1;
87 do {
88 if (unlikely(!page_info->page)) {
89 struct napi_struct *napi = &q_to_qcq(q)->napi;
0f3154e6 90
08f2e4b2
SN		 91 napi->skb = NULL;
92 dev_kfree_skb(skb);
93 return NULL;
94 }
95
96 frag_len = min(len, (u16)PAGE_SIZE);
97 len -= frag_len;
98
99 dma_unmap_page(dev, dma_unmap_addr(page_info, dma_addr),
100 PAGE_SIZE, DMA_FROM_DEVICE);
101 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
102 page_info->page, 0, frag_len, PAGE_SIZE);
103 page_info->page = NULL;
104 page_info++;
105 i--;
106 } while (i > 0);
107
108 return skb;
109}
110
111static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
112 struct ionic_desc_info *desc_info,
113 struct ionic_cq_info *cq_info)
114{
115 struct ionic_rxq_comp *comp = cq_info->cq_desc;
116 struct device *dev = q->lif->ionic->dev;
117 struct ionic_page_info *page_info;
118 struct sk_buff *skb;
119 u16 len;
120
121 page_info = &desc_info->pages[0];
122 len = le16_to_cpu(comp->len);
123
124 skb = ionic_rx_skb_alloc(q, len, false);
125 if (unlikely(!skb))
126 return NULL;
127
128 if (unlikely(!page_info->page)) {
129 dev_kfree_skb(skb);
130 return NULL;
131 }
132
133 dma_sync_single_for_cpu(dev, dma_unmap_addr(page_info, dma_addr),
134 len, DMA_FROM_DEVICE);
135 skb_copy_to_linear_data(skb, page_address(page_info->page), len);
136 dma_sync_single_for_device(dev, dma_unmap_addr(page_info, dma_addr),
137 len, DMA_FROM_DEVICE);
138
139 skb_put(skb, len);
140 skb->protocol = eth_type_trans(skb, q->lif->netdev);
141
142 return skb;
0f3154e6
SN		 143}
144
5b3f3f2a
SN		 145static void ionic_rx_clean(struct ionic_queue *q,
146 struct ionic_desc_info *desc_info,
147 struct ionic_cq_info *cq_info,
148 void *cb_arg)
0f3154e6
SN		 149{
150 struct ionic_rxq_comp *comp = cq_info->cq_desc;
151 struct ionic_qcq *qcq = q_to_qcq(q);
0f3154e6
SN		 152 struct ionic_rx_stats *stats;
153 struct net_device *netdev;
08f2e4b2 154 struct sk_buff *skb;
0f3154e6
SN		 155
156 stats = q_to_rx_stats(q);
157 netdev = q->lif->netdev;
158
24cfa8c7
SN		 159 if (comp->status) {
160 stats->dropped++;
0f3154e6 161 return;
24cfa8c7 162 }
0f3154e6 163
0f3154e6
SN		 164 stats->pkts++;
165 stats->bytes += le16_to_cpu(comp->len);
166
08f2e4b2
SN		 167 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
168 skb = ionic_rx_copybreak(q, desc_info, cq_info);
169 else
170 skb = ionic_rx_frags(q, desc_info, cq_info);
0f3154e6 171
24cfa8c7
SN		 172 if (unlikely(!skb)) {
173 stats->dropped++;
08f2e4b2 174 return;
24cfa8c7 175 }
0f3154e6
SN		 176
177 skb_record_rx_queue(skb, q->index);
178
08f2e4b2 179 if (likely(netdev->features & NETIF_F_RXHASH)) {
0f3154e6
SN		 180 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
181 case IONIC_PKT_TYPE_IPV4:
182 case IONIC_PKT_TYPE_IPV6:
183 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
184 PKT_HASH_TYPE_L3);
185 break;
186 case IONIC_PKT_TYPE_IPV4_TCP:
187 case IONIC_PKT_TYPE_IPV6_TCP:
188 case IONIC_PKT_TYPE_IPV4_UDP:
189 case IONIC_PKT_TYPE_IPV6_UDP:
190 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
191 PKT_HASH_TYPE_L4);
192 break;
193 }
194 }
195
08f2e4b2 196 if (likely(netdev->features & NETIF_F_RXCSUM)) {
0f3154e6
SN		 197 if (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
198 skb->ip_summed = CHECKSUM_COMPLETE;
199 skb->csum = (__wsum)le16_to_cpu(comp->csum);
200 stats->csum_complete++;
201 }
202 } else {
203 stats->csum_none++;
204 }
205
08f2e4b2
SN		 206 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
207 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
208 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
0f3154e6
SN		 209 stats->csum_error++;
210
f64e0c56
SN		 211 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
212 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)) {
213 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
214 le16_to_cpu(comp->vlan_tci));
215 stats->vlan_stripped++;
0f3154e6
SN		 216 }
217
08f2e4b2
SN		 218 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
219 napi_gro_receive(&qcq->napi, skb);
220 else
221 napi_gro_frags(&qcq->napi);
0f3154e6
SN		 222}
223
224static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
225{
226 struct ionic_rxq_comp *comp = cq_info->cq_desc;
227 struct ionic_queue *q = cq->bound_q;
228 struct ionic_desc_info *desc_info;
229
230 if (!color_match(comp->pkt_type_color, cq->done_color))
231 return false;
232
233 /* check for empty queue */
234 if (q->tail->index == q->head->index)
235 return false;
236
237 desc_info = q->tail;
238 if (desc_info->index != le16_to_cpu(comp->comp_index))
239 return false;
240
241 q->tail = desc_info->next;
242
243 /* clean the related q entry, only one per qc completion */
244 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
245
246 desc_info->cb = NULL;
247 desc_info->cb_arg = NULL;
248
249 return true;
250}
251
252static u32 ionic_rx_walk_cq(struct ionic_cq *rxcq, u32 limit)
253{
254 u32 work_done = 0;
255
256 while (ionic_rx_service(rxcq, rxcq->tail)) {
257 if (rxcq->tail->last)
258 rxcq->done_color = !rxcq->done_color;
259 rxcq->tail = rxcq->tail->next;
260 DEBUG_STATS_CQE_CNT(rxcq);
261
262 if (++work_done >= limit)
263 break;
264 }
265
266 return work_done;
267}
268
269void ionic_rx_flush(struct ionic_cq *cq)
270{
271 struct ionic_dev *idev = &cq->lif->ionic->idev;
272 u32 work_done;
273
274 work_done = ionic_rx_walk_cq(cq, cq->num_descs);
275
276 if (work_done)
277 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
278 work_done, IONIC_INTR_CRED_RESET_COALESCE);
279}
280
08f2e4b2
SN		 281static struct page *ionic_rx_page_alloc(struct ionic_queue *q,
282 dma_addr_t *dma_addr)
0f3154e6
SN		 283{
284 struct ionic_lif *lif = q->lif;
285 struct ionic_rx_stats *stats;
286 struct net_device *netdev;
0f3154e6 287 struct device *dev;
08f2e4b2 288 struct page *page;
0f3154e6
SN		 289
290 netdev = lif->netdev;
291 dev = lif->ionic->dev;
292 stats = q_to_rx_stats(q);
08f2e4b2
SN		 293 page = alloc_page(GFP_ATOMIC);
294 if (unlikely(!page)) {
295 net_err_ratelimited("%s: Page alloc failed on %s!\n",
296 netdev->name, q->name);
0f3154e6
SN		 297 stats->alloc_err++;
298 return NULL;
299 }
300
08f2e4b2
SN		 301 *dma_addr = dma_map_page(dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
302 if (unlikely(dma_mapping_error(dev, *dma_addr))) {
303 __free_page(page);
304 net_err_ratelimited("%s: DMA single map failed on %s!\n",
305 netdev->name, q->name);
0f3154e6
SN		 306 stats->dma_map_err++;
307 return NULL;
308 }
309
08f2e4b2 310 return page;
0f3154e6
SN		 311}
312
08f2e4b2
SN		 313static void ionic_rx_page_free(struct ionic_queue *q, struct page *page,
314 dma_addr_t dma_addr)
315{
316 struct ionic_lif *lif = q->lif;
317 struct net_device *netdev;
318 struct device *dev;
319
320 netdev = lif->netdev;
321 dev = lif->ionic->dev;
322
323 if (unlikely(!page)) {
324 net_err_ratelimited("%s: Trying to free unallocated buffer on %s!\n",
325 netdev->name, q->name);
326 return;
327 }
328
329 dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE);
330
331 __free_page(page);
332}
333
0f3154e6
SN		 334void ionic_rx_fill(struct ionic_queue *q)
335{
336 struct net_device *netdev = q->lif->netdev;
08f2e4b2
SN		 337 struct ionic_desc_info *desc_info;
338 struct ionic_page_info *page_info;
339 struct ionic_rxq_sg_desc *sg_desc;
340 struct ionic_rxq_sg_elem *sg_elem;
0f3154e6 341 struct ionic_rxq_desc *desc;
c37d6e3f
SN		 342 unsigned int remain_len;
343 unsigned int seg_len;
08f2e4b2 344 unsigned int nfrags;
08f2e4b2 345 unsigned int i, j;
0f3154e6 346 unsigned int len;
0f3154e6
SN		 347
348 len = netdev->mtu + ETH_HLEN;
08f2e4b2 349 nfrags = round_up(len, PAGE_SIZE) / PAGE_SIZE;
0f3154e6
SN		 350
351 for (i = ionic_q_space_avail(q); i; i--) {
c37d6e3f 352 remain_len = len;
08f2e4b2
SN		 353 desc_info = q->head;
354 desc = desc_info->desc;
355 sg_desc = desc_info->sg_desc;
356 page_info = &desc_info->pages[0];
357
358 if (page_info->page) { /* recycle the buffer */
155f15ad 359 ionic_rxq_post(q, false, ionic_rx_clean, NULL);
08f2e4b2
SN		 360 continue;
361 }
0f3154e6 362
08f2e4b2
SN		 363 /* fill main descriptor - pages[0] */
364 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
365 IONIC_RXQ_DESC_OPCODE_SIMPLE;
366 desc_info->npages = nfrags;
367 page_info->page = ionic_rx_page_alloc(q, &page_info->dma_addr);
368 if (unlikely(!page_info->page)) {
369 desc->addr = 0;
370 desc->len = 0;
371 return;
372 }
373 desc->addr = cpu_to_le64(page_info->dma_addr);
c37d6e3f
SN		 374 seg_len = min_t(unsigned int, PAGE_SIZE, len);
375 desc->len = cpu_to_le16(seg_len);
376 remain_len -= seg_len;
08f2e4b2
SN		 377 page_info++;
378
379 /* fill sg descriptors - pages[1..n] */
380 for (j = 0; j < nfrags - 1; j++) {
381 if (page_info->page) /* recycle the sg buffer */
382 continue;
383
384 sg_elem = &sg_desc->elems[j];
385 page_info->page = ionic_rx_page_alloc(q, &page_info->dma_addr);
386 if (unlikely(!page_info->page)) {
387 sg_elem->addr = 0;
388 sg_elem->len = 0;
389 return;
390 }
391 sg_elem->addr = cpu_to_le64(page_info->dma_addr);
c37d6e3f
SN		 392 seg_len = min_t(unsigned int, PAGE_SIZE, remain_len);
393 sg_elem->len = cpu_to_le16(seg_len);
394 remain_len -= seg_len;
08f2e4b2
SN		 395 page_info++;
396 }
0f3154e6 397
155f15ad 398 ionic_rxq_post(q, false, ionic_rx_clean, NULL);
0f3154e6 399 }
155f15ad
SN		 400
401 ionic_dbell_ring(q->lif->kern_dbpage, q->hw_type,
402 q->dbval | q->head->index);
0f3154e6
SN		 403}
404
405static void ionic_rx_fill_cb(void *arg)
406{
407 ionic_rx_fill(arg);
408}
409
410void ionic_rx_empty(struct ionic_queue *q)
411{
0f3154e6
SN		 412 struct ionic_desc_info *cur;
413 struct ionic_rxq_desc *desc;
08f2e4b2 414 unsigned int i;
0f3154e6
SN		 415
416 for (cur = q->tail; cur != q->head; cur = cur->next) {
417 desc = cur->desc;
08f2e4b2
SN		 418 desc->addr = 0;
419 desc->len = 0;
420
08f2e4b2
SN		 421 for (i = 0; i < cur->npages; i++) {
422 if (likely(cur->pages[i].page)) {
423 ionic_rx_page_free(q, cur->pages[i].page,
424 cur->pages[i].dma_addr);
425 cur->pages[i].page = NULL;
426 cur->pages[i].dma_addr = 0;
427 }
428 }
429
0f3154e6
SN		 430 cur->cb_arg = NULL;
431 }
432}
433
434int ionic_rx_napi(struct napi_struct *napi, int budget)
435{
436 struct ionic_qcq *qcq = napi_to_qcq(napi);
437 struct ionic_cq *rxcq = napi_to_cq(napi);
438 unsigned int qi = rxcq->bound_q->index;
439 struct ionic_dev *idev;
440 struct ionic_lif *lif;
441 struct ionic_cq *txcq;
442 u32 work_done = 0;
443 u32 flags = 0;
444
445 lif = rxcq->bound_q->lif;
446 idev = &lif->ionic->idev;
447 txcq = &lif->txqcqs[qi].qcq->cq;
448
449 ionic_tx_flush(txcq);
450
451 work_done = ionic_rx_walk_cq(rxcq, budget);
452
453 if (work_done)
454 ionic_rx_fill_cb(rxcq->bound_q);
455
456 if (work_done < budget && napi_complete_done(napi, work_done)) {
457 flags |= IONIC_INTR_CRED_UNMASK;
458 DEBUG_STATS_INTR_REARM(rxcq->bound_intr);
459 }
460
461 if (work_done || flags) {
462 flags |= IONIC_INTR_CRED_RESET_COALESCE;
463 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
464 work_done, flags);
465 }
466
467 DEBUG_STATS_NAPI_POLL(qcq, work_done);
468
469 return work_done;
470}
471
5b3f3f2a
SN		 472static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
473 void *data, size_t len)
0f3154e6
SN		 474{
475 struct ionic_tx_stats *stats = q_to_tx_stats(q);
476 struct device *dev = q->lif->ionic->dev;
477 dma_addr_t dma_addr;
478
479 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
480 if (dma_mapping_error(dev, dma_addr)) {
481 net_warn_ratelimited("%s: DMA single map failed on %s!\n",
482 q->lif->netdev->name, q->name);
483 stats->dma_map_err++;
484 return 0;
485 }
486 return dma_addr;
487}
488
5b3f3f2a
SN		 489static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
490 const skb_frag_t *frag,
0f3154e6
SN		 491 size_t offset, size_t len)
492{
493 struct ionic_tx_stats *stats = q_to_tx_stats(q);
494 struct device *dev = q->lif->ionic->dev;
495 dma_addr_t dma_addr;
496
497 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
498 if (dma_mapping_error(dev, dma_addr)) {
499 net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
500 q->lif->netdev->name, q->name);
501 stats->dma_map_err++;
502 }
503 return dma_addr;
504}
505
5b3f3f2a
SN		 506static void ionic_tx_clean(struct ionic_queue *q,
507 struct ionic_desc_info *desc_info,
508 struct ionic_cq_info *cq_info,
509 void *cb_arg)
0f3154e6
SN		 510{
511 struct ionic_txq_sg_desc *sg_desc = desc_info->sg_desc;
512 struct ionic_txq_sg_elem *elem = sg_desc->elems;
513 struct ionic_tx_stats *stats = q_to_tx_stats(q);
514 struct ionic_txq_desc *desc = desc_info->desc;
515 struct device *dev = q->lif->ionic->dev;
516 u8 opcode, flags, nsge;
517 u16 queue_index;
518 unsigned int i;
519 u64 addr;
520
521 decode_txq_desc_cmd(le64_to_cpu(desc->cmd),
522 &opcode, &flags, &nsge, &addr);
523
524 /* use unmap_single only if either this is not TSO,
525 * or this is first descriptor of a TSO
526 */
527 if (opcode != IONIC_TXQ_DESC_OPCODE_TSO ||
528 flags & IONIC_TXQ_DESC_FLAG_TSO_SOT)
529 dma_unmap_single(dev, (dma_addr_t)addr,
530 le16_to_cpu(desc->len), DMA_TO_DEVICE);
531 else
532 dma_unmap_page(dev, (dma_addr_t)addr,
533 le16_to_cpu(desc->len), DMA_TO_DEVICE);
534
535 for (i = 0; i < nsge; i++, elem++)
536 dma_unmap_page(dev, (dma_addr_t)le64_to_cpu(elem->addr),
537 le16_to_cpu(elem->len), DMA_TO_DEVICE);
538
539 if (cb_arg) {
540 struct sk_buff *skb = cb_arg;
541 u32 len = skb->len;
542
543 queue_index = skb_get_queue_mapping(skb);
544 if (unlikely(__netif_subqueue_stopped(q->lif->netdev,
545 queue_index))) {
546 netif_wake_subqueue(q->lif->netdev, queue_index);
547 q->wake++;
548 }
549 dev_kfree_skb_any(skb);
550 stats->clean++;
551 netdev_tx_completed_queue(q_to_ndq(q), 1, len);
552 }
553}
554
555void ionic_tx_flush(struct ionic_cq *cq)
556{
557 struct ionic_txq_comp *comp = cq->tail->cq_desc;
558 struct ionic_dev *idev = &cq->lif->ionic->idev;
559 struct ionic_queue *q = cq->bound_q;
560 struct ionic_desc_info *desc_info;
561 unsigned int work_done = 0;
562
563 /* walk the completed cq entries */
564 while (work_done < cq->num_descs &&
565 color_match(comp->color, cq->done_color)) {
566
567 /* clean the related q entries, there could be
568 * several q entries completed for each cq completion
569 */
570 do {
571 desc_info = q->tail;
572 q->tail = desc_info->next;
573 ionic_tx_clean(q, desc_info, cq->tail,
574 desc_info->cb_arg);
575 desc_info->cb = NULL;
576 desc_info->cb_arg = NULL;
577 } while (desc_info->index != le16_to_cpu(comp->comp_index));
578
579 if (cq->tail->last)
580 cq->done_color = !cq->done_color;
581
582 cq->tail = cq->tail->next;
583 comp = cq->tail->cq_desc;
584 DEBUG_STATS_CQE_CNT(cq);
585
586 work_done++;
587 }
588
589 if (work_done)
590 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
591 work_done, 0);
592}
593
f9c00e2c
SN		 594void ionic_tx_empty(struct ionic_queue *q)
595{
596 struct ionic_desc_info *desc_info;
597 int done = 0;
598
599 /* walk the not completed tx entries, if any */
600 while (q->head != q->tail) {
601 desc_info = q->tail;
602 q->tail = desc_info->next;
603 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
604 desc_info->cb = NULL;
605 desc_info->cb_arg = NULL;
606 done++;
607 }
608}
609
0f3154e6
SN		 610static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
611{
612 int err;
613
614 err = skb_cow_head(skb, 0);
615 if (err)
616 return err;
617
618 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
619 inner_ip_hdr(skb)->check = 0;
620 inner_tcp_hdr(skb)->check =
621 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
622 inner_ip_hdr(skb)->daddr,
623 0, IPPROTO_TCP, 0);
624 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
625 inner_tcp_hdr(skb)->check =
626 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
627 &inner_ipv6_hdr(skb)->daddr,
628 0, IPPROTO_TCP, 0);
629 }
630
631 return 0;
632}
633
634static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
635{
636 int err;
637
638 err = skb_cow_head(skb, 0);
639 if (err)
640 return err;
641
642 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
643 ip_hdr(skb)->check = 0;
644 tcp_hdr(skb)->check =
645 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
646 ip_hdr(skb)->daddr,
647 0, IPPROTO_TCP, 0);
648 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
fa6b8429 649 tcp_v6_gso_csum_prep(skb);
0f3154e6
SN		 650 }
651
652 return 0;
653}
654
655static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
656 struct sk_buff *skb,
657 dma_addr_t addr, u8 nsge, u16 len,
658 unsigned int hdrlen, unsigned int mss,
659 bool outer_csum,
660 u16 vlan_tci, bool has_vlan,
661 bool start, bool done)
662{
663 u8 flags = 0;
664 u64 cmd;
665
666 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
667 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
668 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
669 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
670
671 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
672 desc->cmd = cpu_to_le64(cmd);
673 desc->len = cpu_to_le16(len);
674 desc->vlan_tci = cpu_to_le16(vlan_tci);
675 desc->hdr_len = cpu_to_le16(hdrlen);
676 desc->mss = cpu_to_le16(mss);
677
678 if (done) {
679 skb_tx_timestamp(skb);
680 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
681 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
682 } else {
683 ionic_txq_post(q, false, ionic_tx_clean, NULL);
684 }
685}
686
687static struct ionic_txq_desc *ionic_tx_tso_next(struct ionic_queue *q,
688 struct ionic_txq_sg_elem **elem)
689{
690 struct ionic_txq_sg_desc *sg_desc = q->head->sg_desc;
691 struct ionic_txq_desc *desc = q->head->desc;
692
693 *elem = sg_desc->elems;
694 return desc;
695}
696
697static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
698{
699 struct ionic_tx_stats *stats = q_to_tx_stats(q);
700 struct ionic_desc_info *abort = q->head;
701 struct device *dev = q->lif->ionic->dev;
702 struct ionic_desc_info *rewind = abort;
703 struct ionic_txq_sg_elem *elem;
704 struct ionic_txq_desc *desc;
705 unsigned int frag_left = 0;
706 unsigned int offset = 0;
707 unsigned int len_left;
708 dma_addr_t desc_addr;
709 unsigned int hdrlen;
710 unsigned int nfrags;
711 unsigned int seglen;
712 u64 total_bytes = 0;
713 u64 total_pkts = 0;
714 unsigned int left;
715 unsigned int len;
716 unsigned int mss;
717 skb_frag_t *frag;
718 bool start, done;
719 bool outer_csum;
720 bool has_vlan;
721 u16 desc_len;
722 u8 desc_nsge;
723 u16 vlan_tci;
724 bool encap;
725 int err;
726
727 mss = skb_shinfo(skb)->gso_size;
728 nfrags = skb_shinfo(skb)->nr_frags;
729 len_left = skb->len - skb_headlen(skb);
730 outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) ||
731 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
732 has_vlan = !!skb_vlan_tag_present(skb);
733 vlan_tci = skb_vlan_tag_get(skb);
734 encap = skb->encapsulation;
735
736 /* Preload inner-most TCP csum field with IP pseudo hdr
737 * calculated with IP length set to zero. HW will later
738 * add in length to each TCP segment resulting from the TSO.
739 */
740
741 if (encap)
742 err = ionic_tx_tcp_inner_pseudo_csum(skb);
743 else
744 err = ionic_tx_tcp_pseudo_csum(skb);
745 if (err)
746 return err;
747
748 if (encap)
749 hdrlen = skb_inner_transport_header(skb) - skb->data +
750 inner_tcp_hdrlen(skb);
751 else
752 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
753
754 seglen = hdrlen + mss;
755 left = skb_headlen(skb);
756
757 desc = ionic_tx_tso_next(q, &elem);
758 start = true;
759
760 /* Chop skb->data up into desc segments */
761
762 while (left > 0) {
763 len = min(seglen, left);
764 frag_left = seglen - len;
765 desc_addr = ionic_tx_map_single(q, skb->data + offset, len);
766 if (dma_mapping_error(dev, desc_addr))
767 goto err_out_abort;
768 desc_len = len;
769 desc_nsge = 0;
770 left -= len;
771 offset += len;
772 if (nfrags > 0 && frag_left > 0)
773 continue;
774 done = (nfrags == 0 && left == 0);
775 ionic_tx_tso_post(q, desc, skb,
776 desc_addr, desc_nsge, desc_len,
777 hdrlen, mss,
778 outer_csum,
779 vlan_tci, has_vlan,
780 start, done);
781 total_pkts++;
782 total_bytes += start ? len : len + hdrlen;
783 desc = ionic_tx_tso_next(q, &elem);
784 start = false;
785 seglen = mss;
786 }
787
788 /* Chop skb frags into desc segments */
789
790 for (frag = skb_shinfo(skb)->frags; len_left; frag++) {
791 offset = 0;
792 left = skb_frag_size(frag);
793 len_left -= left;
794 nfrags--;
795 stats->frags++;
796
797 while (left > 0) {
798 if (frag_left > 0) {
799 len = min(frag_left, left);
800 frag_left -= len;
801 elem->addr =
802 cpu_to_le64(ionic_tx_map_frag(q, frag,
803 offset, len));
804 if (dma_mapping_error(dev, elem->addr))
805 goto err_out_abort;
806 elem->len = cpu_to_le16(len);
807 elem++;
808 desc_nsge++;
809 left -= len;
810 offset += len;
811 if (nfrags > 0 && frag_left > 0)
812 continue;
813 done = (nfrags == 0 && left == 0);
814 ionic_tx_tso_post(q, desc, skb, desc_addr,
815 desc_nsge, desc_len,
816 hdrlen, mss, outer_csum,
817 vlan_tci, has_vlan,
818 start, done);
819 total_pkts++;
820 total_bytes += start ? len : len + hdrlen;
821 desc = ionic_tx_tso_next(q, &elem);
822 start = false;
823 } else {
824 len = min(mss, left);
825 frag_left = mss - len;
826 desc_addr = ionic_tx_map_frag(q, frag,
827 offset, len);
828 if (dma_mapping_error(dev, desc_addr))
829 goto err_out_abort;
830 desc_len = len;
831 desc_nsge = 0;
832 left -= len;
833 offset += len;
834 if (nfrags > 0 && frag_left > 0)
835 continue;
836 done = (nfrags == 0 && left == 0);
837 ionic_tx_tso_post(q, desc, skb, desc_addr,
838 desc_nsge, desc_len,
839 hdrlen, mss, outer_csum,
840 vlan_tci, has_vlan,
841 start, done);
842 total_pkts++;
843 total_bytes += start ? len : len + hdrlen;
844 desc = ionic_tx_tso_next(q, &elem);
845 start = false;
846 }
847 }
848 }
849
850 stats->pkts += total_pkts;
851 stats->bytes += total_bytes;
852 stats->tso++;
f64e0c56 853 stats->tso_bytes += total_bytes;
0f3154e6
SN		 854
855 return 0;
856
857err_out_abort:
858 while (rewind->desc != q->head->desc) {
859 ionic_tx_clean(q, rewind, NULL, NULL);
860 rewind = rewind->next;
861 }
862 q->head = abort;
863
864 return -ENOMEM;
865}
866
867static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb)
868{
869 struct ionic_tx_stats *stats = q_to_tx_stats(q);
870 struct ionic_txq_desc *desc = q->head->desc;
871 struct device *dev = q->lif->ionic->dev;
872 dma_addr_t dma_addr;
873 bool has_vlan;
874 u8 flags = 0;
875 bool encap;
876 u64 cmd;
877
878 has_vlan = !!skb_vlan_tag_present(skb);
879 encap = skb->encapsulation;
880
881 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
882 if (dma_mapping_error(dev, dma_addr))
883 return -ENOMEM;
884
885 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
886 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
887
888 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
889 flags, skb_shinfo(skb)->nr_frags, dma_addr);
890 desc->cmd = cpu_to_le64(cmd);
891 desc->len = cpu_to_le16(skb_headlen(skb));
0f3154e6
SN		 892 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
893 desc->csum_offset = cpu_to_le16(skb->csum_offset);
f64e0c56
SN		 894 if (has_vlan) {
895 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
896 stats->vlan_inserted++;
897 }
0f3154e6
SN		 898
899 if (skb->csum_not_inet)
900 stats->crc32_csum++;
901 else
902 stats->csum++;
903
904 return 0;
905}
906
907static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb)
908{
909 struct ionic_tx_stats *stats = q_to_tx_stats(q);
910 struct ionic_txq_desc *desc = q->head->desc;
911 struct device *dev = q->lif->ionic->dev;
912 dma_addr_t dma_addr;
913 bool has_vlan;
914 u8 flags = 0;
915 bool encap;
916 u64 cmd;
917
918 has_vlan = !!skb_vlan_tag_present(skb);
919 encap = skb->encapsulation;
920
921 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
922 if (dma_mapping_error(dev, dma_addr))
923 return -ENOMEM;
924
925 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
926 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
927
928 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
929 flags, skb_shinfo(skb)->nr_frags, dma_addr);
930 desc->cmd = cpu_to_le64(cmd);
931 desc->len = cpu_to_le16(skb_headlen(skb));
f64e0c56
SN		 932 if (has_vlan) {
933 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
934 stats->vlan_inserted++;
935 }
0f3154e6 936
f64e0c56 937 stats->csum_none++;
0f3154e6
SN		 938
939 return 0;
940}
941
942static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb)
943{
944 struct ionic_txq_sg_desc *sg_desc = q->head->sg_desc;
945 unsigned int len_left = skb->len - skb_headlen(skb);
946 struct ionic_txq_sg_elem *elem = sg_desc->elems;
947 struct ionic_tx_stats *stats = q_to_tx_stats(q);
948 struct device *dev = q->lif->ionic->dev;
949 dma_addr_t dma_addr;
950 skb_frag_t *frag;
951 u16 len;
952
953 for (frag = skb_shinfo(skb)->frags; len_left; frag++, elem++) {
954 len = skb_frag_size(frag);
955 elem->len = cpu_to_le16(len);
956 dma_addr = ionic_tx_map_frag(q, frag, 0, len);
957 if (dma_mapping_error(dev, dma_addr))
958 return -ENOMEM;
959 elem->addr = cpu_to_le64(dma_addr);
960 len_left -= len;
961 stats->frags++;
962 }
963
964 return 0;
965}
966
967static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
968{
969 struct ionic_tx_stats *stats = q_to_tx_stats(q);
970 int err;
971
972 /* set up the initial descriptor */
973 if (skb->ip_summed == CHECKSUM_PARTIAL)
974 err = ionic_tx_calc_csum(q, skb);
975 else
976 err = ionic_tx_calc_no_csum(q, skb);
977 if (err)
978 return err;
979
980 /* add frags */
981 err = ionic_tx_skb_frags(q, skb);
982 if (err)
983 return err;
984
985 skb_tx_timestamp(skb);
986 stats->pkts++;
987 stats->bytes += skb->len;
988
989 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
990 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
991
992 return 0;
993}
994
995static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
996{
5b3f3f2a 997 int sg_elems = q->lif->qtype_info[IONIC_QTYPE_TXQ].max_sg_elems;
0f3154e6
SN		 998 struct ionic_tx_stats *stats = q_to_tx_stats(q);
999 int err;
1000
1001 /* If TSO, need roundup(skb->len/mss) descs */
1002 if (skb_is_gso(skb))
1003 return (skb->len / skb_shinfo(skb)->gso_size) + 1;
1004
1005 /* If non-TSO, just need 1 desc and nr_frags sg elems */
5b3f3f2a 1006 if (skb_shinfo(skb)->nr_frags <= sg_elems)
0f3154e6
SN		 1007 return 1;
1008
1009 /* Too many frags, so linearize */
1010 err = skb_linearize(skb);
1011 if (err)
1012 return err;
1013
1014 stats->linearize++;
1015
1016 /* Need 1 desc and zero sg elems */
1017 return 1;
1018}
1019
1020static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)
1021{
1022 int stopped = 0;
1023
1024 if (unlikely(!ionic_q_has_space(q, ndescs))) {
1025 netif_stop_subqueue(q->lif->netdev, q->index);
1026 q->stop++;
1027 stopped = 1;
1028
1029 /* Might race with ionic_tx_clean, check again */
1030 smp_rmb();
1031 if (ionic_q_has_space(q, ndescs)) {
1032 netif_wake_subqueue(q->lif->netdev, q->index);
1033 stopped = 0;
1034 }
1035 }
1036
1037 return stopped;
1038}
1039
1040netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
1041{
1042 u16 queue_index = skb_get_queue_mapping(skb);
1043 struct ionic_lif *lif = netdev_priv(netdev);
1044 struct ionic_queue *q;
1045 int ndescs;
1046 int err;
1047
c6d3d73a 1048 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
0f3154e6
SN		 1049 dev_kfree_skb(skb);
1050 return NETDEV_TX_OK;
1051 }
1052
1053 if (unlikely(!lif_to_txqcq(lif, queue_index)))
1054 queue_index = 0;
1055 q = lif_to_txq(lif, queue_index);
1056
1057 ndescs = ionic_tx_descs_needed(q, skb);
1058 if (ndescs < 0)
1059 goto err_out_drop;
1060
1061 if (unlikely(ionic_maybe_stop_tx(q, ndescs)))
1062 return NETDEV_TX_BUSY;
1063
1064 if (skb_is_gso(skb))
1065 err = ionic_tx_tso(q, skb);
1066 else
1067 err = ionic_tx(q, skb);
1068
1069 if (err)
1070 goto err_out_drop;
1071
1072 /* Stop the queue if there aren't descriptors for the next packet.
1073 * Since our SG lists per descriptor take care of most of the possible
1074 * fragmentation, we don't need to have many descriptors available.
1075 */
1076 ionic_maybe_stop_tx(q, 4);
1077
1078 return NETDEV_TX_OK;
1079
1080err_out_drop:
1081 q->stop++;
1082 q->drop++;
1083 dev_kfree_skb(skb);
1084 return NETDEV_TX_OK;
1085}
Linux 6.x block layer and io_uring tree(s)