projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
checkpatch: warn about uses of ENOTSUPP
[linux-block.git] / drivers / net / ethernet / pensando / ionic / ionic_txrx.c
CommitLineData
0f3154e6
SN		 1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3
4#include <linux/ip.h>
5#include <linux/ipv6.h>
6#include <linux/if_vlan.h>
7#include <net/ip6_checksum.h>
8
9#include "ionic.h"
10#include "ionic_lif.h"
11#include "ionic_txrx.h"
12
13static void ionic_rx_clean(struct ionic_queue *q, struct ionic_desc_info *desc_info,
14 struct ionic_cq_info *cq_info, void *cb_arg);
15
16static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
17 ionic_desc_cb cb_func, void *cb_arg)
18{
19 DEBUG_STATS_TXQ_POST(q_to_qcq(q), q->head->desc, ring_dbell);
20
21 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
22}
23
24static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
25 ionic_desc_cb cb_func, void *cb_arg)
26{
27 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
28
29 DEBUG_STATS_RX_BUFF_CNT(q_to_qcq(q));
30}
31
32static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
33{
34 return netdev_get_tx_queue(q->lif->netdev, q->index);
35}
36
08f2e4b2
SN		 37static struct sk_buff *ionic_rx_skb_alloc(struct ionic_queue *q,
38 unsigned int len, bool frags)
0f3154e6 39{
08f2e4b2
SN		 40 struct ionic_lif *lif = q->lif;
41 struct ionic_rx_stats *stats;
42 struct net_device *netdev;
43 struct sk_buff *skb;
0f3154e6 44
08f2e4b2
SN		 45 netdev = lif->netdev;
46 stats = q_to_rx_stats(q);
0f3154e6 47
08f2e4b2
SN		 48 if (frags)
49 skb = napi_get_frags(&q_to_qcq(q)->napi);
50 else
51 skb = netdev_alloc_skb_ip_align(netdev, len);
52
53 if (unlikely(!skb)) {
54 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
55 netdev->name, q->name);
56 stats->alloc_err++;
57 return NULL;
58 }
59
60 return skb;
0f3154e6
SN		 61}
62
08f2e4b2
SN		 63static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
64 struct ionic_desc_info *desc_info,
65 struct ionic_cq_info *cq_info)
0f3154e6
SN		 66{
67 struct ionic_rxq_comp *comp = cq_info->cq_desc;
0f3154e6 68 struct device *dev = q->lif->ionic->dev;
08f2e4b2
SN		 69 struct ionic_page_info *page_info;
70 struct sk_buff *skb;
71 unsigned int i;
72 u16 frag_len;
73 u16 len;
0f3154e6 74
08f2e4b2
SN		 75 page_info = &desc_info->pages[0];
76 len = le16_to_cpu(comp->len);
0f3154e6 77
08f2e4b2 78 prefetch(page_address(page_info->page) + NET_IP_ALIGN);
0f3154e6 79
08f2e4b2
SN		 80 skb = ionic_rx_skb_alloc(q, len, true);
81 if (unlikely(!skb))
82 return NULL;
0f3154e6 83
08f2e4b2
SN		 84 i = comp->num_sg_elems + 1;
85 do {
86 if (unlikely(!page_info->page)) {
87 struct napi_struct *napi = &q_to_qcq(q)->napi;
0f3154e6 88
08f2e4b2
SN		 89 napi->skb = NULL;
90 dev_kfree_skb(skb);
91 return NULL;
92 }
93
94 frag_len = min(len, (u16)PAGE_SIZE);
95 len -= frag_len;
96
97 dma_unmap_page(dev, dma_unmap_addr(page_info, dma_addr),
98 PAGE_SIZE, DMA_FROM_DEVICE);
99 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
100 page_info->page, 0, frag_len, PAGE_SIZE);
101 page_info->page = NULL;
102 page_info++;
103 i--;
104 } while (i > 0);
105
106 return skb;
107}
108
109static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
110 struct ionic_desc_info *desc_info,
111 struct ionic_cq_info *cq_info)
112{
113 struct ionic_rxq_comp *comp = cq_info->cq_desc;
114 struct device *dev = q->lif->ionic->dev;
115 struct ionic_page_info *page_info;
116 struct sk_buff *skb;
117 u16 len;
118
119 page_info = &desc_info->pages[0];
120 len = le16_to_cpu(comp->len);
121
122 skb = ionic_rx_skb_alloc(q, len, false);
123 if (unlikely(!skb))
124 return NULL;
125
126 if (unlikely(!page_info->page)) {
127 dev_kfree_skb(skb);
128 return NULL;
129 }
130
131 dma_sync_single_for_cpu(dev, dma_unmap_addr(page_info, dma_addr),
132 len, DMA_FROM_DEVICE);
133 skb_copy_to_linear_data(skb, page_address(page_info->page), len);
134 dma_sync_single_for_device(dev, dma_unmap_addr(page_info, dma_addr),
135 len, DMA_FROM_DEVICE);
136
137 skb_put(skb, len);
138 skb->protocol = eth_type_trans(skb, q->lif->netdev);
139
140 return skb;
0f3154e6
SN		 141}
142
143static void ionic_rx_clean(struct ionic_queue *q, struct ionic_desc_info *desc_info,
144 struct ionic_cq_info *cq_info, void *cb_arg)
145{
146 struct ionic_rxq_comp *comp = cq_info->cq_desc;
147 struct ionic_qcq *qcq = q_to_qcq(q);
0f3154e6
SN		 148 struct ionic_rx_stats *stats;
149 struct net_device *netdev;
08f2e4b2 150 struct sk_buff *skb;
0f3154e6
SN		 151
152 stats = q_to_rx_stats(q);
153 netdev = q->lif->netdev;
154
24cfa8c7
SN		 155 if (comp->status) {
156 stats->dropped++;
0f3154e6 157 return;
24cfa8c7 158 }
0f3154e6 159
08f2e4b2 160 /* no packet processing while resetting */
c6d3d73a 161 if (unlikely(test_bit(IONIC_LIF_F_QUEUE_RESET, q->lif->state))) {
24cfa8c7 162 stats->dropped++;
0f3154e6 163 return;
24cfa8c7 164 }
0f3154e6
SN		 165
166 stats->pkts++;
167 stats->bytes += le16_to_cpu(comp->len);
168
08f2e4b2
SN		 169 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
170 skb = ionic_rx_copybreak(q, desc_info, cq_info);
171 else
172 skb = ionic_rx_frags(q, desc_info, cq_info);
0f3154e6 173
24cfa8c7
SN		 174 if (unlikely(!skb)) {
175 stats->dropped++;
08f2e4b2 176 return;
24cfa8c7 177 }
0f3154e6
SN		 178
179 skb_record_rx_queue(skb, q->index);
180
08f2e4b2 181 if (likely(netdev->features & NETIF_F_RXHASH)) {
0f3154e6
SN		 182 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
183 case IONIC_PKT_TYPE_IPV4:
184 case IONIC_PKT_TYPE_IPV6:
185 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
186 PKT_HASH_TYPE_L3);
187 break;
188 case IONIC_PKT_TYPE_IPV4_TCP:
189 case IONIC_PKT_TYPE_IPV6_TCP:
190 case IONIC_PKT_TYPE_IPV4_UDP:
191 case IONIC_PKT_TYPE_IPV6_UDP:
192 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
193 PKT_HASH_TYPE_L4);
194 break;
195 }
196 }
197
08f2e4b2 198 if (likely(netdev->features & NETIF_F_RXCSUM)) {
0f3154e6
SN		 199 if (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
200 skb->ip_summed = CHECKSUM_COMPLETE;
201 skb->csum = (__wsum)le16_to_cpu(comp->csum);
202 stats->csum_complete++;
203 }
204 } else {
205 stats->csum_none++;
206 }
207
08f2e4b2
SN		 208 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
209 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
210 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
0f3154e6
SN		 211 stats->csum_error++;
212
08f2e4b2 213 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
0f3154e6
SN		 214 if (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)
215 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
216 le16_to_cpu(comp->vlan_tci));
217 }
218
08f2e4b2
SN		 219 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
220 napi_gro_receive(&qcq->napi, skb);
221 else
222 napi_gro_frags(&qcq->napi);
0f3154e6
SN		 223}
224
225static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
226{
227 struct ionic_rxq_comp *comp = cq_info->cq_desc;
228 struct ionic_queue *q = cq->bound_q;
229 struct ionic_desc_info *desc_info;
230
231 if (!color_match(comp->pkt_type_color, cq->done_color))
232 return false;
233
234 /* check for empty queue */
235 if (q->tail->index == q->head->index)
236 return false;
237
238 desc_info = q->tail;
239 if (desc_info->index != le16_to_cpu(comp->comp_index))
240 return false;
241
242 q->tail = desc_info->next;
243
244 /* clean the related q entry, only one per qc completion */
245 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
246
247 desc_info->cb = NULL;
248 desc_info->cb_arg = NULL;
249
250 return true;
251}
252
253static u32 ionic_rx_walk_cq(struct ionic_cq *rxcq, u32 limit)
254{
255 u32 work_done = 0;
256
257 while (ionic_rx_service(rxcq, rxcq->tail)) {
258 if (rxcq->tail->last)
259 rxcq->done_color = !rxcq->done_color;
260 rxcq->tail = rxcq->tail->next;
261 DEBUG_STATS_CQE_CNT(rxcq);
262
263 if (++work_done >= limit)
264 break;
265 }
266
267 return work_done;
268}
269
270void ionic_rx_flush(struct ionic_cq *cq)
271{
272 struct ionic_dev *idev = &cq->lif->ionic->idev;
273 u32 work_done;
274
275 work_done = ionic_rx_walk_cq(cq, cq->num_descs);
276
277 if (work_done)
278 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
279 work_done, IONIC_INTR_CRED_RESET_COALESCE);
280}
281
08f2e4b2
SN		 282static struct page *ionic_rx_page_alloc(struct ionic_queue *q,
283 dma_addr_t *dma_addr)
0f3154e6
SN		 284{
285 struct ionic_lif *lif = q->lif;
286 struct ionic_rx_stats *stats;
287 struct net_device *netdev;
0f3154e6 288 struct device *dev;
08f2e4b2 289 struct page *page;
0f3154e6
SN		 290
291 netdev = lif->netdev;
292 dev = lif->ionic->dev;
293 stats = q_to_rx_stats(q);
08f2e4b2
SN		 294 page = alloc_page(GFP_ATOMIC);
295 if (unlikely(!page)) {
296 net_err_ratelimited("%s: Page alloc failed on %s!\n",
297 netdev->name, q->name);
0f3154e6
SN		 298 stats->alloc_err++;
299 return NULL;
300 }
301
08f2e4b2
SN		 302 *dma_addr = dma_map_page(dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
303 if (unlikely(dma_mapping_error(dev, *dma_addr))) {
304 __free_page(page);
305 net_err_ratelimited("%s: DMA single map failed on %s!\n",
306 netdev->name, q->name);
0f3154e6
SN		 307 stats->dma_map_err++;
308 return NULL;
309 }
310
08f2e4b2 311 return page;
0f3154e6
SN		 312}
313
08f2e4b2
SN		 314static void ionic_rx_page_free(struct ionic_queue *q, struct page *page,
315 dma_addr_t dma_addr)
316{
317 struct ionic_lif *lif = q->lif;
318 struct net_device *netdev;
319 struct device *dev;
320
321 netdev = lif->netdev;
322 dev = lif->ionic->dev;
323
324 if (unlikely(!page)) {
325 net_err_ratelimited("%s: Trying to free unallocated buffer on %s!\n",
326 netdev->name, q->name);
327 return;
328 }
329
330 dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE);
331
332 __free_page(page);
333}
334
335#define IONIC_RX_RING_DOORBELL_STRIDE ((1 << 5) - 1)
336#define IONIC_RX_RING_HEAD_BUF_SZ 2048
0f3154e6
SN		 337
338void ionic_rx_fill(struct ionic_queue *q)
339{
340 struct net_device *netdev = q->lif->netdev;
08f2e4b2
SN		 341 struct ionic_desc_info *desc_info;
342 struct ionic_page_info *page_info;
343 struct ionic_rxq_sg_desc *sg_desc;
344 struct ionic_rxq_sg_elem *sg_elem;
0f3154e6 345 struct ionic_rxq_desc *desc;
c37d6e3f
SN		 346 unsigned int remain_len;
347 unsigned int seg_len;
08f2e4b2 348 unsigned int nfrags;
0f3154e6 349 bool ring_doorbell;
08f2e4b2 350 unsigned int i, j;
0f3154e6 351 unsigned int len;
0f3154e6
SN		 352
353 len = netdev->mtu + ETH_HLEN;
08f2e4b2 354 nfrags = round_up(len, PAGE_SIZE) / PAGE_SIZE;
0f3154e6
SN		 355
356 for (i = ionic_q_space_avail(q); i; i--) {
c37d6e3f 357 remain_len = len;
08f2e4b2
SN		 358 desc_info = q->head;
359 desc = desc_info->desc;
360 sg_desc = desc_info->sg_desc;
361 page_info = &desc_info->pages[0];
362
363 if (page_info->page) { /* recycle the buffer */
364 ring_doorbell = ((q->head->index + 1) &
365 IONIC_RX_RING_DOORBELL_STRIDE) == 0;
366 ionic_rxq_post(q, ring_doorbell, ionic_rx_clean, NULL);
367 continue;
368 }
0f3154e6 369
08f2e4b2
SN		 370 /* fill main descriptor - pages[0] */
371 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
372 IONIC_RXQ_DESC_OPCODE_SIMPLE;
373 desc_info->npages = nfrags;
374 page_info->page = ionic_rx_page_alloc(q, &page_info->dma_addr);
375 if (unlikely(!page_info->page)) {
376 desc->addr = 0;
377 desc->len = 0;
378 return;
379 }
380 desc->addr = cpu_to_le64(page_info->dma_addr);
c37d6e3f
SN		 381 seg_len = min_t(unsigned int, PAGE_SIZE, len);
382 desc->len = cpu_to_le16(seg_len);
383 remain_len -= seg_len;
08f2e4b2
SN		 384 page_info++;
385
386 /* fill sg descriptors - pages[1..n] */
387 for (j = 0; j < nfrags - 1; j++) {
388 if (page_info->page) /* recycle the sg buffer */
389 continue;
390
391 sg_elem = &sg_desc->elems[j];
392 page_info->page = ionic_rx_page_alloc(q, &page_info->dma_addr);
393 if (unlikely(!page_info->page)) {
394 sg_elem->addr = 0;
395 sg_elem->len = 0;
396 return;
397 }
398 sg_elem->addr = cpu_to_le64(page_info->dma_addr);
c37d6e3f
SN		 399 seg_len = min_t(unsigned int, PAGE_SIZE, remain_len);
400 sg_elem->len = cpu_to_le16(seg_len);
401 remain_len -= seg_len;
08f2e4b2
SN		 402 page_info++;
403 }
0f3154e6
SN		 404
405 ring_doorbell = ((q->head->index + 1) &
406 IONIC_RX_RING_DOORBELL_STRIDE) == 0;
08f2e4b2 407 ionic_rxq_post(q, ring_doorbell, ionic_rx_clean, NULL);
0f3154e6
SN		 408 }
409}
410
411static void ionic_rx_fill_cb(void *arg)
412{
413 ionic_rx_fill(arg);
414}
415
416void ionic_rx_empty(struct ionic_queue *q)
417{
0f3154e6
SN		 418 struct ionic_desc_info *cur;
419 struct ionic_rxq_desc *desc;
08f2e4b2 420 unsigned int i;
0f3154e6
SN		 421
422 for (cur = q->tail; cur != q->head; cur = cur->next) {
423 desc = cur->desc;
08f2e4b2
SN		 424 desc->addr = 0;
425 desc->len = 0;
426
08f2e4b2
SN		 427 for (i = 0; i < cur->npages; i++) {
428 if (likely(cur->pages[i].page)) {
429 ionic_rx_page_free(q, cur->pages[i].page,
430 cur->pages[i].dma_addr);
431 cur->pages[i].page = NULL;
432 cur->pages[i].dma_addr = 0;
433 }
434 }
435
0f3154e6
SN		 436 cur->cb_arg = NULL;
437 }
438}
439
440int ionic_rx_napi(struct napi_struct *napi, int budget)
441{
442 struct ionic_qcq *qcq = napi_to_qcq(napi);
443 struct ionic_cq *rxcq = napi_to_cq(napi);
444 unsigned int qi = rxcq->bound_q->index;
445 struct ionic_dev *idev;
446 struct ionic_lif *lif;
447 struct ionic_cq *txcq;
448 u32 work_done = 0;
449 u32 flags = 0;
450
451 lif = rxcq->bound_q->lif;
452 idev = &lif->ionic->idev;
453 txcq = &lif->txqcqs[qi].qcq->cq;
454
455 ionic_tx_flush(txcq);
456
457 work_done = ionic_rx_walk_cq(rxcq, budget);
458
459 if (work_done)
460 ionic_rx_fill_cb(rxcq->bound_q);
461
462 if (work_done < budget && napi_complete_done(napi, work_done)) {
463 flags |= IONIC_INTR_CRED_UNMASK;
464 DEBUG_STATS_INTR_REARM(rxcq->bound_intr);
465 }
466
467 if (work_done || flags) {
468 flags |= IONIC_INTR_CRED_RESET_COALESCE;
469 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
470 work_done, flags);
471 }
472
473 DEBUG_STATS_NAPI_POLL(qcq, work_done);
474
475 return work_done;
476}
477
478static dma_addr_t ionic_tx_map_single(struct ionic_queue *q, void *data, size_t len)
479{
480 struct ionic_tx_stats *stats = q_to_tx_stats(q);
481 struct device *dev = q->lif->ionic->dev;
482 dma_addr_t dma_addr;
483
484 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
485 if (dma_mapping_error(dev, dma_addr)) {
486 net_warn_ratelimited("%s: DMA single map failed on %s!\n",
487 q->lif->netdev->name, q->name);
488 stats->dma_map_err++;
489 return 0;
490 }
491 return dma_addr;
492}
493
494static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q, const skb_frag_t *frag,
495 size_t offset, size_t len)
496{
497 struct ionic_tx_stats *stats = q_to_tx_stats(q);
498 struct device *dev = q->lif->ionic->dev;
499 dma_addr_t dma_addr;
500
501 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
502 if (dma_mapping_error(dev, dma_addr)) {
503 net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
504 q->lif->netdev->name, q->name);
505 stats->dma_map_err++;
506 }
507 return dma_addr;
508}
509
510static void ionic_tx_clean(struct ionic_queue *q, struct ionic_desc_info *desc_info,
511 struct ionic_cq_info *cq_info, void *cb_arg)
512{
513 struct ionic_txq_sg_desc *sg_desc = desc_info->sg_desc;
514 struct ionic_txq_sg_elem *elem = sg_desc->elems;
515 struct ionic_tx_stats *stats = q_to_tx_stats(q);
516 struct ionic_txq_desc *desc = desc_info->desc;
517 struct device *dev = q->lif->ionic->dev;
518 u8 opcode, flags, nsge;
519 u16 queue_index;
520 unsigned int i;
521 u64 addr;
522
523 decode_txq_desc_cmd(le64_to_cpu(desc->cmd),
524 &opcode, &flags, &nsge, &addr);
525
526 /* use unmap_single only if either this is not TSO,
527 * or this is first descriptor of a TSO
528 */
529 if (opcode != IONIC_TXQ_DESC_OPCODE_TSO ||
530 flags & IONIC_TXQ_DESC_FLAG_TSO_SOT)
531 dma_unmap_single(dev, (dma_addr_t)addr,
532 le16_to_cpu(desc->len), DMA_TO_DEVICE);
533 else
534 dma_unmap_page(dev, (dma_addr_t)addr,
535 le16_to_cpu(desc->len), DMA_TO_DEVICE);
536
537 for (i = 0; i < nsge; i++, elem++)
538 dma_unmap_page(dev, (dma_addr_t)le64_to_cpu(elem->addr),
539 le16_to_cpu(elem->len), DMA_TO_DEVICE);
540
541 if (cb_arg) {
542 struct sk_buff *skb = cb_arg;
543 u32 len = skb->len;
544
545 queue_index = skb_get_queue_mapping(skb);
546 if (unlikely(__netif_subqueue_stopped(q->lif->netdev,
547 queue_index))) {
548 netif_wake_subqueue(q->lif->netdev, queue_index);
549 q->wake++;
550 }
551 dev_kfree_skb_any(skb);
552 stats->clean++;
553 netdev_tx_completed_queue(q_to_ndq(q), 1, len);
554 }
555}
556
557void ionic_tx_flush(struct ionic_cq *cq)
558{
559 struct ionic_txq_comp *comp = cq->tail->cq_desc;
560 struct ionic_dev *idev = &cq->lif->ionic->idev;
561 struct ionic_queue *q = cq->bound_q;
562 struct ionic_desc_info *desc_info;
563 unsigned int work_done = 0;
564
565 /* walk the completed cq entries */
566 while (work_done < cq->num_descs &&
567 color_match(comp->color, cq->done_color)) {
568
569 /* clean the related q entries, there could be
570 * several q entries completed for each cq completion
571 */
572 do {
573 desc_info = q->tail;
574 q->tail = desc_info->next;
575 ionic_tx_clean(q, desc_info, cq->tail,
576 desc_info->cb_arg);
577 desc_info->cb = NULL;
578 desc_info->cb_arg = NULL;
579 } while (desc_info->index != le16_to_cpu(comp->comp_index));
580
581 if (cq->tail->last)
582 cq->done_color = !cq->done_color;
583
584 cq->tail = cq->tail->next;
585 comp = cq->tail->cq_desc;
586 DEBUG_STATS_CQE_CNT(cq);
587
588 work_done++;
589 }
590
591 if (work_done)
592 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
593 work_done, 0);
594}
595
f9c00e2c
SN		 596void ionic_tx_empty(struct ionic_queue *q)
597{
598 struct ionic_desc_info *desc_info;
599 int done = 0;
600
601 /* walk the not completed tx entries, if any */
602 while (q->head != q->tail) {
603 desc_info = q->tail;
604 q->tail = desc_info->next;
605 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
606 desc_info->cb = NULL;
607 desc_info->cb_arg = NULL;
608 done++;
609 }
610}
611
0f3154e6
SN		 612static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
613{
614 int err;
615
616 err = skb_cow_head(skb, 0);
617 if (err)
618 return err;
619
620 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
621 inner_ip_hdr(skb)->check = 0;
622 inner_tcp_hdr(skb)->check =
623 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
624 inner_ip_hdr(skb)->daddr,
625 0, IPPROTO_TCP, 0);
626 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
627 inner_tcp_hdr(skb)->check =
628 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
629 &inner_ipv6_hdr(skb)->daddr,
630 0, IPPROTO_TCP, 0);
631 }
632
633 return 0;
634}
635
636static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
637{
638 int err;
639
640 err = skb_cow_head(skb, 0);
641 if (err)
642 return err;
643
644 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
645 ip_hdr(skb)->check = 0;
646 tcp_hdr(skb)->check =
647 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
648 ip_hdr(skb)->daddr,
649 0, IPPROTO_TCP, 0);
650 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
fa6b8429 651 tcp_v6_gso_csum_prep(skb);
0f3154e6
SN		 652 }
653
654 return 0;
655}
656
657static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
658 struct sk_buff *skb,
659 dma_addr_t addr, u8 nsge, u16 len,
660 unsigned int hdrlen, unsigned int mss,
661 bool outer_csum,
662 u16 vlan_tci, bool has_vlan,
663 bool start, bool done)
664{
665 u8 flags = 0;
666 u64 cmd;
667
668 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
669 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
670 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
671 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
672
673 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
674 desc->cmd = cpu_to_le64(cmd);
675 desc->len = cpu_to_le16(len);
676 desc->vlan_tci = cpu_to_le16(vlan_tci);
677 desc->hdr_len = cpu_to_le16(hdrlen);
678 desc->mss = cpu_to_le16(mss);
679
680 if (done) {
681 skb_tx_timestamp(skb);
682 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
683 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
684 } else {
685 ionic_txq_post(q, false, ionic_tx_clean, NULL);
686 }
687}
688
689static struct ionic_txq_desc *ionic_tx_tso_next(struct ionic_queue *q,
690 struct ionic_txq_sg_elem **elem)
691{
692 struct ionic_txq_sg_desc *sg_desc = q->head->sg_desc;
693 struct ionic_txq_desc *desc = q->head->desc;
694
695 *elem = sg_desc->elems;
696 return desc;
697}
698
699static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
700{
701 struct ionic_tx_stats *stats = q_to_tx_stats(q);
702 struct ionic_desc_info *abort = q->head;
703 struct device *dev = q->lif->ionic->dev;
704 struct ionic_desc_info *rewind = abort;
705 struct ionic_txq_sg_elem *elem;
706 struct ionic_txq_desc *desc;
707 unsigned int frag_left = 0;
708 unsigned int offset = 0;
709 unsigned int len_left;
710 dma_addr_t desc_addr;
711 unsigned int hdrlen;
712 unsigned int nfrags;
713 unsigned int seglen;
714 u64 total_bytes = 0;
715 u64 total_pkts = 0;
716 unsigned int left;
717 unsigned int len;
718 unsigned int mss;
719 skb_frag_t *frag;
720 bool start, done;
721 bool outer_csum;
722 bool has_vlan;
723 u16 desc_len;
724 u8 desc_nsge;
725 u16 vlan_tci;
726 bool encap;
727 int err;
728
729 mss = skb_shinfo(skb)->gso_size;
730 nfrags = skb_shinfo(skb)->nr_frags;
731 len_left = skb->len - skb_headlen(skb);
732 outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) ||
733 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
734 has_vlan = !!skb_vlan_tag_present(skb);
735 vlan_tci = skb_vlan_tag_get(skb);
736 encap = skb->encapsulation;
737
738 /* Preload inner-most TCP csum field with IP pseudo hdr
739 * calculated with IP length set to zero. HW will later
740 * add in length to each TCP segment resulting from the TSO.
741 */
742
743 if (encap)
744 err = ionic_tx_tcp_inner_pseudo_csum(skb);
745 else
746 err = ionic_tx_tcp_pseudo_csum(skb);
747 if (err)
748 return err;
749
750 if (encap)
751 hdrlen = skb_inner_transport_header(skb) - skb->data +
752 inner_tcp_hdrlen(skb);
753 else
754 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
755
756 seglen = hdrlen + mss;
757 left = skb_headlen(skb);
758
759 desc = ionic_tx_tso_next(q, &elem);
760 start = true;
761
762 /* Chop skb->data up into desc segments */
763
764 while (left > 0) {
765 len = min(seglen, left);
766 frag_left = seglen - len;
767 desc_addr = ionic_tx_map_single(q, skb->data + offset, len);
768 if (dma_mapping_error(dev, desc_addr))
769 goto err_out_abort;
770 desc_len = len;
771 desc_nsge = 0;
772 left -= len;
773 offset += len;
774 if (nfrags > 0 && frag_left > 0)
775 continue;
776 done = (nfrags == 0 && left == 0);
777 ionic_tx_tso_post(q, desc, skb,
778 desc_addr, desc_nsge, desc_len,
779 hdrlen, mss,
780 outer_csum,
781 vlan_tci, has_vlan,
782 start, done);
783 total_pkts++;
784 total_bytes += start ? len : len + hdrlen;
785 desc = ionic_tx_tso_next(q, &elem);
786 start = false;
787 seglen = mss;
788 }
789
790 /* Chop skb frags into desc segments */
791
792 for (frag = skb_shinfo(skb)->frags; len_left; frag++) {
793 offset = 0;
794 left = skb_frag_size(frag);
795 len_left -= left;
796 nfrags--;
797 stats->frags++;
798
799 while (left > 0) {
800 if (frag_left > 0) {
801 len = min(frag_left, left);
802 frag_left -= len;
803 elem->addr =
804 cpu_to_le64(ionic_tx_map_frag(q, frag,
805 offset, len));
806 if (dma_mapping_error(dev, elem->addr))
807 goto err_out_abort;
808 elem->len = cpu_to_le16(len);
809 elem++;
810 desc_nsge++;
811 left -= len;
812 offset += len;
813 if (nfrags > 0 && frag_left > 0)
814 continue;
815 done = (nfrags == 0 && left == 0);
816 ionic_tx_tso_post(q, desc, skb, desc_addr,
817 desc_nsge, desc_len,
818 hdrlen, mss, outer_csum,
819 vlan_tci, has_vlan,
820 start, done);
821 total_pkts++;
822 total_bytes += start ? len : len + hdrlen;
823 desc = ionic_tx_tso_next(q, &elem);
824 start = false;
825 } else {
826 len = min(mss, left);
827 frag_left = mss - len;
828 desc_addr = ionic_tx_map_frag(q, frag,
829 offset, len);
830 if (dma_mapping_error(dev, desc_addr))
831 goto err_out_abort;
832 desc_len = len;
833 desc_nsge = 0;
834 left -= len;
835 offset += len;
836 if (nfrags > 0 && frag_left > 0)
837 continue;
838 done = (nfrags == 0 && left == 0);
839 ionic_tx_tso_post(q, desc, skb, desc_addr,
840 desc_nsge, desc_len,
841 hdrlen, mss, outer_csum,
842 vlan_tci, has_vlan,
843 start, done);
844 total_pkts++;
845 total_bytes += start ? len : len + hdrlen;
846 desc = ionic_tx_tso_next(q, &elem);
847 start = false;
848 }
849 }
850 }
851
852 stats->pkts += total_pkts;
853 stats->bytes += total_bytes;
854 stats->tso++;
855
856 return 0;
857
858err_out_abort:
859 while (rewind->desc != q->head->desc) {
860 ionic_tx_clean(q, rewind, NULL, NULL);
861 rewind = rewind->next;
862 }
863 q->head = abort;
864
865 return -ENOMEM;
866}
867
868static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb)
869{
870 struct ionic_tx_stats *stats = q_to_tx_stats(q);
871 struct ionic_txq_desc *desc = q->head->desc;
872 struct device *dev = q->lif->ionic->dev;
873 dma_addr_t dma_addr;
874 bool has_vlan;
875 u8 flags = 0;
876 bool encap;
877 u64 cmd;
878
879 has_vlan = !!skb_vlan_tag_present(skb);
880 encap = skb->encapsulation;
881
882 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
883 if (dma_mapping_error(dev, dma_addr))
884 return -ENOMEM;
885
886 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
887 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
888
889 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
890 flags, skb_shinfo(skb)->nr_frags, dma_addr);
891 desc->cmd = cpu_to_le64(cmd);
892 desc->len = cpu_to_le16(skb_headlen(skb));
893 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
894 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
895 desc->csum_offset = cpu_to_le16(skb->csum_offset);
896
897 if (skb->csum_not_inet)
898 stats->crc32_csum++;
899 else
900 stats->csum++;
901
902 return 0;
903}
904
905static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb)
906{
907 struct ionic_tx_stats *stats = q_to_tx_stats(q);
908 struct ionic_txq_desc *desc = q->head->desc;
909 struct device *dev = q->lif->ionic->dev;
910 dma_addr_t dma_addr;
911 bool has_vlan;
912 u8 flags = 0;
913 bool encap;
914 u64 cmd;
915
916 has_vlan = !!skb_vlan_tag_present(skb);
917 encap = skb->encapsulation;
918
919 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
920 if (dma_mapping_error(dev, dma_addr))
921 return -ENOMEM;
922
923 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
924 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
925
926 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
927 flags, skb_shinfo(skb)->nr_frags, dma_addr);
928 desc->cmd = cpu_to_le64(cmd);
929 desc->len = cpu_to_le16(skb_headlen(skb));
930 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
931
932 stats->no_csum++;
933
934 return 0;
935}
936
937static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb)
938{
939 struct ionic_txq_sg_desc *sg_desc = q->head->sg_desc;
940 unsigned int len_left = skb->len - skb_headlen(skb);
941 struct ionic_txq_sg_elem *elem = sg_desc->elems;
942 struct ionic_tx_stats *stats = q_to_tx_stats(q);
943 struct device *dev = q->lif->ionic->dev;
944 dma_addr_t dma_addr;
945 skb_frag_t *frag;
946 u16 len;
947
948 for (frag = skb_shinfo(skb)->frags; len_left; frag++, elem++) {
949 len = skb_frag_size(frag);
950 elem->len = cpu_to_le16(len);
951 dma_addr = ionic_tx_map_frag(q, frag, 0, len);
952 if (dma_mapping_error(dev, dma_addr))
953 return -ENOMEM;
954 elem->addr = cpu_to_le64(dma_addr);
955 len_left -= len;
956 stats->frags++;
957 }
958
959 return 0;
960}
961
962static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
963{
964 struct ionic_tx_stats *stats = q_to_tx_stats(q);
965 int err;
966
967 /* set up the initial descriptor */
968 if (skb->ip_summed == CHECKSUM_PARTIAL)
969 err = ionic_tx_calc_csum(q, skb);
970 else
971 err = ionic_tx_calc_no_csum(q, skb);
972 if (err)
973 return err;
974
975 /* add frags */
976 err = ionic_tx_skb_frags(q, skb);
977 if (err)
978 return err;
979
980 skb_tx_timestamp(skb);
981 stats->pkts++;
982 stats->bytes += skb->len;
983
984 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
985 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
986
987 return 0;
988}
989
990static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
991{
992 struct ionic_tx_stats *stats = q_to_tx_stats(q);
993 int err;
994
995 /* If TSO, need roundup(skb->len/mss) descs */
996 if (skb_is_gso(skb))
997 return (skb->len / skb_shinfo(skb)->gso_size) + 1;
998
999 /* If non-TSO, just need 1 desc and nr_frags sg elems */
1000 if (skb_shinfo(skb)->nr_frags <= IONIC_TX_MAX_SG_ELEMS)
1001 return 1;
1002
1003 /* Too many frags, so linearize */
1004 err = skb_linearize(skb);
1005 if (err)
1006 return err;
1007
1008 stats->linearize++;
1009
1010 /* Need 1 desc and zero sg elems */
1011 return 1;
1012}
1013
1014static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)
1015{
1016 int stopped = 0;
1017
1018 if (unlikely(!ionic_q_has_space(q, ndescs))) {
1019 netif_stop_subqueue(q->lif->netdev, q->index);
1020 q->stop++;
1021 stopped = 1;
1022
1023 /* Might race with ionic_tx_clean, check again */
1024 smp_rmb();
1025 if (ionic_q_has_space(q, ndescs)) {
1026 netif_wake_subqueue(q->lif->netdev, q->index);
1027 stopped = 0;
1028 }
1029 }
1030
1031 return stopped;
1032}
1033
1034netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
1035{
1036 u16 queue_index = skb_get_queue_mapping(skb);
1037 struct ionic_lif *lif = netdev_priv(netdev);
1038 struct ionic_queue *q;
1039 int ndescs;
1040 int err;
1041
c6d3d73a 1042 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
0f3154e6
SN		 1043 dev_kfree_skb(skb);
1044 return NETDEV_TX_OK;
1045 }
1046
1047 if (unlikely(!lif_to_txqcq(lif, queue_index)))
1048 queue_index = 0;
1049 q = lif_to_txq(lif, queue_index);
1050
1051 ndescs = ionic_tx_descs_needed(q, skb);
1052 if (ndescs < 0)
1053 goto err_out_drop;
1054
1055 if (unlikely(ionic_maybe_stop_tx(q, ndescs)))
1056 return NETDEV_TX_BUSY;
1057
1058 if (skb_is_gso(skb))
1059 err = ionic_tx_tso(q, skb);
1060 else
1061 err = ionic_tx(q, skb);
1062
1063 if (err)
1064 goto err_out_drop;
1065
1066 /* Stop the queue if there aren't descriptors for the next packet.
1067 * Since our SG lists per descriptor take care of most of the possible
1068 * fragmentation, we don't need to have many descriptors available.
1069 */
1070 ionic_maybe_stop_tx(q, 4);
1071
1072 return NETDEV_TX_OK;
1073
1074err_out_drop:
1075 q->stop++;
1076 q->drop++;
1077 dev_kfree_skb(skb);
1078 return NETDEV_TX_OK;
1079}
Linux 6.x block layer and io_uring tree(s)