projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ionic: more ionic name tweaks
[linux-block.git] / drivers / net / ethernet / pensando / ionic / ionic_txrx.c
CommitLineData
0f3154e6
SN		 1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3
4#include <linux/ip.h>
5#include <linux/ipv6.h>
6#include <linux/if_vlan.h>
7#include <net/ip6_checksum.h>
8
9#include "ionic.h"
10#include "ionic_lif.h"
11#include "ionic_txrx.h"
12
5b3f3f2a
SN		 13static void ionic_rx_clean(struct ionic_queue *q,
14 struct ionic_desc_info *desc_info,
15 struct ionic_cq_info *cq_info,
16 void *cb_arg);
0f3154e6
SN		 17
18static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
19 ionic_desc_cb cb_func, void *cb_arg)
20{
21 DEBUG_STATS_TXQ_POST(q_to_qcq(q), q->head->desc, ring_dbell);
22
23 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
24}
25
26static inline void ionic_rxq_post(struct ionic_queue *q, bool ring_dbell,
27 ionic_desc_cb cb_func, void *cb_arg)
28{
29 ionic_q_post(q, ring_dbell, cb_func, cb_arg);
30
31 DEBUG_STATS_RX_BUFF_CNT(q_to_qcq(q));
32}
33
34static inline struct netdev_queue *q_to_ndq(struct ionic_queue *q)
35{
36 return netdev_get_tx_queue(q->lif->netdev, q->index);
37}
38
08f2e4b2
SN		 39static struct sk_buff *ionic_rx_skb_alloc(struct ionic_queue *q,
40 unsigned int len, bool frags)
0f3154e6 41{
08f2e4b2
SN		 42 struct ionic_lif *lif = q->lif;
43 struct ionic_rx_stats *stats;
44 struct net_device *netdev;
45 struct sk_buff *skb;
0f3154e6 46
08f2e4b2
SN		 47 netdev = lif->netdev;
48 stats = q_to_rx_stats(q);
0f3154e6 49
08f2e4b2
SN		 50 if (frags)
51 skb = napi_get_frags(&q_to_qcq(q)->napi);
52 else
53 skb = netdev_alloc_skb_ip_align(netdev, len);
54
55 if (unlikely(!skb)) {
56 net_warn_ratelimited("%s: SKB alloc failed on %s!\n",
57 netdev->name, q->name);
58 stats->alloc_err++;
59 return NULL;
60 }
61
62 return skb;
0f3154e6
SN		 63}
64
08f2e4b2
SN		 65static struct sk_buff *ionic_rx_frags(struct ionic_queue *q,
66 struct ionic_desc_info *desc_info,
67 struct ionic_cq_info *cq_info)
0f3154e6
SN		 68{
69 struct ionic_rxq_comp *comp = cq_info->cq_desc;
0f3154e6 70 struct device *dev = q->lif->ionic->dev;
08f2e4b2
SN		 71 struct ionic_page_info *page_info;
72 struct sk_buff *skb;
73 unsigned int i;
74 u16 frag_len;
75 u16 len;
0f3154e6 76
08f2e4b2
SN		 77 page_info = &desc_info->pages[0];
78 len = le16_to_cpu(comp->len);
0f3154e6 79
08f2e4b2 80 prefetch(page_address(page_info->page) + NET_IP_ALIGN);
0f3154e6 81
08f2e4b2
SN		 82 skb = ionic_rx_skb_alloc(q, len, true);
83 if (unlikely(!skb))
84 return NULL;
0f3154e6 85
08f2e4b2
SN		 86 i = comp->num_sg_elems + 1;
87 do {
88 if (unlikely(!page_info->page)) {
89 struct napi_struct *napi = &q_to_qcq(q)->napi;
0f3154e6 90
08f2e4b2
SN		 91 napi->skb = NULL;
92 dev_kfree_skb(skb);
93 return NULL;
94 }
95
96 frag_len = min(len, (u16)PAGE_SIZE);
97 len -= frag_len;
98
99 dma_unmap_page(dev, dma_unmap_addr(page_info, dma_addr),
100 PAGE_SIZE, DMA_FROM_DEVICE);
101 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
102 page_info->page, 0, frag_len, PAGE_SIZE);
103 page_info->page = NULL;
104 page_info++;
105 i--;
106 } while (i > 0);
107
108 return skb;
109}
110
111static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
112 struct ionic_desc_info *desc_info,
113 struct ionic_cq_info *cq_info)
114{
115 struct ionic_rxq_comp *comp = cq_info->cq_desc;
116 struct device *dev = q->lif->ionic->dev;
117 struct ionic_page_info *page_info;
118 struct sk_buff *skb;
119 u16 len;
120
121 page_info = &desc_info->pages[0];
122 len = le16_to_cpu(comp->len);
123
124 skb = ionic_rx_skb_alloc(q, len, false);
125 if (unlikely(!skb))
126 return NULL;
127
128 if (unlikely(!page_info->page)) {
129 dev_kfree_skb(skb);
130 return NULL;
131 }
132
133 dma_sync_single_for_cpu(dev, dma_unmap_addr(page_info, dma_addr),
134 len, DMA_FROM_DEVICE);
135 skb_copy_to_linear_data(skb, page_address(page_info->page), len);
136 dma_sync_single_for_device(dev, dma_unmap_addr(page_info, dma_addr),
137 len, DMA_FROM_DEVICE);
138
139 skb_put(skb, len);
140 skb->protocol = eth_type_trans(skb, q->lif->netdev);
141
142 return skb;
0f3154e6
SN		 143}
144
5b3f3f2a
SN		 145static void ionic_rx_clean(struct ionic_queue *q,
146 struct ionic_desc_info *desc_info,
147 struct ionic_cq_info *cq_info,
148 void *cb_arg)
0f3154e6
SN		 149{
150 struct ionic_rxq_comp *comp = cq_info->cq_desc;
151 struct ionic_qcq *qcq = q_to_qcq(q);
0f3154e6
SN		 152 struct ionic_rx_stats *stats;
153 struct net_device *netdev;
08f2e4b2 154 struct sk_buff *skb;
0f3154e6
SN		 155
156 stats = q_to_rx_stats(q);
157 netdev = q->lif->netdev;
158
24cfa8c7
SN		 159 if (comp->status) {
160 stats->dropped++;
0f3154e6 161 return;
24cfa8c7 162 }
0f3154e6 163
08f2e4b2 164 /* no packet processing while resetting */
c6d3d73a 165 if (unlikely(test_bit(IONIC_LIF_F_QUEUE_RESET, q->lif->state))) {
24cfa8c7 166 stats->dropped++;
0f3154e6 167 return;
24cfa8c7 168 }
0f3154e6
SN		 169
170 stats->pkts++;
171 stats->bytes += le16_to_cpu(comp->len);
172
08f2e4b2
SN		 173 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
174 skb = ionic_rx_copybreak(q, desc_info, cq_info);
175 else
176 skb = ionic_rx_frags(q, desc_info, cq_info);
0f3154e6 177
24cfa8c7
SN		 178 if (unlikely(!skb)) {
179 stats->dropped++;
08f2e4b2 180 return;
24cfa8c7 181 }
0f3154e6
SN		 182
183 skb_record_rx_queue(skb, q->index);
184
08f2e4b2 185 if (likely(netdev->features & NETIF_F_RXHASH)) {
0f3154e6
SN		 186 switch (comp->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
187 case IONIC_PKT_TYPE_IPV4:
188 case IONIC_PKT_TYPE_IPV6:
189 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
190 PKT_HASH_TYPE_L3);
191 break;
192 case IONIC_PKT_TYPE_IPV4_TCP:
193 case IONIC_PKT_TYPE_IPV6_TCP:
194 case IONIC_PKT_TYPE_IPV4_UDP:
195 case IONIC_PKT_TYPE_IPV6_UDP:
196 skb_set_hash(skb, le32_to_cpu(comp->rss_hash),
197 PKT_HASH_TYPE_L4);
198 break;
199 }
200 }
201
08f2e4b2 202 if (likely(netdev->features & NETIF_F_RXCSUM)) {
0f3154e6
SN		 203 if (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
204 skb->ip_summed = CHECKSUM_COMPLETE;
205 skb->csum = (__wsum)le16_to_cpu(comp->csum);
206 stats->csum_complete++;
207 }
208 } else {
209 stats->csum_none++;
210 }
211
08f2e4b2
SN		 212 if (unlikely((comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
213 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_BAD) ||
214 (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)))
0f3154e6
SN		 215 stats->csum_error++;
216
08f2e4b2 217 if (likely(netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) {
0f3154e6
SN		 218 if (comp->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN)
219 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
220 le16_to_cpu(comp->vlan_tci));
221 }
222
08f2e4b2
SN		 223 if (le16_to_cpu(comp->len) <= q->lif->rx_copybreak)
224 napi_gro_receive(&qcq->napi, skb);
225 else
226 napi_gro_frags(&qcq->napi);
0f3154e6
SN		 227}
228
229static bool ionic_rx_service(struct ionic_cq *cq, struct ionic_cq_info *cq_info)
230{
231 struct ionic_rxq_comp *comp = cq_info->cq_desc;
232 struct ionic_queue *q = cq->bound_q;
233 struct ionic_desc_info *desc_info;
234
235 if (!color_match(comp->pkt_type_color, cq->done_color))
236 return false;
237
238 /* check for empty queue */
239 if (q->tail->index == q->head->index)
240 return false;
241
242 desc_info = q->tail;
243 if (desc_info->index != le16_to_cpu(comp->comp_index))
244 return false;
245
246 q->tail = desc_info->next;
247
248 /* clean the related q entry, only one per qc completion */
249 ionic_rx_clean(q, desc_info, cq_info, desc_info->cb_arg);
250
251 desc_info->cb = NULL;
252 desc_info->cb_arg = NULL;
253
254 return true;
255}
256
257static u32 ionic_rx_walk_cq(struct ionic_cq *rxcq, u32 limit)
258{
259 u32 work_done = 0;
260
261 while (ionic_rx_service(rxcq, rxcq->tail)) {
262 if (rxcq->tail->last)
263 rxcq->done_color = !rxcq->done_color;
264 rxcq->tail = rxcq->tail->next;
265 DEBUG_STATS_CQE_CNT(rxcq);
266
267 if (++work_done >= limit)
268 break;
269 }
270
271 return work_done;
272}
273
274void ionic_rx_flush(struct ionic_cq *cq)
275{
276 struct ionic_dev *idev = &cq->lif->ionic->idev;
277 u32 work_done;
278
279 work_done = ionic_rx_walk_cq(cq, cq->num_descs);
280
281 if (work_done)
282 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
283 work_done, IONIC_INTR_CRED_RESET_COALESCE);
284}
285
08f2e4b2
SN		 286static struct page *ionic_rx_page_alloc(struct ionic_queue *q,
287 dma_addr_t *dma_addr)
0f3154e6
SN		 288{
289 struct ionic_lif *lif = q->lif;
290 struct ionic_rx_stats *stats;
291 struct net_device *netdev;
0f3154e6 292 struct device *dev;
08f2e4b2 293 struct page *page;
0f3154e6
SN		 294
295 netdev = lif->netdev;
296 dev = lif->ionic->dev;
297 stats = q_to_rx_stats(q);
08f2e4b2
SN		 298 page = alloc_page(GFP_ATOMIC);
299 if (unlikely(!page)) {
300 net_err_ratelimited("%s: Page alloc failed on %s!\n",
301 netdev->name, q->name);
0f3154e6
SN		 302 stats->alloc_err++;
303 return NULL;
304 }
305
08f2e4b2
SN		 306 *dma_addr = dma_map_page(dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
307 if (unlikely(dma_mapping_error(dev, *dma_addr))) {
308 __free_page(page);
309 net_err_ratelimited("%s: DMA single map failed on %s!\n",
310 netdev->name, q->name);
0f3154e6
SN		 311 stats->dma_map_err++;
312 return NULL;
313 }
314
08f2e4b2 315 return page;
0f3154e6
SN		 316}
317
08f2e4b2
SN		 318static void ionic_rx_page_free(struct ionic_queue *q, struct page *page,
319 dma_addr_t dma_addr)
320{
321 struct ionic_lif *lif = q->lif;
322 struct net_device *netdev;
323 struct device *dev;
324
325 netdev = lif->netdev;
326 dev = lif->ionic->dev;
327
328 if (unlikely(!page)) {
329 net_err_ratelimited("%s: Trying to free unallocated buffer on %s!\n",
330 netdev->name, q->name);
331 return;
332 }
333
334 dma_unmap_page(dev, dma_addr, PAGE_SIZE, DMA_FROM_DEVICE);
335
336 __free_page(page);
337}
338
339#define IONIC_RX_RING_DOORBELL_STRIDE ((1 << 5) - 1)
340#define IONIC_RX_RING_HEAD_BUF_SZ 2048
0f3154e6
SN		 341
342void ionic_rx_fill(struct ionic_queue *q)
343{
344 struct net_device *netdev = q->lif->netdev;
08f2e4b2
SN		 345 struct ionic_desc_info *desc_info;
346 struct ionic_page_info *page_info;
347 struct ionic_rxq_sg_desc *sg_desc;
348 struct ionic_rxq_sg_elem *sg_elem;
0f3154e6 349 struct ionic_rxq_desc *desc;
c37d6e3f
SN		 350 unsigned int remain_len;
351 unsigned int seg_len;
08f2e4b2 352 unsigned int nfrags;
0f3154e6 353 bool ring_doorbell;
08f2e4b2 354 unsigned int i, j;
0f3154e6 355 unsigned int len;
0f3154e6
SN		 356
357 len = netdev->mtu + ETH_HLEN;
08f2e4b2 358 nfrags = round_up(len, PAGE_SIZE) / PAGE_SIZE;
0f3154e6
SN		 359
360 for (i = ionic_q_space_avail(q); i; i--) {
c37d6e3f 361 remain_len = len;
08f2e4b2
SN		 362 desc_info = q->head;
363 desc = desc_info->desc;
364 sg_desc = desc_info->sg_desc;
365 page_info = &desc_info->pages[0];
366
367 if (page_info->page) { /* recycle the buffer */
368 ring_doorbell = ((q->head->index + 1) &
369 IONIC_RX_RING_DOORBELL_STRIDE) == 0;
370 ionic_rxq_post(q, ring_doorbell, ionic_rx_clean, NULL);
371 continue;
372 }
0f3154e6 373
08f2e4b2
SN		 374 /* fill main descriptor - pages[0] */
375 desc->opcode = (nfrags > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
376 IONIC_RXQ_DESC_OPCODE_SIMPLE;
377 desc_info->npages = nfrags;
378 page_info->page = ionic_rx_page_alloc(q, &page_info->dma_addr);
379 if (unlikely(!page_info->page)) {
380 desc->addr = 0;
381 desc->len = 0;
382 return;
383 }
384 desc->addr = cpu_to_le64(page_info->dma_addr);
c37d6e3f
SN		 385 seg_len = min_t(unsigned int, PAGE_SIZE, len);
386 desc->len = cpu_to_le16(seg_len);
387 remain_len -= seg_len;
08f2e4b2
SN		 388 page_info++;
389
390 /* fill sg descriptors - pages[1..n] */
391 for (j = 0; j < nfrags - 1; j++) {
392 if (page_info->page) /* recycle the sg buffer */
393 continue;
394
395 sg_elem = &sg_desc->elems[j];
396 page_info->page = ionic_rx_page_alloc(q, &page_info->dma_addr);
397 if (unlikely(!page_info->page)) {
398 sg_elem->addr = 0;
399 sg_elem->len = 0;
400 return;
401 }
402 sg_elem->addr = cpu_to_le64(page_info->dma_addr);
c37d6e3f
SN		 403 seg_len = min_t(unsigned int, PAGE_SIZE, remain_len);
404 sg_elem->len = cpu_to_le16(seg_len);
405 remain_len -= seg_len;
08f2e4b2
SN		 406 page_info++;
407 }
0f3154e6
SN		 408
409 ring_doorbell = ((q->head->index + 1) &
410 IONIC_RX_RING_DOORBELL_STRIDE) == 0;
08f2e4b2 411 ionic_rxq_post(q, ring_doorbell, ionic_rx_clean, NULL);
0f3154e6
SN		 412 }
413}
414
415static void ionic_rx_fill_cb(void *arg)
416{
417 ionic_rx_fill(arg);
418}
419
420void ionic_rx_empty(struct ionic_queue *q)
421{
0f3154e6
SN		 422 struct ionic_desc_info *cur;
423 struct ionic_rxq_desc *desc;
08f2e4b2 424 unsigned int i;
0f3154e6
SN		 425
426 for (cur = q->tail; cur != q->head; cur = cur->next) {
427 desc = cur->desc;
08f2e4b2
SN		 428 desc->addr = 0;
429 desc->len = 0;
430
08f2e4b2
SN		 431 for (i = 0; i < cur->npages; i++) {
432 if (likely(cur->pages[i].page)) {
433 ionic_rx_page_free(q, cur->pages[i].page,
434 cur->pages[i].dma_addr);
435 cur->pages[i].page = NULL;
436 cur->pages[i].dma_addr = 0;
437 }
438 }
439
0f3154e6
SN		 440 cur->cb_arg = NULL;
441 }
442}
443
444int ionic_rx_napi(struct napi_struct *napi, int budget)
445{
446 struct ionic_qcq *qcq = napi_to_qcq(napi);
447 struct ionic_cq *rxcq = napi_to_cq(napi);
448 unsigned int qi = rxcq->bound_q->index;
449 struct ionic_dev *idev;
450 struct ionic_lif *lif;
451 struct ionic_cq *txcq;
452 u32 work_done = 0;
453 u32 flags = 0;
454
455 lif = rxcq->bound_q->lif;
456 idev = &lif->ionic->idev;
457 txcq = &lif->txqcqs[qi].qcq->cq;
458
459 ionic_tx_flush(txcq);
460
461 work_done = ionic_rx_walk_cq(rxcq, budget);
462
463 if (work_done)
464 ionic_rx_fill_cb(rxcq->bound_q);
465
466 if (work_done < budget && napi_complete_done(napi, work_done)) {
467 flags |= IONIC_INTR_CRED_UNMASK;
468 DEBUG_STATS_INTR_REARM(rxcq->bound_intr);
469 }
470
471 if (work_done || flags) {
472 flags |= IONIC_INTR_CRED_RESET_COALESCE;
473 ionic_intr_credits(idev->intr_ctrl, rxcq->bound_intr->index,
474 work_done, flags);
475 }
476
477 DEBUG_STATS_NAPI_POLL(qcq, work_done);
478
479 return work_done;
480}
481
5b3f3f2a
SN		 482static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
483 void *data, size_t len)
0f3154e6
SN		 484{
485 struct ionic_tx_stats *stats = q_to_tx_stats(q);
486 struct device *dev = q->lif->ionic->dev;
487 dma_addr_t dma_addr;
488
489 dma_addr = dma_map_single(dev, data, len, DMA_TO_DEVICE);
490 if (dma_mapping_error(dev, dma_addr)) {
491 net_warn_ratelimited("%s: DMA single map failed on %s!\n",
492 q->lif->netdev->name, q->name);
493 stats->dma_map_err++;
494 return 0;
495 }
496 return dma_addr;
497}
498
5b3f3f2a
SN		 499static dma_addr_t ionic_tx_map_frag(struct ionic_queue *q,
500 const skb_frag_t *frag,
0f3154e6
SN		 501 size_t offset, size_t len)
502{
503 struct ionic_tx_stats *stats = q_to_tx_stats(q);
504 struct device *dev = q->lif->ionic->dev;
505 dma_addr_t dma_addr;
506
507 dma_addr = skb_frag_dma_map(dev, frag, offset, len, DMA_TO_DEVICE);
508 if (dma_mapping_error(dev, dma_addr)) {
509 net_warn_ratelimited("%s: DMA frag map failed on %s!\n",
510 q->lif->netdev->name, q->name);
511 stats->dma_map_err++;
512 }
513 return dma_addr;
514}
515
5b3f3f2a
SN		 516static void ionic_tx_clean(struct ionic_queue *q,
517 struct ionic_desc_info *desc_info,
518 struct ionic_cq_info *cq_info,
519 void *cb_arg)
0f3154e6
SN		 520{
521 struct ionic_txq_sg_desc *sg_desc = desc_info->sg_desc;
522 struct ionic_txq_sg_elem *elem = sg_desc->elems;
523 struct ionic_tx_stats *stats = q_to_tx_stats(q);
524 struct ionic_txq_desc *desc = desc_info->desc;
525 struct device *dev = q->lif->ionic->dev;
526 u8 opcode, flags, nsge;
527 u16 queue_index;
528 unsigned int i;
529 u64 addr;
530
531 decode_txq_desc_cmd(le64_to_cpu(desc->cmd),
532 &opcode, &flags, &nsge, &addr);
533
534 /* use unmap_single only if either this is not TSO,
535 * or this is first descriptor of a TSO
536 */
537 if (opcode != IONIC_TXQ_DESC_OPCODE_TSO ||
538 flags & IONIC_TXQ_DESC_FLAG_TSO_SOT)
539 dma_unmap_single(dev, (dma_addr_t)addr,
540 le16_to_cpu(desc->len), DMA_TO_DEVICE);
541 else
542 dma_unmap_page(dev, (dma_addr_t)addr,
543 le16_to_cpu(desc->len), DMA_TO_DEVICE);
544
545 for (i = 0; i < nsge; i++, elem++)
546 dma_unmap_page(dev, (dma_addr_t)le64_to_cpu(elem->addr),
547 le16_to_cpu(elem->len), DMA_TO_DEVICE);
548
549 if (cb_arg) {
550 struct sk_buff *skb = cb_arg;
551 u32 len = skb->len;
552
553 queue_index = skb_get_queue_mapping(skb);
554 if (unlikely(__netif_subqueue_stopped(q->lif->netdev,
555 queue_index))) {
556 netif_wake_subqueue(q->lif->netdev, queue_index);
557 q->wake++;
558 }
559 dev_kfree_skb_any(skb);
560 stats->clean++;
561 netdev_tx_completed_queue(q_to_ndq(q), 1, len);
562 }
563}
564
565void ionic_tx_flush(struct ionic_cq *cq)
566{
567 struct ionic_txq_comp *comp = cq->tail->cq_desc;
568 struct ionic_dev *idev = &cq->lif->ionic->idev;
569 struct ionic_queue *q = cq->bound_q;
570 struct ionic_desc_info *desc_info;
571 unsigned int work_done = 0;
572
573 /* walk the completed cq entries */
574 while (work_done < cq->num_descs &&
575 color_match(comp->color, cq->done_color)) {
576
577 /* clean the related q entries, there could be
578 * several q entries completed for each cq completion
579 */
580 do {
581 desc_info = q->tail;
582 q->tail = desc_info->next;
583 ionic_tx_clean(q, desc_info, cq->tail,
584 desc_info->cb_arg);
585 desc_info->cb = NULL;
586 desc_info->cb_arg = NULL;
587 } while (desc_info->index != le16_to_cpu(comp->comp_index));
588
589 if (cq->tail->last)
590 cq->done_color = !cq->done_color;
591
592 cq->tail = cq->tail->next;
593 comp = cq->tail->cq_desc;
594 DEBUG_STATS_CQE_CNT(cq);
595
596 work_done++;
597 }
598
599 if (work_done)
600 ionic_intr_credits(idev->intr_ctrl, cq->bound_intr->index,
601 work_done, 0);
602}
603
f9c00e2c
SN		 604void ionic_tx_empty(struct ionic_queue *q)
605{
606 struct ionic_desc_info *desc_info;
607 int done = 0;
608
609 /* walk the not completed tx entries, if any */
610 while (q->head != q->tail) {
611 desc_info = q->tail;
612 q->tail = desc_info->next;
613 ionic_tx_clean(q, desc_info, NULL, desc_info->cb_arg);
614 desc_info->cb = NULL;
615 desc_info->cb_arg = NULL;
616 done++;
617 }
618}
619
0f3154e6
SN		 620static int ionic_tx_tcp_inner_pseudo_csum(struct sk_buff *skb)
621{
622 int err;
623
624 err = skb_cow_head(skb, 0);
625 if (err)
626 return err;
627
628 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
629 inner_ip_hdr(skb)->check = 0;
630 inner_tcp_hdr(skb)->check =
631 ~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
632 inner_ip_hdr(skb)->daddr,
633 0, IPPROTO_TCP, 0);
634 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
635 inner_tcp_hdr(skb)->check =
636 ~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
637 &inner_ipv6_hdr(skb)->daddr,
638 0, IPPROTO_TCP, 0);
639 }
640
641 return 0;
642}
643
644static int ionic_tx_tcp_pseudo_csum(struct sk_buff *skb)
645{
646 int err;
647
648 err = skb_cow_head(skb, 0);
649 if (err)
650 return err;
651
652 if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
653 ip_hdr(skb)->check = 0;
654 tcp_hdr(skb)->check =
655 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
656 ip_hdr(skb)->daddr,
657 0, IPPROTO_TCP, 0);
658 } else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
fa6b8429 659 tcp_v6_gso_csum_prep(skb);
0f3154e6
SN		 660 }
661
662 return 0;
663}
664
665static void ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
666 struct sk_buff *skb,
667 dma_addr_t addr, u8 nsge, u16 len,
668 unsigned int hdrlen, unsigned int mss,
669 bool outer_csum,
670 u16 vlan_tci, bool has_vlan,
671 bool start, bool done)
672{
673 u8 flags = 0;
674 u64 cmd;
675
676 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
677 flags |= outer_csum ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
678 flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
679 flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
680
681 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO, flags, nsge, addr);
682 desc->cmd = cpu_to_le64(cmd);
683 desc->len = cpu_to_le16(len);
684 desc->vlan_tci = cpu_to_le16(vlan_tci);
685 desc->hdr_len = cpu_to_le16(hdrlen);
686 desc->mss = cpu_to_le16(mss);
687
688 if (done) {
689 skb_tx_timestamp(skb);
690 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
691 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
692 } else {
693 ionic_txq_post(q, false, ionic_tx_clean, NULL);
694 }
695}
696
697static struct ionic_txq_desc *ionic_tx_tso_next(struct ionic_queue *q,
698 struct ionic_txq_sg_elem **elem)
699{
700 struct ionic_txq_sg_desc *sg_desc = q->head->sg_desc;
701 struct ionic_txq_desc *desc = q->head->desc;
702
703 *elem = sg_desc->elems;
704 return desc;
705}
706
707static int ionic_tx_tso(struct ionic_queue *q, struct sk_buff *skb)
708{
709 struct ionic_tx_stats *stats = q_to_tx_stats(q);
710 struct ionic_desc_info *abort = q->head;
711 struct device *dev = q->lif->ionic->dev;
712 struct ionic_desc_info *rewind = abort;
713 struct ionic_txq_sg_elem *elem;
714 struct ionic_txq_desc *desc;
715 unsigned int frag_left = 0;
716 unsigned int offset = 0;
717 unsigned int len_left;
718 dma_addr_t desc_addr;
719 unsigned int hdrlen;
720 unsigned int nfrags;
721 unsigned int seglen;
722 u64 total_bytes = 0;
723 u64 total_pkts = 0;
724 unsigned int left;
725 unsigned int len;
726 unsigned int mss;
727 skb_frag_t *frag;
728 bool start, done;
729 bool outer_csum;
730 bool has_vlan;
731 u16 desc_len;
732 u8 desc_nsge;
733 u16 vlan_tci;
734 bool encap;
735 int err;
736
737 mss = skb_shinfo(skb)->gso_size;
738 nfrags = skb_shinfo(skb)->nr_frags;
739 len_left = skb->len - skb_headlen(skb);
740 outer_csum = (skb_shinfo(skb)->gso_type & SKB_GSO_GRE_CSUM) ||
741 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);
742 has_vlan = !!skb_vlan_tag_present(skb);
743 vlan_tci = skb_vlan_tag_get(skb);
744 encap = skb->encapsulation;
745
746 /* Preload inner-most TCP csum field with IP pseudo hdr
747 * calculated with IP length set to zero. HW will later
748 * add in length to each TCP segment resulting from the TSO.
749 */
750
751 if (encap)
752 err = ionic_tx_tcp_inner_pseudo_csum(skb);
753 else
754 err = ionic_tx_tcp_pseudo_csum(skb);
755 if (err)
756 return err;
757
758 if (encap)
759 hdrlen = skb_inner_transport_header(skb) - skb->data +
760 inner_tcp_hdrlen(skb);
761 else
762 hdrlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
763
764 seglen = hdrlen + mss;
765 left = skb_headlen(skb);
766
767 desc = ionic_tx_tso_next(q, &elem);
768 start = true;
769
770 /* Chop skb->data up into desc segments */
771
772 while (left > 0) {
773 len = min(seglen, left);
774 frag_left = seglen - len;
775 desc_addr = ionic_tx_map_single(q, skb->data + offset, len);
776 if (dma_mapping_error(dev, desc_addr))
777 goto err_out_abort;
778 desc_len = len;
779 desc_nsge = 0;
780 left -= len;
781 offset += len;
782 if (nfrags > 0 && frag_left > 0)
783 continue;
784 done = (nfrags == 0 && left == 0);
785 ionic_tx_tso_post(q, desc, skb,
786 desc_addr, desc_nsge, desc_len,
787 hdrlen, mss,
788 outer_csum,
789 vlan_tci, has_vlan,
790 start, done);
791 total_pkts++;
792 total_bytes += start ? len : len + hdrlen;
793 desc = ionic_tx_tso_next(q, &elem);
794 start = false;
795 seglen = mss;
796 }
797
798 /* Chop skb frags into desc segments */
799
800 for (frag = skb_shinfo(skb)->frags; len_left; frag++) {
801 offset = 0;
802 left = skb_frag_size(frag);
803 len_left -= left;
804 nfrags--;
805 stats->frags++;
806
807 while (left > 0) {
808 if (frag_left > 0) {
809 len = min(frag_left, left);
810 frag_left -= len;
811 elem->addr =
812 cpu_to_le64(ionic_tx_map_frag(q, frag,
813 offset, len));
814 if (dma_mapping_error(dev, elem->addr))
815 goto err_out_abort;
816 elem->len = cpu_to_le16(len);
817 elem++;
818 desc_nsge++;
819 left -= len;
820 offset += len;
821 if (nfrags > 0 && frag_left > 0)
822 continue;
823 done = (nfrags == 0 && left == 0);
824 ionic_tx_tso_post(q, desc, skb, desc_addr,
825 desc_nsge, desc_len,
826 hdrlen, mss, outer_csum,
827 vlan_tci, has_vlan,
828 start, done);
829 total_pkts++;
830 total_bytes += start ? len : len + hdrlen;
831 desc = ionic_tx_tso_next(q, &elem);
832 start = false;
833 } else {
834 len = min(mss, left);
835 frag_left = mss - len;
836 desc_addr = ionic_tx_map_frag(q, frag,
837 offset, len);
838 if (dma_mapping_error(dev, desc_addr))
839 goto err_out_abort;
840 desc_len = len;
841 desc_nsge = 0;
842 left -= len;
843 offset += len;
844 if (nfrags > 0 && frag_left > 0)
845 continue;
846 done = (nfrags == 0 && left == 0);
847 ionic_tx_tso_post(q, desc, skb, desc_addr,
848 desc_nsge, desc_len,
849 hdrlen, mss, outer_csum,
850 vlan_tci, has_vlan,
851 start, done);
852 total_pkts++;
853 total_bytes += start ? len : len + hdrlen;
854 desc = ionic_tx_tso_next(q, &elem);
855 start = false;
856 }
857 }
858 }
859
860 stats->pkts += total_pkts;
861 stats->bytes += total_bytes;
862 stats->tso++;
863
864 return 0;
865
866err_out_abort:
867 while (rewind->desc != q->head->desc) {
868 ionic_tx_clean(q, rewind, NULL, NULL);
869 rewind = rewind->next;
870 }
871 q->head = abort;
872
873 return -ENOMEM;
874}
875
876static int ionic_tx_calc_csum(struct ionic_queue *q, struct sk_buff *skb)
877{
878 struct ionic_tx_stats *stats = q_to_tx_stats(q);
879 struct ionic_txq_desc *desc = q->head->desc;
880 struct device *dev = q->lif->ionic->dev;
881 dma_addr_t dma_addr;
882 bool has_vlan;
883 u8 flags = 0;
884 bool encap;
885 u64 cmd;
886
887 has_vlan = !!skb_vlan_tag_present(skb);
888 encap = skb->encapsulation;
889
890 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
891 if (dma_mapping_error(dev, dma_addr))
892 return -ENOMEM;
893
894 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
895 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
896
897 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_PARTIAL,
898 flags, skb_shinfo(skb)->nr_frags, dma_addr);
899 desc->cmd = cpu_to_le64(cmd);
900 desc->len = cpu_to_le16(skb_headlen(skb));
901 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
902 desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
903 desc->csum_offset = cpu_to_le16(skb->csum_offset);
904
905 if (skb->csum_not_inet)
906 stats->crc32_csum++;
907 else
908 stats->csum++;
909
910 return 0;
911}
912
913static int ionic_tx_calc_no_csum(struct ionic_queue *q, struct sk_buff *skb)
914{
915 struct ionic_tx_stats *stats = q_to_tx_stats(q);
916 struct ionic_txq_desc *desc = q->head->desc;
917 struct device *dev = q->lif->ionic->dev;
918 dma_addr_t dma_addr;
919 bool has_vlan;
920 u8 flags = 0;
921 bool encap;
922 u64 cmd;
923
924 has_vlan = !!skb_vlan_tag_present(skb);
925 encap = skb->encapsulation;
926
927 dma_addr = ionic_tx_map_single(q, skb->data, skb_headlen(skb));
928 if (dma_mapping_error(dev, dma_addr))
929 return -ENOMEM;
930
931 flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
932 flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
933
934 cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
935 flags, skb_shinfo(skb)->nr_frags, dma_addr);
936 desc->cmd = cpu_to_le64(cmd);
937 desc->len = cpu_to_le16(skb_headlen(skb));
938 desc->vlan_tci = cpu_to_le16(skb_vlan_tag_get(skb));
939
940 stats->no_csum++;
941
942 return 0;
943}
944
945static int ionic_tx_skb_frags(struct ionic_queue *q, struct sk_buff *skb)
946{
947 struct ionic_txq_sg_desc *sg_desc = q->head->sg_desc;
948 unsigned int len_left = skb->len - skb_headlen(skb);
949 struct ionic_txq_sg_elem *elem = sg_desc->elems;
950 struct ionic_tx_stats *stats = q_to_tx_stats(q);
951 struct device *dev = q->lif->ionic->dev;
952 dma_addr_t dma_addr;
953 skb_frag_t *frag;
954 u16 len;
955
956 for (frag = skb_shinfo(skb)->frags; len_left; frag++, elem++) {
957 len = skb_frag_size(frag);
958 elem->len = cpu_to_le16(len);
959 dma_addr = ionic_tx_map_frag(q, frag, 0, len);
960 if (dma_mapping_error(dev, dma_addr))
961 return -ENOMEM;
962 elem->addr = cpu_to_le64(dma_addr);
963 len_left -= len;
964 stats->frags++;
965 }
966
967 return 0;
968}
969
970static int ionic_tx(struct ionic_queue *q, struct sk_buff *skb)
971{
972 struct ionic_tx_stats *stats = q_to_tx_stats(q);
973 int err;
974
975 /* set up the initial descriptor */
976 if (skb->ip_summed == CHECKSUM_PARTIAL)
977 err = ionic_tx_calc_csum(q, skb);
978 else
979 err = ionic_tx_calc_no_csum(q, skb);
980 if (err)
981 return err;
982
983 /* add frags */
984 err = ionic_tx_skb_frags(q, skb);
985 if (err)
986 return err;
987
988 skb_tx_timestamp(skb);
989 stats->pkts++;
990 stats->bytes += skb->len;
991
992 netdev_tx_sent_queue(q_to_ndq(q), skb->len);
993 ionic_txq_post(q, !netdev_xmit_more(), ionic_tx_clean, skb);
994
995 return 0;
996}
997
998static int ionic_tx_descs_needed(struct ionic_queue *q, struct sk_buff *skb)
999{
5b3f3f2a 1000 int sg_elems = q->lif->qtype_info[IONIC_QTYPE_TXQ].max_sg_elems;
0f3154e6
SN		 1001 struct ionic_tx_stats *stats = q_to_tx_stats(q);
1002 int err;
1003
1004 /* If TSO, need roundup(skb->len/mss) descs */
1005 if (skb_is_gso(skb))
1006 return (skb->len / skb_shinfo(skb)->gso_size) + 1;
1007
1008 /* If non-TSO, just need 1 desc and nr_frags sg elems */
5b3f3f2a 1009 if (skb_shinfo(skb)->nr_frags <= sg_elems)
0f3154e6
SN		 1010 return 1;
1011
1012 /* Too many frags, so linearize */
1013 err = skb_linearize(skb);
1014 if (err)
1015 return err;
1016
1017 stats->linearize++;
1018
1019 /* Need 1 desc and zero sg elems */
1020 return 1;
1021}
1022
1023static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs)
1024{
1025 int stopped = 0;
1026
1027 if (unlikely(!ionic_q_has_space(q, ndescs))) {
1028 netif_stop_subqueue(q->lif->netdev, q->index);
1029 q->stop++;
1030 stopped = 1;
1031
1032 /* Might race with ionic_tx_clean, check again */
1033 smp_rmb();
1034 if (ionic_q_has_space(q, ndescs)) {
1035 netif_wake_subqueue(q->lif->netdev, q->index);
1036 stopped = 0;
1037 }
1038 }
1039
1040 return stopped;
1041}
1042
1043netdev_tx_t ionic_start_xmit(struct sk_buff *skb, struct net_device *netdev)
1044{
1045 u16 queue_index = skb_get_queue_mapping(skb);
1046 struct ionic_lif *lif = netdev_priv(netdev);
1047 struct ionic_queue *q;
1048 int ndescs;
1049 int err;
1050
c6d3d73a 1051 if (unlikely(!test_bit(IONIC_LIF_F_UP, lif->state))) {
0f3154e6
SN		 1052 dev_kfree_skb(skb);
1053 return NETDEV_TX_OK;
1054 }
1055
1056 if (unlikely(!lif_to_txqcq(lif, queue_index)))
1057 queue_index = 0;
1058 q = lif_to_txq(lif, queue_index);
1059
1060 ndescs = ionic_tx_descs_needed(q, skb);
1061 if (ndescs < 0)
1062 goto err_out_drop;
1063
1064 if (unlikely(ionic_maybe_stop_tx(q, ndescs)))
1065 return NETDEV_TX_BUSY;
1066
1067 if (skb_is_gso(skb))
1068 err = ionic_tx_tso(q, skb);
1069 else
1070 err = ionic_tx(q, skb);
1071
1072 if (err)
1073 goto err_out_drop;
1074
1075 /* Stop the queue if there aren't descriptors for the next packet.
1076 * Since our SG lists per descriptor take care of most of the possible
1077 * fragmentation, we don't need to have many descriptors available.
1078 */
1079 ionic_maybe_stop_tx(q, 4);
1080
1081 return NETDEV_TX_OK;
1082
1083err_out_drop:
1084 q->stop++;
1085 q->drop++;
1086 dev_kfree_skb(skb);
1087 return NETDEV_TX_OK;
1088}
Linux 6.x block layer and io_uring tree(s)