octeontx2-pf: AF_XDP zero copy transmit support

This patch implements below changes,

1. To avoid concurrency with normal traffic uses
   XDP queues.
2. Since there are chances that XDP and AF_XDP can
   fall under same queue uses separate flags to handle
   dma buffers.

Signed-off-by: Hariprasad Kelam <hkelam@marvell.com>
Signed-off-by: Suman Ghosh <sumang@marvell.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index 6f7c11d59942f181a457af0547b2f5d15874503b..84cd029a85aab7e47afe46ad98daa7c0c9bed56c 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -1037,6 +1037,10 @@ int otx2_sq_init(struct otx2_nic *pfvf, u16 qidx, u16 sqb_aura)
 
        sq->stats.bytes = 0;
        sq->stats.pkts = 0;
+       /* Attach XSK_BUFF_POOL to XDP queue */
+       if (qidx > pfvf->hw.xdp_queues)
+               otx2_attach_xsk_buff(pfvf, sq, (qidx - pfvf->hw.xdp_queues));
+
 
        chan_offset = qidx % pfvf->hw.tx_chan_cnt;
        err = pfvf->hw_ops->sq_aq_init(pfvf, qidx, chan_offset, sqb_aura);

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
index 19e9e2e72233314a370df5cd85b6e4b3c995cdc1..1e88422825be7af5192709908c688deac0a077fe 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h
@@ -129,6 +129,12 @@ enum otx2_errcodes_re {
        ERRCODE_IL4_CSUM = 0x22,
 };
 
+enum otx2_xdp_action {
+       OTX2_XDP_TX       = BIT(0),
+       OTX2_XDP_REDIRECT = BIT(1),
+       OTX2_AF_XDP_FRAME = BIT(2),
+};
+
 struct otx2_dev_stats {
        u64 rx_bytes;
        u64 rx_frames;

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
index 50a42cd5d50a2c8e87596b8f116992afd88d6543..c7c562f0f5e5ce7d240261f9957d447c9621d67c 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c
@@ -2693,7 +2693,7 @@ static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
                return -ENOMEM;
 
        err = otx2_xdp_sq_append_pkt(pf, xdpf, dma_addr, xdpf->len,
-                                    qidx, XDP_REDIRECT);
+                                    qidx, OTX2_XDP_REDIRECT);
        if (!err) {
                otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);
                xdp_return_frame(xdpf);

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
index 9a6c1f1a3ee0aaf941e23229f2701e5130b062ec..6bc5ce5a9f61cc2830975e7cc5219344760b0e51 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
@@ -20,6 +20,7 @@
 #include "otx2_txrx.h"
 #include "otx2_ptp.h"
 #include "cn10k.h"
+#include "otx2_xsk.h"
 
 #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
 #define PTP_PORT               0x13F
@@ -103,13 +104,19 @@ static unsigned int frag_num(unsigned int i)
 
 static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
                                     struct otx2_snd_queue *sq,
-                                    struct nix_cqe_tx_s *cqe)
+                                    struct nix_cqe_tx_s *cqe,
+                                    int *xsk_frames)
 {
        struct nix_send_comp_s *snd_comp = &cqe->comp;
        struct sg_list *sg;
 
        sg = &sq->sg[snd_comp->sqe_id];
-       if (sg->flags & XDP_REDIRECT)
+       if (sg->flags & OTX2_AF_XDP_FRAME) {
+               (*xsk_frames)++;
+               return;
+       }
+
+       if (sg->flags & OTX2_XDP_REDIRECT)
                otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE);
        xdp_return_frame((struct xdp_frame *)sg->skb);
        sg->skb = (u64)NULL;
@@ -434,6 +441,18 @@ int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
        return cnt - cq->pool_ptrs;
 }
 
+static void otx2_zc_submit_pkts(struct otx2_nic *pfvf, struct xsk_buff_pool *xsk_pool,
+                               int *xsk_frames, int qidx, int budget)
+{
+       if (*xsk_frames)
+               xsk_tx_completed(xsk_pool, *xsk_frames);
+
+       if (xsk_uses_need_wakeup(xsk_pool))
+               xsk_set_tx_need_wakeup(xsk_pool);
+
+       otx2_zc_napi_handler(pfvf, xsk_pool, qidx, budget);
+}
+
 static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
                                struct otx2_cq_queue *cq, int budget)
 {
@@ -442,16 +461,22 @@ static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
        struct nix_cqe_tx_s *cqe;
        struct net_device *ndev;
        int processed_cqe = 0;
+       int xsk_frames = 0;
+
+       qidx = cq->cq_idx - pfvf->hw.rx_queues;
+       sq = &pfvf->qset.sq[qidx];
 
        if (cq->pend_cqe >= budget)
                goto process_cqe;
 
-       if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
+       if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) {
+               if (sq->xsk_pool)
+                       otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames,
+                                           qidx, budget);
                return 0;
+       }
 
 process_cqe:
-       qidx = cq->cq_idx - pfvf->hw.rx_queues;
-       sq = &pfvf->qset.sq[qidx];
 
        while (likely(processed_cqe < budget) && cq->pend_cqe) {
                cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
@@ -461,10 +486,8 @@ process_cqe:
                        break;
                }
 
-               qidx = cq->cq_idx - pfvf->hw.rx_queues;
-
                if (cq->cq_type == CQ_XDP)
-                       otx2_xdp_snd_pkt_handler(pfvf, sq, cqe);
+                       otx2_xdp_snd_pkt_handler(pfvf, sq, cqe, &xsk_frames);
                else
                        otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx],
                                             cqe, budget, &tx_pkts, &tx_bytes);
@@ -505,6 +528,10 @@ process_cqe:
                    netif_carrier_ok(ndev))
                        netif_tx_wake_queue(txq);
        }
+
+       if (sq->xsk_pool)
+               otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, qidx, budget);
+
        return 0;
 }
 
@@ -1499,8 +1526,10 @@ handle_xdp_verdict:
                qidx += pfvf->hw.tx_queues;
                cq->pool_ptrs++;
                xdpf = xdp_convert_buff_to_frame(&xdp);
-               return otx2_xdp_sq_append_pkt(pfvf, xdpf, cqe->sg.seg_addr,
-                                             cqe->sg.seg_size, qidx, XDP_TX);
+               return otx2_xdp_sq_append_pkt(pfvf, xdpf,
+                                             cqe->sg.seg_addr,
+                                             cqe->sg.seg_size,
+                                             qidx, OTX2_XDP_TX);
        case XDP_REDIRECT:
                cq->pool_ptrs++;
                if (xsk_buff) {

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
index 8f346fbc8221fae89cc815c5dd1d338ea359761b..acf259d7200883e12ebf75f0af58005096936c23 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.h
@@ -106,6 +106,8 @@ struct otx2_snd_queue {
        /* SQE ring and CPT response queue for Inline IPSEC */
        struct qmem             *sqe_ring;
        struct qmem             *cpt_resp;
+       /* Buffer pool for af_xdp zero-copy */
+       struct xsk_buff_pool    *xsk_pool;
 } ____cacheline_aligned_in_smp;
 
 enum cq_type {

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
index 3bdee1603facc04b275c60fbcae0412baefce677..ce10caea8511dea981aa7aa4ab01ace92e8b1e1f 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c
@@ -140,11 +140,14 @@ int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx)
 {
        struct net_device *netdev = pf->netdev;
        struct xsk_buff_pool *pool;
+       struct otx2_snd_queue *sq;
 
        pool = xsk_get_pool_from_qid(netdev, qidx);
        if (!pool)
                return -EINVAL;
 
+       sq = &pf->qset.sq[qidx + pf->hw.tx_queues];
+       sq->xsk_pool = NULL;
        otx2_clean_up_rq(pf, qidx);
        clear_bit(qidx, pf->af_xdp_zc_qidx);
        xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
@@ -171,7 +174,7 @@ int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
        if (pf->flags & OTX2_FLAG_INTF_DOWN)
                return -ENETDOWN;
 
-       if (queue_id >= pf->hw.rx_queues)
+       if (queue_id >= pf->hw.rx_queues || queue_id >= pf->hw.tx_queues)
                return -EINVAL;
 
        cq_poll = &qset->napi[queue_id];
@@ -179,8 +182,44 @@ int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
                return -EINVAL;
 
        /* Trigger interrupt */
-       if (!napi_if_scheduled_mark_missed(&cq_poll->napi))
+       if (!napi_if_scheduled_mark_missed(&cq_poll->napi)) {
                otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0));
+               otx2_write64(pf, NIX_LF_CINTX_INT_W1S(cq_poll->cint_idx), BIT_ULL(0));
+       }
 
        return 0;
 }
+
+void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx)
+{
+       if (test_bit(qidx, pfvf->af_xdp_zc_qidx))
+               sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx);
+}
+
+void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
+                         int queue, int budget)
+{
+       struct xdp_desc *xdp_desc = pool->tx_descs;
+       int err, i, work_done = 0, batch;
+
+       budget = min(budget, otx2_read_free_sqe(pfvf, queue));
+       batch = xsk_tx_peek_release_desc_batch(pool, budget);
+       if (!batch)
+               return;
+
+       for (i = 0; i < batch; i++) {
+               dma_addr_t dma_addr;
+
+               dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr);
+               err = otx2_xdp_sq_append_pkt(pfvf, NULL, dma_addr, xdp_desc[i].len,
+                                            queue, OTX2_AF_XDP_FRAME);
+               if (!err) {
+                       netdev_err(pfvf->netdev, "AF_XDP: Unable to transfer packet err%d\n", err);
+                       break;
+               }
+               work_done++;
+       }
+
+       if (work_done)
+               xsk_tx_release(pool);
+}

diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
index 022b3433edbbb5f026ea64ff3f2d86e0aeb6136d..8047fafee8febc70b23bba9fe0e183a8dc03b177 100644 (file)
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.h
@@ -17,5 +17,8 @@ int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qid);
 int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
                            dma_addr_t *dma, int idx);
 int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags);
+void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
+                         int queue, int budget);
+void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx);
 
 #endif /* OTX2_XSK_H */