[linux-2.6-block.git] / drivers / net / ethernet / broadcom / bnxt / bnxt_xdp.c

/* Broadcom NetXtreme-C/E network driver.
 *
 * Copyright (c) 2016-2017 Broadcom Limited
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation.
 */
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/if_vlan.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/filter.h>
#include <net/page_pool.h>
#include "bnxt_hsi.h"
#include "bnxt.h"
#include "bnxt_xdp.h"

DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);

struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
				   struct bnxt_tx_ring_info *txr,
				   dma_addr_t mapping, u32 len,
				   struct xdp_buff *xdp)
{
	struct skb_shared_info *sinfo;
	struct bnxt_sw_tx_bd *tx_buf;
	struct tx_bd *txbd;
	int num_frags = 0;
	u32 flags;
	u16 prod;
	int i;

	if (xdp && xdp_buff_has_frags(xdp)) {
		sinfo = xdp_get_shared_info_from_buff(xdp);
		num_frags = sinfo->nr_frags;
	}

	/* fill up the first buffer */
	prod = txr->tx_prod;
	tx_buf = &txr->tx_buf_ring[prod];
	tx_buf->nr_frags = num_frags;
	if (xdp)
		tx_buf->page = virt_to_head_page(xdp->data);

	txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
	flags = (len << TX_BD_LEN_SHIFT) |
		((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT) |
		bnxt_lhint_arr[len >> 9];
	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
	txbd->tx_bd_opaque = prod;
	txbd->tx_bd_haddr = cpu_to_le64(mapping);

	/* now let us fill up the frags into the next buffers */
	for (i = 0; i < num_frags ; i++) {
		skb_frag_t *frag = &sinfo->frags[i];
		struct bnxt_sw_tx_bd *frag_tx_buf;
		struct pci_dev *pdev = bp->pdev;
		dma_addr_t frag_mapping;
		int frag_len;

		prod = NEXT_TX(prod);
		txr->tx_prod = prod;

		/* first fill up the first buffer */
		frag_tx_buf = &txr->tx_buf_ring[prod];
		frag_tx_buf->page = skb_frag_page(frag);

		txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];

		frag_len = skb_frag_size(frag);
		frag_mapping = skb_frag_dma_map(&pdev->dev, frag, 0,
						frag_len, DMA_TO_DEVICE);

		if (unlikely(dma_mapping_error(&pdev->dev, frag_mapping)))
			return NULL;

		dma_unmap_addr_set(frag_tx_buf, mapping, frag_mapping);

		flags = frag_len << TX_BD_LEN_SHIFT;
		txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
		txbd->tx_bd_haddr = cpu_to_le64(frag_mapping);

		len = frag_len;
	}

	flags &= ~TX_BD_LEN;
	txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags |
			TX_BD_FLAGS_PACKET_END);
	/* Sync TX BD */
	wmb();
	prod = NEXT_TX(prod);
	txr->tx_prod = prod;

	return tx_buf;
}

static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
			    dma_addr_t mapping, u32 len, u16 rx_prod,
			    struct xdp_buff *xdp)
{
	struct bnxt_sw_tx_bd *tx_buf;

	tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp);
	tx_buf->rx_prod = rx_prod;
	tx_buf->action = XDP_TX;

}

static void __bnxt_xmit_xdp_redirect(struct bnxt *bp,
				     struct bnxt_tx_ring_info *txr,
				     dma_addr_t mapping, u32 len,
				     struct xdp_frame *xdpf)
{
	struct bnxt_sw_tx_bd *tx_buf;

	tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL);
	tx_buf->action = XDP_REDIRECT;
	tx_buf->xdpf = xdpf;
	dma_unmap_addr_set(tx_buf, mapping, mapping);
	dma_unmap_len_set(tx_buf, len, 0);
}

void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
{
	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
	bool rx_doorbell_needed = false;
	struct bnxt_sw_tx_bd *tx_buf;
	u16 tx_cons = txr->tx_cons;
	u16 last_tx_cons = tx_cons;
	int i, j, frags;

	for (i = 0; i < nr_pkts; i++) {
		tx_buf = &txr->tx_buf_ring[tx_cons];

		if (tx_buf->action == XDP_REDIRECT) {
			struct pci_dev *pdev = bp->pdev;

			dma_unmap_single(&pdev->dev,
					 dma_unmap_addr(tx_buf, mapping),
					 dma_unmap_len(tx_buf, len),
					 DMA_TO_DEVICE);
			xdp_return_frame(tx_buf->xdpf);
			tx_buf->action = 0;
			tx_buf->xdpf = NULL;
		} else if (tx_buf->action == XDP_TX) {
			rx_doorbell_needed = true;
			last_tx_cons = tx_cons;

			frags = tx_buf->nr_frags;
			for (j = 0; j < frags; j++) {
				tx_cons = NEXT_TX(tx_cons);
				tx_buf = &txr->tx_buf_ring[tx_cons];
				page_pool_recycle_direct(rxr->page_pool, tx_buf->page);
			}
		}
		tx_cons = NEXT_TX(tx_cons);
	}
	txr->tx_cons = tx_cons;
	if (rx_doorbell_needed) {
		tx_buf = &txr->tx_buf_ring[last_tx_cons];
		bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod);

	}
}

bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
{
	struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);

	return !!xdp_prog;
}

void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
			u16 cons, u8 **data_ptr, unsigned int *len,
			struct xdp_buff *xdp)
{
	struct bnxt_sw_rx_bd *rx_buf;
	struct pci_dev *pdev;
	dma_addr_t mapping;
	u32 offset;

	pdev = bp->pdev;
	rx_buf = &rxr->rx_buf_ring[cons];
	offset = bp->rx_offset;

	mapping = rx_buf->mapping - bp->rx_dma_offset;
	dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir);

	xdp_init_buff(xdp, BNXT_PAGE_MODE_BUF_SIZE + offset, &rxr->xdp_rxq);
	xdp_prepare_buff(xdp, *data_ptr - offset, offset, *len, false);
}

void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr,
			      struct xdp_buff *xdp)
{
	struct skb_shared_info *shinfo;
	int i;

	if (!xdp || !xdp_buff_has_frags(xdp))
		return;
	shinfo = xdp_get_shared_info_from_buff(xdp);
	for (i = 0; i < shinfo->nr_frags; i++) {
		struct page *page = skb_frag_page(&shinfo->frags[i]);

		page_pool_recycle_direct(rxr->page_pool, page);
	}
	shinfo->nr_frags = 0;
}

/* returns the following:
 * true    - packet consumed by XDP and new buffer is allocated.
 * false   - packet should be passed to the stack.
 */
bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
		 struct xdp_buff xdp, struct page *page, unsigned int *len, u8 *event)
{
	struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
	struct bnxt_tx_ring_info *txr;
	struct bnxt_sw_rx_bd *rx_buf;
	struct pci_dev *pdev;
	dma_addr_t mapping;
	u32 tx_needed = 1;
	void *orig_data;
	u32 tx_avail;
	u32 offset;
	u32 act;

	if (!xdp_prog)
		return false;

	pdev = bp->pdev;
	offset = bp->rx_offset;

	txr = rxr->bnapi->tx_ring;
	/* BNXT_RX_PAGE_MODE(bp) when XDP enabled */
	orig_data = xdp.data;

	act = bpf_prog_run_xdp(xdp_prog, &xdp);

	tx_avail = bnxt_tx_avail(bp, txr);
	/* If the tx ring is not full, we must not update the rx producer yet
	 * because we may still be transmitting on some BDs.
	 */
	if (tx_avail != bp->tx_ring_size)
		*event &= ~BNXT_RX_EVENT;

	*len = xdp.data_end - xdp.data;
	if (orig_data != xdp.data)
		offset = xdp.data - xdp.data_hard_start;

	switch (act) {
	case XDP_PASS:
		return false;

	case XDP_TX:
		rx_buf = &rxr->rx_buf_ring[cons];
		mapping = rx_buf->mapping - bp->rx_dma_offset;
		*event = 0;

		if (unlikely(xdp_buff_has_frags(&xdp))) {
			struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(&xdp);

			tx_needed += sinfo->nr_frags;
			*event = BNXT_AGG_EVENT;
		}

		if (tx_avail < tx_needed) {
			trace_xdp_exception(bp->dev, xdp_prog, act);
			bnxt_xdp_buff_frags_free(rxr, &xdp);
			bnxt_reuse_rx_data(rxr, cons, page);
			return true;
		}

		dma_sync_single_for_device(&pdev->dev, mapping + offset, *len,
					   bp->rx_dir);

		*event |= BNXT_TX_EVENT;
		__bnxt_xmit_xdp(bp, txr, mapping + offset, *len,
				NEXT_RX(rxr->rx_prod), &xdp);
		bnxt_reuse_rx_data(rxr, cons, page);
		return true;
	case XDP_REDIRECT:
		/* if we are calling this here then we know that the
		 * redirect is coming from a frame received by the
		 * bnxt_en driver.
		 */
		rx_buf = &rxr->rx_buf_ring[cons];
		mapping = rx_buf->mapping - bp->rx_dma_offset;
		dma_unmap_page_attrs(&pdev->dev, mapping,
				     PAGE_SIZE, bp->rx_dir,
				     DMA_ATTR_WEAK_ORDERING);

		/* if we are unable to allocate a new buffer, abort and reuse */
		if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) {
			trace_xdp_exception(bp->dev, xdp_prog, act);
			bnxt_xdp_buff_frags_free(rxr, &xdp);
			bnxt_reuse_rx_data(rxr, cons, page);
			return true;
		}

		if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) {
			trace_xdp_exception(bp->dev, xdp_prog, act);
			page_pool_recycle_direct(rxr->page_pool, page);
			return true;
		}

		*event |= BNXT_REDIRECT_EVENT;
		break;
	default:
		bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act);
		fallthrough;
	case XDP_ABORTED:
		trace_xdp_exception(bp->dev, xdp_prog, act);
		fallthrough;
	case XDP_DROP:
		bnxt_xdp_buff_frags_free(rxr, &xdp);
		bnxt_reuse_rx_data(rxr, cons, page);
		break;
	}
	return true;
}

int bnxt_xdp_xmit(struct net_device *dev, int num_frames,
		  struct xdp_frame **frames, u32 flags)
{
	struct bnxt *bp = netdev_priv(dev);
	struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog);
	struct pci_dev *pdev = bp->pdev;
	struct bnxt_tx_ring_info *txr;
	dma_addr_t mapping;
	int nxmit = 0;
	int ring;
	int i;

	if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
	    !bp->tx_nr_rings_xdp ||
	    !xdp_prog)
		return -EINVAL;

	ring = smp_processor_id() % bp->tx_nr_rings_xdp;
	txr = &bp->tx_ring[ring];

	if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING)
		return -EINVAL;

	if (static_branch_unlikely(&bnxt_xdp_locking_key))
		spin_lock(&txr->xdp_tx_lock);

	for (i = 0; i < num_frames; i++) {
		struct xdp_frame *xdp = frames[i];

		if (!bnxt_tx_avail(bp, txr))
			break;

		mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,
					 DMA_TO_DEVICE);

		if (dma_mapping_error(&pdev->dev, mapping))
			break;

		__bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp);
		nxmit++;
	}

	if (flags & XDP_XMIT_FLUSH) {
		/* Sync BD data before updating doorbell */
		wmb();
		bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);
	}

	if (static_branch_unlikely(&bnxt_xdp_locking_key))
		spin_unlock(&txr->xdp_tx_lock);

	return nxmit;
}

/* Under rtnl_lock */
static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)
{
	struct net_device *dev = bp->dev;
	int tx_xdp = 0, rc, tc;
	struct bpf_prog *old;

	if (prog && !prog->aux->xdp_has_frags &&
	    bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
		netdev_warn(dev, "MTU %d larger than %d without XDP frag support.\n",
			    bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU);
		return -EOPNOTSUPP;
	}
	if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) {
		netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n");
		return -EOPNOTSUPP;
	}
	if (prog)
		tx_xdp = bp->rx_nr_rings;

	tc = netdev_get_num_tc(dev);
	if (!tc)
		tc = 1;
	rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
			      true, tc, tx_xdp);
	if (rc) {
		netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n");
		return rc;
	}
	if (netif_running(dev))
		bnxt_close_nic(bp, true, false);

	old = xchg(&bp->xdp_prog, prog);
	if (old)
		bpf_prog_put(old);

	if (prog) {
		bnxt_set_rx_skb_mode(bp, true);
	} else {
		int rx, tx;

		bnxt_set_rx_skb_mode(bp, false);
		bnxt_get_max_rings(bp, &rx, &tx, true);
		if (rx > 1) {
			bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS;
			bp->dev->hw_features |= NETIF_F_LRO;
		}
	}
	bp->tx_nr_rings_xdp = tx_xdp;
	bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp;
	bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
	bnxt_set_tpa_flags(bp);
	bnxt_set_ring_params(bp);

	if (netif_running(dev))
		return bnxt_open_nic(bp, true, false);

	return 0;
}

int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp)
{
	struct bnxt *bp = netdev_priv(dev);
	int rc;

	switch (xdp->command) {
	case XDP_SETUP_PROG:
		rc = bnxt_xdp_set(bp, xdp->prog);
		break;
	default:
		rc = -EINVAL;
		break;
	}
	return rc;
}

struct sk_buff *
bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags,
		   struct page_pool *pool, struct xdp_buff *xdp,
		   struct rx_cmp_ext *rxcmp1)
{
	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);

	if (!skb)
		return NULL;
	skb_checksum_none_assert(skb);
	if (RX_CMP_L4_CS_OK(rxcmp1)) {
		if (bp->dev->features & NETIF_F_RXCSUM) {
			skb->ip_summed = CHECKSUM_UNNECESSARY;
			skb->csum_level = RX_CMP_ENCAP(rxcmp1);
		}
	}
	xdp_update_skb_shared_info(skb, num_frags,
				   sinfo->xdp_frags_size,
				   PAGE_SIZE * sinfo->nr_frags,
				   xdp_buff_is_frag_pfmemalloc(xdp));
	return skb;
}
Commit	Line	Data
c6d30e83 MC	1	/* Broadcom NetXtreme-C/E network driver.
	2	*
	3	* Copyright (c) 2016-2017 Broadcom Limited
	4	*
	5	* This program is free software; you can redistribute it and/or modify
	6	* it under the terms of the GNU General Public License as published by
	7	* the Free Software Foundation.
	8	*/
	9	#include <linux/kernel.h>
	10	#include <linux/errno.h>
	11	#include <linux/pci.h>
	12	#include <linux/netdevice.h>
	13	#include <linux/etherdevice.h>
	14	#include <linux/if_vlan.h>
	15	#include <linux/bpf.h>
	16	#include <linux/bpf_trace.h>
	17	#include <linux/filter.h>
322b87ca	18	#include <net/page_pool.h>
c6d30e83 MC	19	#include "bnxt_hsi.h"
	20	#include "bnxt.h"
	21	#include "bnxt_xdp.h"
	22
4f81def2 PC	23	DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);
4f81def2 PC	24
c1ba92a8 MC	25	struct bnxt_sw_tx_bd bnxt_xmit_bd(struct bnxt bp,
c1ba92a8 MC	26	struct bnxt_tx_ring_info *txr,
a7559bc8 AG	27	dma_addr_t mapping, u32 len,
a7559bc8 AG	28	struct xdp_buff *xdp)
38413406	29	{
a7559bc8	30	struct skb_shared_info *sinfo;
53f8c2d3	31	struct bnxt_sw_tx_bd *tx_buf;
38413406	32	struct tx_bd *txbd;
a7559bc8	33	int num_frags = 0;
38413406 MC	34	u32 flags;
38413406 MC	35	u16 prod;
a7559bc8 AG	36	int i;
	37
	38	if (xdp && xdp_buff_has_frags(xdp)) {
	39	sinfo = xdp_get_shared_info_from_buff(xdp);
	40	num_frags = sinfo->nr_frags;
	41	}
38413406	42
a7559bc8	43	/* fill up the first buffer */
38413406 MC	44	prod = txr->tx_prod;
38413406 MC	45	tx_buf = &txr->tx_buf_ring[prod];
a7559bc8 AG	46	tx_buf->nr_frags = num_frags;
	47	if (xdp)
	48	tx_buf->page = virt_to_head_page(xdp->data);
38413406 MC	49
38413406 MC	50	txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
53f8c2d3 MC	51	flags = (len << TX_BD_LEN_SHIFT) \|
	52	((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT) \|
	53	bnxt_lhint_arr[len >> 9];
38413406 MC	54	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
	55	txbd->tx_bd_opaque = prod;
	56	txbd->tx_bd_haddr = cpu_to_le64(mapping);
	57
a7559bc8 AG	58	/* now let us fill up the frags into the next buffers */
	59	for (i = 0; i < num_frags ; i++) {
	60	skb_frag_t *frag = &sinfo->frags[i];
	61	struct bnxt_sw_tx_bd *frag_tx_buf;
	62	struct pci_dev *pdev = bp->pdev;
	63	dma_addr_t frag_mapping;
	64	int frag_len;
	65
	66	prod = NEXT_TX(prod);
	67	txr->tx_prod = prod;
	68
	69	/* first fill up the first buffer */
	70	frag_tx_buf = &txr->tx_buf_ring[prod];
	71	frag_tx_buf->page = skb_frag_page(frag);
	72
	73	txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
	74
	75	frag_len = skb_frag_size(frag);
	76	frag_mapping = skb_frag_dma_map(&pdev->dev, frag, 0,
	77	frag_len, DMA_TO_DEVICE);
	78
	79	if (unlikely(dma_mapping_error(&pdev->dev, frag_mapping)))
	80	return NULL;
	81
	82	dma_unmap_addr_set(frag_tx_buf, mapping, frag_mapping);
	83
	84	flags = frag_len << TX_BD_LEN_SHIFT;
	85	txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
a7559bc8 AG	86	txbd->tx_bd_haddr = cpu_to_le64(frag_mapping);
	87
	88	len = frag_len;
	89	}
	90
	91	flags &= ~TX_BD_LEN;
	92	txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) \| flags \|
	93	TX_BD_FLAGS_PACKET_END);
	94	/* Sync TX BD */
	95	wmb();
38413406 MC	96	prod = NEXT_TX(prod);
38413406 MC	97	txr->tx_prod = prod;
a7559bc8	98
53f8c2d3	99	return tx_buf;
c1ba92a8 MC	100	}
	101
	102	static void __bnxt_xmit_xdp(struct bnxt bp, struct bnxt_tx_ring_info txr,
a7559bc8 AG	103	dma_addr_t mapping, u32 len, u16 rx_prod,
a7559bc8 AG	104	struct xdp_buff *xdp)
c1ba92a8 MC	105	{
	106	struct bnxt_sw_tx_bd *tx_buf;
	107
a7559bc8	108	tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp);
c1ba92a8 MC	109	tx_buf->rx_prod = rx_prod;
c1ba92a8 MC	110	tx_buf->action = XDP_TX;
a7559bc8	111
38413406 MC	112	}
38413406 MC	113
f18c2b77 AG	114	static void __bnxt_xmit_xdp_redirect(struct bnxt *bp,
	115	struct bnxt_tx_ring_info *txr,
	116	dma_addr_t mapping, u32 len,
	117	struct xdp_frame *xdpf)
	118	{
	119	struct bnxt_sw_tx_bd *tx_buf;
	120
a7559bc8	121	tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL);
f18c2b77 AG	122	tx_buf->action = XDP_REDIRECT;
	123	tx_buf->xdpf = xdpf;
	124	dma_unmap_addr_set(tx_buf, mapping, mapping);
	125	dma_unmap_len_set(tx_buf, len, 0);
	126	}
	127
38413406 MC	128	void bnxt_tx_int_xdp(struct bnxt bp, struct bnxt_napi bnapi, int nr_pkts)
	129	{
	130	struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
	131	struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
c1ba92a8	132	bool rx_doorbell_needed = false;
38413406 MC	133	struct bnxt_sw_tx_bd *tx_buf;
	134	u16 tx_cons = txr->tx_cons;
	135	u16 last_tx_cons = tx_cons;
a7559bc8	136	int i, j, frags;
38413406 MC	137
38413406 MC	138	for (i = 0; i < nr_pkts; i++) {
c1ba92a8 MC	139	tx_buf = &txr->tx_buf_ring[tx_cons];
c1ba92a8 MC	140
f18c2b77 AG	141	if (tx_buf->action == XDP_REDIRECT) {
	142	struct pci_dev *pdev = bp->pdev;
	143
	144	dma_unmap_single(&pdev->dev,
	145	dma_unmap_addr(tx_buf, mapping),
	146	dma_unmap_len(tx_buf, len),
df70303d	147	DMA_TO_DEVICE);
f18c2b77 AG	148	xdp_return_frame(tx_buf->xdpf);
	149	tx_buf->action = 0;
	150	tx_buf->xdpf = NULL;
	151	} else if (tx_buf->action == XDP_TX) {
c1ba92a8 MC	152	rx_doorbell_needed = true;
c1ba92a8 MC	153	last_tx_cons = tx_cons;
a7559bc8 AG	154
	155	frags = tx_buf->nr_frags;
	156	for (j = 0; j < frags; j++) {
	157	tx_cons = NEXT_TX(tx_cons);
	158	tx_buf = &txr->tx_buf_ring[tx_cons];
	159	page_pool_recycle_direct(rxr->page_pool, tx_buf->page);
	160	}
c1ba92a8	161	}
38413406	162	tx_cons = NEXT_TX(tx_cons);
38413406 MC	163	}
38413406 MC	164	txr->tx_cons = tx_cons;
c1ba92a8	165	if (rx_doorbell_needed) {
38413406	166	tx_buf = &txr->tx_buf_ring[last_tx_cons];
c1ba92a8	167	bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod);
a7559bc8	168
38413406	169	}
38413406 MC	170	}
38413406 MC	171
b231c3f3 AG	172	bool bnxt_xdp_attached(struct bnxt bp, struct bnxt_rx_ring_info rxr)
	173	{
	174	struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
	175
	176	return !!xdp_prog;
	177	}
	178
	179	void bnxt_xdp_buff_init(struct bnxt bp, struct bnxt_rx_ring_info rxr,
	180	u16 cons, u8 *data_ptr, unsigned int len,
	181	struct xdp_buff *xdp)
	182	{
	183	struct bnxt_sw_rx_bd *rx_buf;
	184	struct pci_dev *pdev;
	185	dma_addr_t mapping;
	186	u32 offset;
	187
	188	pdev = bp->pdev;
	189	rx_buf = &rxr->rx_buf_ring[cons];
	190	offset = bp->rx_offset;
	191
	192	mapping = rx_buf->mapping - bp->rx_dma_offset;
	193	dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir);
	194
	195	xdp_init_buff(xdp, BNXT_PAGE_MODE_BUF_SIZE + offset, &rxr->xdp_rxq);
	196	xdp_prepare_buff(xdp, data_ptr - offset, offset, len, false);
	197	}
	198
a7559bc8 AG	199	void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr,
	200	struct xdp_buff *xdp)
	201	{
	202	struct skb_shared_info *shinfo;
	203	int i;
	204
	205	if (!xdp \|\| !xdp_buff_has_frags(xdp))
	206	return;
	207	shinfo = xdp_get_shared_info_from_buff(xdp);
	208	for (i = 0; i < shinfo->nr_frags; i++) {
	209	struct page *page = skb_frag_page(&shinfo->frags[i]);
	210
	211	page_pool_recycle_direct(rxr->page_pool, page);
	212	}
	213	shinfo->nr_frags = 0;
	214	}
	215
c6d30e83 MC	216	/* returns the following:
	217	* true - packet consumed by XDP and new buffer is allocated.
	218	* false - packet should be passed to the stack.
	219	*/
	220	bool bnxt_rx_xdp(struct bnxt bp, struct bnxt_rx_ring_info rxr, u16 cons,
b231c3f3	221	struct xdp_buff xdp, struct page page, unsigned int len, u8 *event)
c6d30e83 MC	222	{
c6d30e83 MC	223	struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
38413406	224	struct bnxt_tx_ring_info *txr;
c6d30e83 MC	225	struct bnxt_sw_rx_bd *rx_buf;
c6d30e83 MC	226	struct pci_dev *pdev;
c6d30e83	227	dma_addr_t mapping;
a7559bc8	228	u32 tx_needed = 1;
c6d30e83	229	void *orig_data;
38413406	230	u32 tx_avail;
c6d30e83 MC	231	u32 offset;
	232	u32 act;
	233
	234	if (!xdp_prog)
	235	return false;
	236
	237	pdev = bp->pdev;
c6d30e83 MC	238	offset = bp->rx_offset;
c6d30e83 MC	239
f18c2b77	240	txr = rxr->bnapi->tx_ring;
43b5169d	241	/* BNXT_RX_PAGE_MODE(bp) when XDP enabled */
c6d30e83	242	orig_data = xdp.data;
c6d30e83	243
c6d30e83	244	act = bpf_prog_run_xdp(xdp_prog, &xdp);
c6d30e83	245
38413406 MC	246	tx_avail = bnxt_tx_avail(bp, txr);
	247	/* If the tx ring is not full, we must not update the rx producer yet
	248	* because we may still be transmitting on some BDs.
	249	*/
	250	if (tx_avail != bp->tx_ring_size)
	251	*event &= ~BNXT_RX_EVENT;
	252
b968e735	253	*len = xdp.data_end - xdp.data;
b231c3f3	254	if (orig_data != xdp.data)
c6d30e83	255	offset = xdp.data - xdp.data_hard_start;
b231c3f3	256
c6d30e83 MC	257	switch (act) {
	258	case XDP_PASS:
	259	return false;
	260
38413406	261	case XDP_TX:
b231c3f3 AG	262	rx_buf = &rxr->rx_buf_ring[cons];
b231c3f3 AG	263	mapping = rx_buf->mapping - bp->rx_dma_offset;
a7559bc8 AG	264	*event = 0;
	265
	266	if (unlikely(xdp_buff_has_frags(&xdp))) {
	267	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(&xdp);
b231c3f3	268
a7559bc8 AG	269	tx_needed += sinfo->nr_frags;
	270	*event = BNXT_AGG_EVENT;
	271	}
	272
	273	if (tx_avail < tx_needed) {
38413406	274	trace_xdp_exception(bp->dev, xdp_prog, act);
a7559bc8	275	bnxt_xdp_buff_frags_free(rxr, &xdp);
38413406 MC	276	bnxt_reuse_rx_data(rxr, cons, page);
	277	return true;
	278	}
	279
38413406 MC	280	dma_sync_single_for_device(&pdev->dev, mapping + offset, *len,
38413406 MC	281	bp->rx_dir);
a7559bc8 AG	282
a7559bc8 AG	283	*event \|= BNXT_TX_EVENT;
52c06092	284	__bnxt_xmit_xdp(bp, txr, mapping + offset, *len,
a7559bc8	285	NEXT_RX(rxr->rx_prod), &xdp);
38413406 MC	286	bnxt_reuse_rx_data(rxr, cons, page);
38413406 MC	287	return true;
f18c2b77 AG	288	case XDP_REDIRECT:
	289	/* if we are calling this here then we know that the
	290	* redirect is coming from a frame received by the
	291	* bnxt_en driver.
	292	*/
b231c3f3 AG	293	rx_buf = &rxr->rx_buf_ring[cons];
b231c3f3 AG	294	mapping = rx_buf->mapping - bp->rx_dma_offset;
f18c2b77 AG	295	dma_unmap_page_attrs(&pdev->dev, mapping,
	296	PAGE_SIZE, bp->rx_dir,
	297	DMA_ATTR_WEAK_ORDERING);
	298
	299	/* if we are unable to allocate a new buffer, abort and reuse */
	300	if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) {
	301	trace_xdp_exception(bp->dev, xdp_prog, act);
a7559bc8	302	bnxt_xdp_buff_frags_free(rxr, &xdp);
f18c2b77 AG	303	bnxt_reuse_rx_data(rxr, cons, page);
	304	return true;
	305	}
	306
	307	if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) {
	308	trace_xdp_exception(bp->dev, xdp_prog, act);
322b87ca	309	page_pool_recycle_direct(rxr->page_pool, page);
f18c2b77 AG	310	return true;
	311	}
	312
	313	*event \|= BNXT_REDIRECT_EVENT;
	314	break;
c6d30e83	315	default:
c8064e5b	316	bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act);
df561f66	317	fallthrough;
c6d30e83 MC	318	case XDP_ABORTED:
c6d30e83 MC	319	trace_xdp_exception(bp->dev, xdp_prog, act);
df561f66	320	fallthrough;
c6d30e83	321	case XDP_DROP:
a7559bc8	322	bnxt_xdp_buff_frags_free(rxr, &xdp);
c6d30e83 MC	323	bnxt_reuse_rx_data(rxr, cons, page);
	324	break;
	325	}
	326	return true;
	327	}
	328
f18c2b77 AG	329	int bnxt_xdp_xmit(struct net_device *dev, int num_frames,
	330	struct xdp_frame **frames, u32 flags)
	331	{
	332	struct bnxt *bp = netdev_priv(dev);
	333	struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog);
	334	struct pci_dev *pdev = bp->pdev;
	335	struct bnxt_tx_ring_info *txr;
	336	dma_addr_t mapping;
fdc13979	337	int nxmit = 0;
f18c2b77 AG	338	int ring;
	339	int i;
	340
	341	if (!test_bit(BNXT_STATE_OPEN, &bp->state) \|\|
	342	!bp->tx_nr_rings_xdp \|\|
	343	!xdp_prog)
	344	return -EINVAL;
	345
	346	ring = smp_processor_id() % bp->tx_nr_rings_xdp;
	347	txr = &bp->tx_ring[ring];
	348
27d4073f RJ	349	if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING)
	350	return -EINVAL;
	351
4f81def2 PC	352	if (static_branch_unlikely(&bnxt_xdp_locking_key))
	353	spin_lock(&txr->xdp_tx_lock);
	354
f18c2b77 AG	355	for (i = 0; i < num_frames; i++) {
	356	struct xdp_frame *xdp = frames[i];
	357
27d4073f	358	if (!bnxt_tx_avail(bp, txr))
fdc13979	359	break;
f18c2b77 AG	360
	361	mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,
	362	DMA_TO_DEVICE);
	363
fdc13979 LB	364	if (dma_mapping_error(&pdev->dev, mapping))
	365	break;
	366
f18c2b77	367	__bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp);
fdc13979	368	nxmit++;
f18c2b77 AG	369	}
	370
	371	if (flags & XDP_XMIT_FLUSH) {
	372	/* Sync BD data before updating doorbell */
	373	wmb();
	374	bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);
	375	}
	376
4f81def2 PC	377	if (static_branch_unlikely(&bnxt_xdp_locking_key))
	378	spin_unlock(&txr->xdp_tx_lock);
	379
fdc13979	380	return nxmit;
f18c2b77 AG	381	}
f18c2b77 AG	382
c6d30e83 MC	383	/* Under rtnl_lock */
	384	static int bnxt_xdp_set(struct bnxt bp, struct bpf_prog prog)
	385	{
	386	struct net_device *dev = bp->dev;
	387	int tx_xdp = 0, rc, tc;
	388	struct bpf_prog *old;
	389
9f4b2830 AG	390	if (prog && !prog->aux->xdp_has_frags &&
	391	bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
	392	netdev_warn(dev, "MTU %d larger than %d without XDP frag support.\n",
c6d30e83 MC	393	bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU);
	394	return -EOPNOTSUPP;
	395	}
	396	if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) {
	397	netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n");
	398	return -EOPNOTSUPP;
	399	}
	400	if (prog)
	401	tx_xdp = bp->rx_nr_rings;
	402
	403	tc = netdev_get_num_tc(dev);
	404	if (!tc)
	405	tc = 1;
98fdbe73 MC	406	rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
98fdbe73 MC	407	true, tc, tx_xdp);
c6d30e83 MC	408	if (rc) {
	409	netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n");
	410	return rc;
	411	}
	412	if (netif_running(dev))
	413	bnxt_close_nic(bp, true, false);
	414
	415	old = xchg(&bp->xdp_prog, prog);
	416	if (old)
	417	bpf_prog_put(old);
	418
	419	if (prog) {
	420	bnxt_set_rx_skb_mode(bp, true);
	421	} else {
	422	int rx, tx;
	423
	424	bnxt_set_rx_skb_mode(bp, false);
	425	bnxt_get_max_rings(bp, &rx, &tx, true);
	426	if (rx > 1) {
	427	bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS;
	428	bp->dev->hw_features \|= NETIF_F_LRO;
	429	}
	430	}
	431	bp->tx_nr_rings_xdp = tx_xdp;
	432	bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp;
	433	bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
c6d30e83 MC	434	bnxt_set_tpa_flags(bp);
	435	bnxt_set_ring_params(bp);
	436
	437	if (netif_running(dev))
	438	return bnxt_open_nic(bp, true, false);
	439
	440	return 0;
	441	}
	442
f4e63525	443	int bnxt_xdp(struct net_device dev, struct netdev_bpf xdp)
c6d30e83 MC	444	{
	445	struct bnxt *bp = netdev_priv(dev);
	446	int rc;
	447
	448	switch (xdp->command) {
	449	case XDP_SETUP_PROG:
	450	rc = bnxt_xdp_set(bp, xdp->prog);
	451	break;
c6d30e83 MC	452	default:
	453	rc = -EINVAL;
	454	break;
	455	}
	456	return rc;
	457	}
1dc4c557 AG	458
	459	struct sk_buff *
	460	bnxt_xdp_build_skb(struct bnxt bp, struct sk_buff skb, u8 num_frags,
	461	struct page_pool pool, struct xdp_buff xdp,
	462	struct rx_cmp_ext *rxcmp1)
	463	{
	464	struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
	465
	466	if (!skb)
	467	return NULL;
	468	skb_checksum_none_assert(skb);
	469	if (RX_CMP_L4_CS_OK(rxcmp1)) {
	470	if (bp->dev->features & NETIF_F_RXCSUM) {
	471	skb->ip_summed = CHECKSUM_UNNECESSARY;
	472	skb->csum_level = RX_CMP_ENCAP(rxcmp1);
	473	}
	474	}
	475	xdp_update_skb_shared_info(skb, num_frags,
	476	sinfo->xdp_frags_size,
	477	PAGE_SIZE * sinfo->nr_frags,
	478	xdp_buff_is_frag_pfmemalloc(xdp));
	479	return skb;
	480	}