drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2016-2017 Broadcom Limited
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  */
   9 #include <linux/kernel.h>
  10 #include <linux/errno.h>
  11 #include <linux/pci.h>
  12 #include <linux/netdevice.h>
  13 #include <linux/etherdevice.h>
  14 #include <linux/if_vlan.h>
  15 #include <linux/bpf.h>
  16 #include <linux/bpf_trace.h>
  17 #include <linux/filter.h>
  18 #include <net/page_pool.h>
  19 #include "bnxt_hsi.h"
  20 #include "bnxt.h"
  21 #include "bnxt_xdp.h"
  22
  23 DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key);
  24
  25 struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp,
  26                                    struct bnxt_tx_ring_info *txr,
  27                                    dma_addr_t mapping, u32 len,
  28                                    struct xdp_buff *xdp)
  29 {
  30         struct skb_shared_info *sinfo;
  31         struct bnxt_sw_tx_bd *tx_buf, *first_buf;
  32         struct tx_bd *txbd;
  33         int num_frags = 0;
  34         u32 flags;
  35         u16 prod;
  36         int i;
  37
  38         if (xdp && xdp_buff_has_frags(xdp)) {
  39                 sinfo = xdp_get_shared_info_from_buff(xdp);
  40                 num_frags = sinfo->nr_frags;
  41         }
  42
  43         /* fill up the first buffer */
  44         prod = txr->tx_prod;
  45         tx_buf = &txr->tx_buf_ring[prod];
  46         first_buf = tx_buf;
  47         tx_buf->nr_frags = num_frags;
  48         if (xdp)
  49                 tx_buf->page = virt_to_head_page(xdp->data);
  50
  51         txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
  52         flags = ((len) << TX_BD_LEN_SHIFT) | ((num_frags + 1) << TX_BD_FLAGS_BD_CNT_SHIFT);
  53         txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
  54         txbd->tx_bd_opaque = prod;
  55         txbd->tx_bd_haddr = cpu_to_le64(mapping);
  56
  57         /* now let us fill up the frags into the next buffers */
  58         for (i = 0; i < num_frags ; i++) {
  59                 skb_frag_t *frag = &sinfo->frags[i];
  60                 struct bnxt_sw_tx_bd *frag_tx_buf;
  61                 struct pci_dev *pdev = bp->pdev;
  62                 dma_addr_t frag_mapping;
  63                 int frag_len;
  64
  65                 prod = NEXT_TX(prod);
  66                 txr->tx_prod = prod;
  67
  68                 /* first fill up the first buffer */
  69                 frag_tx_buf = &txr->tx_buf_ring[prod];
  70                 frag_tx_buf->page = skb_frag_page(frag);
  71
  72                 txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)];
  73
  74                 frag_len = skb_frag_size(frag);
  75                 frag_mapping = skb_frag_dma_map(&pdev->dev, frag, 0,
  76                                                 frag_len, DMA_TO_DEVICE);
  77
  78                 if (unlikely(dma_mapping_error(&pdev->dev, frag_mapping)))
  79                         return NULL;
  80
  81                 dma_unmap_addr_set(frag_tx_buf, mapping, frag_mapping);
  82
  83                 flags = frag_len << TX_BD_LEN_SHIFT;
  84                 txbd->tx_bd_len_flags_type = cpu_to_le32(flags);
  85                 txbd->tx_bd_opaque = prod;
  86                 txbd->tx_bd_haddr = cpu_to_le64(frag_mapping);
  87
  88                 len = frag_len;
  89         }
  90
  91         flags &= ~TX_BD_LEN;
  92         txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags |
  93                         TX_BD_FLAGS_PACKET_END);
  94         /* Sync TX BD */
  95         wmb();
  96         prod = NEXT_TX(prod);
  97         txr->tx_prod = prod;
  98
  99         return first_buf;
 100 }
 101
 102 static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr,
 103                             dma_addr_t mapping, u32 len, u16 rx_prod,
 104                             struct xdp_buff *xdp)
 105 {
 106         struct bnxt_sw_tx_bd *tx_buf;
 107
 108         tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp);
 109         tx_buf->rx_prod = rx_prod;
 110         tx_buf->action = XDP_TX;
 111
 112 }
 113
 114 static void __bnxt_xmit_xdp_redirect(struct bnxt *bp,
 115                                      struct bnxt_tx_ring_info *txr,
 116                                      dma_addr_t mapping, u32 len,
 117                                      struct xdp_frame *xdpf)
 118 {
 119         struct bnxt_sw_tx_bd *tx_buf;
 120
 121         tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL);
 122         tx_buf->action = XDP_REDIRECT;
 123         tx_buf->xdpf = xdpf;
 124         dma_unmap_addr_set(tx_buf, mapping, mapping);
 125         dma_unmap_len_set(tx_buf, len, 0);
 126 }
 127
 128 void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts)
 129 {
 130         struct bnxt_tx_ring_info *txr = bnapi->tx_ring;
 131         struct bnxt_rx_ring_info *rxr = bnapi->rx_ring;
 132         bool rx_doorbell_needed = false;
 133         struct bnxt_sw_tx_bd *tx_buf;
 134         u16 tx_cons = txr->tx_cons;
 135         u16 last_tx_cons = tx_cons;
 136         int i, j, frags;
 137
 138         for (i = 0; i < nr_pkts; i++) {
 139                 tx_buf = &txr->tx_buf_ring[tx_cons];
 140
 141                 if (tx_buf->action == XDP_REDIRECT) {
 142                         struct pci_dev *pdev = bp->pdev;
 143
 144                         dma_unmap_single(&pdev->dev,
 145                                          dma_unmap_addr(tx_buf, mapping),
 146                                          dma_unmap_len(tx_buf, len),
 147                                          DMA_TO_DEVICE);
 148                         xdp_return_frame(tx_buf->xdpf);
 149                         tx_buf->action = 0;
 150                         tx_buf->xdpf = NULL;
 151                 } else if (tx_buf->action == XDP_TX) {
 152                         rx_doorbell_needed = true;
 153                         last_tx_cons = tx_cons;
 154
 155                         frags = tx_buf->nr_frags;
 156                         for (j = 0; j < frags; j++) {
 157                                 tx_cons = NEXT_TX(tx_cons);
 158                                 tx_buf = &txr->tx_buf_ring[tx_cons];
 159                                 page_pool_recycle_direct(rxr->page_pool, tx_buf->page);
 160                         }
 161                 }
 162                 tx_cons = NEXT_TX(tx_cons);
 163         }
 164         txr->tx_cons = tx_cons;
 165         if (rx_doorbell_needed) {
 166                 tx_buf = &txr->tx_buf_ring[last_tx_cons];
 167                 bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod);
 168
 169         }
 170 }
 171
 172 bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr)
 173 {
 174         struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
 175
 176         return !!xdp_prog;
 177 }
 178
 179 void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr,
 180                         u16 cons, u8 **data_ptr, unsigned int *len,
 181                         struct xdp_buff *xdp)
 182 {
 183         struct bnxt_sw_rx_bd *rx_buf;
 184         struct pci_dev *pdev;
 185         dma_addr_t mapping;
 186         u32 offset;
 187
 188         pdev = bp->pdev;
 189         rx_buf = &rxr->rx_buf_ring[cons];
 190         offset = bp->rx_offset;
 191
 192         mapping = rx_buf->mapping - bp->rx_dma_offset;
 193         dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir);
 194
 195         xdp_init_buff(xdp, BNXT_PAGE_MODE_BUF_SIZE + offset, &rxr->xdp_rxq);
 196         xdp_prepare_buff(xdp, *data_ptr - offset, offset, *len, false);
 197 }
 198
 199 void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr,
 200                               struct xdp_buff *xdp)
 201 {
 202         struct skb_shared_info *shinfo;
 203         int i;
 204
 205         if (!xdp || !xdp_buff_has_frags(xdp))
 206                 return;
 207         shinfo = xdp_get_shared_info_from_buff(xdp);
 208         for (i = 0; i < shinfo->nr_frags; i++) {
 209                 struct page *page = skb_frag_page(&shinfo->frags[i]);
 210
 211                 page_pool_recycle_direct(rxr->page_pool, page);
 212         }
 213         shinfo->nr_frags = 0;
 214 }
 215
 216 /* returns the following:
 217  * true    - packet consumed by XDP and new buffer is allocated.
 218  * false   - packet should be passed to the stack.
 219  */
 220 bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons,
 221                  struct xdp_buff xdp, struct page *page, unsigned int *len, u8 *event)
 222 {
 223         struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog);
 224         struct bnxt_tx_ring_info *txr;
 225         struct bnxt_sw_rx_bd *rx_buf;
 226         struct pci_dev *pdev;
 227         dma_addr_t mapping;
 228         u32 tx_needed = 1;
 229         void *orig_data;
 230         u32 tx_avail;
 231         u32 offset;
 232         u32 act;
 233
 234         if (!xdp_prog)
 235                 return false;
 236
 237         pdev = bp->pdev;
 238         offset = bp->rx_offset;
 239
 240         txr = rxr->bnapi->tx_ring;
 241         /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */
 242         orig_data = xdp.data;
 243
 244         act = bpf_prog_run_xdp(xdp_prog, &xdp);
 245
 246         tx_avail = bnxt_tx_avail(bp, txr);
 247         /* If the tx ring is not full, we must not update the rx producer yet
 248          * because we may still be transmitting on some BDs.
 249          */
 250         if (tx_avail != bp->tx_ring_size)
 251                 *event &= ~BNXT_RX_EVENT;
 252
 253         *len = xdp.data_end - xdp.data;
 254         if (orig_data != xdp.data)
 255                 offset = xdp.data - xdp.data_hard_start;
 256
 257         switch (act) {
 258         case XDP_PASS:
 259                 return false;
 260
 261         case XDP_TX:
 262                 rx_buf = &rxr->rx_buf_ring[cons];
 263                 mapping = rx_buf->mapping - bp->rx_dma_offset;
 264                 *event = 0;
 265
 266                 if (unlikely(xdp_buff_has_frags(&xdp))) {
 267                         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(&xdp);
 268
 269                         tx_needed += sinfo->nr_frags;
 270                         *event = BNXT_AGG_EVENT;
 271                 }
 272
 273                 if (tx_avail < tx_needed) {
 274                         trace_xdp_exception(bp->dev, xdp_prog, act);
 275                         bnxt_xdp_buff_frags_free(rxr, &xdp);
 276                         bnxt_reuse_rx_data(rxr, cons, page);
 277                         return true;
 278                 }
 279
 280                 dma_sync_single_for_device(&pdev->dev, mapping + offset, *len,
 281                                            bp->rx_dir);
 282
 283                 *event |= BNXT_TX_EVENT;
 284                 __bnxt_xmit_xdp(bp, txr, mapping + offset, *len,
 285                                 NEXT_RX(rxr->rx_prod), &xdp);
 286                 bnxt_reuse_rx_data(rxr, cons, page);
 287                 return true;
 288         case XDP_REDIRECT:
 289                 /* if we are calling this here then we know that the
 290                  * redirect is coming from a frame received by the
 291                  * bnxt_en driver.
 292                  */
 293                 rx_buf = &rxr->rx_buf_ring[cons];
 294                 mapping = rx_buf->mapping - bp->rx_dma_offset;
 295                 dma_unmap_page_attrs(&pdev->dev, mapping,
 296                                      PAGE_SIZE, bp->rx_dir,
 297                                      DMA_ATTR_WEAK_ORDERING);
 298
 299                 /* if we are unable to allocate a new buffer, abort and reuse */
 300                 if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) {
 301                         trace_xdp_exception(bp->dev, xdp_prog, act);
 302                         bnxt_xdp_buff_frags_free(rxr, &xdp);
 303                         bnxt_reuse_rx_data(rxr, cons, page);
 304                         return true;
 305                 }
 306
 307                 if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) {
 308                         trace_xdp_exception(bp->dev, xdp_prog, act);
 309                         page_pool_recycle_direct(rxr->page_pool, page);
 310                         return true;
 311                 }
 312
 313                 *event |= BNXT_REDIRECT_EVENT;
 314                 break;
 315         default:
 316                 bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act);
 317                 fallthrough;
 318         case XDP_ABORTED:
 319                 trace_xdp_exception(bp->dev, xdp_prog, act);
 320                 fallthrough;
 321         case XDP_DROP:
 322                 bnxt_xdp_buff_frags_free(rxr, &xdp);
 323                 bnxt_reuse_rx_data(rxr, cons, page);
 324                 break;
 325         }
 326         return true;
 327 }
 328
 329 int bnxt_xdp_xmit(struct net_device *dev, int num_frames,
 330                   struct xdp_frame **frames, u32 flags)
 331 {
 332         struct bnxt *bp = netdev_priv(dev);
 333         struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog);
 334         struct pci_dev *pdev = bp->pdev;
 335         struct bnxt_tx_ring_info *txr;
 336         dma_addr_t mapping;
 337         int nxmit = 0;
 338         int ring;
 339         int i;
 340
 341         if (!test_bit(BNXT_STATE_OPEN, &bp->state) ||
 342             !bp->tx_nr_rings_xdp ||
 343             !xdp_prog)
 344                 return -EINVAL;
 345
 346         ring = smp_processor_id() % bp->tx_nr_rings_xdp;
 347         txr = &bp->tx_ring[ring];
 348
 349         if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING)
 350                 return -EINVAL;
 351
 352         if (static_branch_unlikely(&bnxt_xdp_locking_key))
 353                 spin_lock(&txr->xdp_tx_lock);
 354
 355         for (i = 0; i < num_frames; i++) {
 356                 struct xdp_frame *xdp = frames[i];
 357
 358                 if (!bnxt_tx_avail(bp, txr))
 359                         break;
 360
 361                 mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len,
 362                                          DMA_TO_DEVICE);
 363
 364                 if (dma_mapping_error(&pdev->dev, mapping))
 365                         break;
 366
 367                 __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp);
 368                 nxmit++;
 369         }
 370
 371         if (flags & XDP_XMIT_FLUSH) {
 372                 /* Sync BD data before updating doorbell */
 373                 wmb();
 374                 bnxt_db_write(bp, &txr->tx_db, txr->tx_prod);
 375         }
 376
 377         if (static_branch_unlikely(&bnxt_xdp_locking_key))
 378                 spin_unlock(&txr->xdp_tx_lock);
 379
 380         return nxmit;
 381 }
 382
 383 /* Under rtnl_lock */
 384 static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog)
 385 {
 386         struct net_device *dev = bp->dev;
 387         int tx_xdp = 0, rc, tc;
 388         struct bpf_prog *old;
 389
 390         if (prog && !prog->aux->xdp_has_frags &&
 391             bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) {
 392                 netdev_warn(dev, "MTU %d larger than %d without XDP frag support.\n",
 393                             bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU);
 394                 return -EOPNOTSUPP;
 395         }
 396         if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) {
 397                 netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n");
 398                 return -EOPNOTSUPP;
 399         }
 400         if (prog)
 401                 tx_xdp = bp->rx_nr_rings;
 402
 403         tc = netdev_get_num_tc(dev);
 404         if (!tc)
 405                 tc = 1;
 406         rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings,
 407                               true, tc, tx_xdp);
 408         if (rc) {
 409                 netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n");
 410                 return rc;
 411         }
 412         if (netif_running(dev))
 413                 bnxt_close_nic(bp, true, false);
 414
 415         old = xchg(&bp->xdp_prog, prog);
 416         if (old)
 417                 bpf_prog_put(old);
 418
 419         if (prog) {
 420                 bnxt_set_rx_skb_mode(bp, true);
 421         } else {
 422                 int rx, tx;
 423
 424                 bnxt_set_rx_skb_mode(bp, false);
 425                 bnxt_get_max_rings(bp, &rx, &tx, true);
 426                 if (rx > 1) {
 427                         bp->flags &= ~BNXT_FLAG_NO_AGG_RINGS;
 428                         bp->dev->hw_features |= NETIF_F_LRO;
 429                 }
 430         }
 431         bp->tx_nr_rings_xdp = tx_xdp;
 432         bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp;
 433         bp->cp_nr_rings = max_t(int, bp->tx_nr_rings, bp->rx_nr_rings);
 434         bnxt_set_tpa_flags(bp);
 435         bnxt_set_ring_params(bp);
 436
 437         if (netif_running(dev))
 438                 return bnxt_open_nic(bp, true, false);
 439
 440         return 0;
 441 }
 442
 443 int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 444 {
 445         struct bnxt *bp = netdev_priv(dev);
 446         int rc;
 447
 448         switch (xdp->command) {
 449         case XDP_SETUP_PROG:
 450                 rc = bnxt_xdp_set(bp, xdp->prog);
 451                 break;
 452         default:
 453                 rc = -EINVAL;
 454                 break;
 455         }
 456         return rc;
 457 }
 458
 459 struct sk_buff *
 460 bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags,
 461                    struct page_pool *pool, struct xdp_buff *xdp,
 462                    struct rx_cmp_ext *rxcmp1)
 463 {
 464         struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp);
 465
 466         if (!skb)
 467                 return NULL;
 468         skb_checksum_none_assert(skb);
 469         if (RX_CMP_L4_CS_OK(rxcmp1)) {
 470                 if (bp->dev->features & NETIF_F_RXCSUM) {
 471                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 472                         skb->csum_level = RX_CMP_ENCAP(rxcmp1);
 473                 }
 474         }
 475         xdp_update_skb_shared_info(skb, num_frags,
 476                                    sinfo->xdp_frags_size,
 477                                    PAGE_SIZE * sinfo->nr_frags,
 478                                    xdp_buff_is_frag_pfmemalloc(xdp));
 479         return skb;
 480 }