drivers/net/ethernet/netronome/nfp/bpf/main.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */
   3
   4 #include <net/pkt_cls.h>
   5
   6 #include "../nfpcore/nfp_cpp.h"
   7 #include "../nfpcore/nfp_nffw.h"
   8 #include "../nfpcore/nfp_nsp.h"
   9 #include "../nfp_app.h"
  10 #include "../nfp_main.h"
  11 #include "../nfp_net.h"
  12 #include "../nfp_port.h"
  13 #include "fw.h"
  14 #include "main.h"
  15
  16 const struct rhashtable_params nfp_bpf_maps_neutral_params = {
  17         .nelem_hint             = 4,
  18         .key_len                = sizeof_field(struct bpf_map, id),
  19         .key_offset             = offsetof(struct nfp_bpf_neutral_map, map_id),
  20         .head_offset            = offsetof(struct nfp_bpf_neutral_map, l),
  21         .automatic_shrinking    = true,
  22 };
  23
  24 static bool nfp_net_ebpf_capable(struct nfp_net *nn)
  25 {
  26 #ifdef __LITTLE_ENDIAN
  27         struct nfp_app_bpf *bpf = nn->app->priv;
  28
  29         return nn->cap & NFP_NET_CFG_CTRL_BPF &&
  30                bpf->abi_version &&
  31                nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version;
  32 #else
  33         return false;
  34 #endif
  35 }
  36
  37 static int
  38 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
  39                     struct bpf_prog *prog, struct netlink_ext_ack *extack)
  40 {
  41         bool running, xdp_running;
  42
  43         if (!nfp_net_ebpf_capable(nn))
  44                 return -EINVAL;
  45
  46         running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
  47         xdp_running = running && nn->xdp_hw.prog;
  48
  49         if (!prog && !xdp_running)
  50                 return 0;
  51         if (prog && running && !xdp_running)
  52                 return -EBUSY;
  53
  54         return nfp_net_bpf_offload(nn, prog, running, extack);
  55 }
  56
  57 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
  58 {
  59         return nfp_net_ebpf_capable(nn) ? "BPF" : "";
  60 }
  61
  62 static int
  63 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
  64 {
  65         struct nfp_pf *pf = app->pf;
  66         struct nfp_bpf_vnic *bv;
  67         int err;
  68
  69         if (!pf->eth_tbl) {
  70                 nfp_err(pf->cpp, "No ETH table\n");
  71                 return -EINVAL;
  72         }
  73         if (pf->max_data_vnics != pf->eth_tbl->count) {
  74                 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
  75                         pf->max_data_vnics, pf->eth_tbl->count);
  76                 return -EINVAL;
  77         }
  78
  79         bv = kzalloc(sizeof(*bv), GFP_KERNEL);
  80         if (!bv)
  81                 return -ENOMEM;
  82         nn->app_priv = bv;
  83
  84         err = nfp_app_nic_vnic_alloc(app, nn, id);
  85         if (err)
  86                 goto err_free_priv;
  87
  88         bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
  89         bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
  90
  91         return 0;
  92 err_free_priv:
  93         kfree(nn->app_priv);
  94         return err;
  95 }
  96
  97 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
  98 {
  99         struct nfp_bpf_vnic *bv = nn->app_priv;
 100
 101         WARN_ON(bv->tc_prog);
 102         kfree(bv);
 103 }
 104
 105 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
 106                                      void *type_data, void *cb_priv)
 107 {
 108         struct tc_cls_bpf_offload *cls_bpf = type_data;
 109         struct nfp_net *nn = cb_priv;
 110         struct bpf_prog *oldprog;
 111         struct nfp_bpf_vnic *bv;
 112         int err;
 113
 114         if (type != TC_SETUP_CLSBPF) {
 115                 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
 116                                    "only offload of BPF classifiers supported");
 117                 return -EOPNOTSUPP;
 118         }
 119         if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common))
 120                 return -EOPNOTSUPP;
 121         if (!nfp_net_ebpf_capable(nn)) {
 122                 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
 123                                    "NFP firmware does not support eBPF offload");
 124                 return -EOPNOTSUPP;
 125         }
 126         if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
 127                 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
 128                                    "only ETH_P_ALL supported as filter protocol");
 129                 return -EOPNOTSUPP;
 130         }
 131
 132         /* Only support TC direct action */
 133         if (!cls_bpf->exts_integrated ||
 134             tcf_exts_has_actions(cls_bpf->exts)) {
 135                 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
 136                                    "only direct action with no legacy actions supported");
 137                 return -EOPNOTSUPP;
 138         }
 139
 140         if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
 141                 return -EOPNOTSUPP;
 142
 143         bv = nn->app_priv;
 144         oldprog = cls_bpf->oldprog;
 145
 146         /* Don't remove if oldprog doesn't match driver's state */
 147         if (bv->tc_prog != oldprog) {
 148                 oldprog = NULL;
 149                 if (!cls_bpf->prog)
 150                         return 0;
 151         }
 152
 153         err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog,
 154                                   cls_bpf->common.extack);
 155         if (err)
 156                 return err;
 157
 158         bv->tc_prog = cls_bpf->prog;
 159         nn->port->tc_offload_cnt = !!bv->tc_prog;
 160         return 0;
 161 }
 162
 163 static LIST_HEAD(nfp_bpf_block_cb_list);
 164
 165 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
 166                             enum tc_setup_type type, void *type_data)
 167 {
 168         struct nfp_net *nn = netdev_priv(netdev);
 169
 170         switch (type) {
 171         case TC_SETUP_BLOCK:
 172                 return flow_block_cb_setup_simple(type_data,
 173                                                   &nfp_bpf_block_cb_list,
 174                                                   nfp_bpf_setup_tc_block_cb,
 175                                                   nn, nn, true);
 176         default:
 177                 return -EOPNOTSUPP;
 178         }
 179 }
 180
 181 static int
 182 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
 183 {
 184         struct nfp_net *nn = netdev_priv(netdev);
 185         struct nfp_bpf_vnic *bv;
 186         struct bpf_prog *prog;
 187
 188         if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
 189                 return 0;
 190
 191         if (nn->xdp_hw.prog) {
 192                 prog = nn->xdp_hw.prog;
 193         } else {
 194                 bv = nn->app_priv;
 195                 prog = bv->tc_prog;
 196         }
 197
 198         if (nfp_bpf_offload_check_mtu(nn, prog, new_mtu)) {
 199                 nn_info(nn, "BPF offload active, potential packet access beyond hardware packet boundary");
 200                 return -EBUSY;
 201         }
 202         return 0;
 203 }
 204
 205 static int
 206 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value,
 207                               u32 length)
 208 {
 209         struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
 210         struct nfp_cpp *cpp = bpf->app->pf->cpp;
 211
 212         if (length < sizeof(*cap)) {
 213                 nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
 214                 return -EINVAL;
 215         }
 216
 217         bpf->adjust_head.flags = readl(&cap->flags);
 218         bpf->adjust_head.off_min = readl(&cap->off_min);
 219         bpf->adjust_head.off_max = readl(&cap->off_max);
 220         bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
 221         bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);
 222
 223         if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
 224                 nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
 225                 return -EINVAL;
 226         }
 227         if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) ||
 228             !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
 229                 nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
 230                 memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
 231                 return 0;
 232         }
 233
 234         return 0;
 235 }
 236
 237 static int
 238 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
 239 {
 240         struct nfp_bpf_cap_tlv_func __iomem *cap = value;
 241
 242         if (length < sizeof(*cap)) {
 243                 nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length);
 244                 return -EINVAL;
 245         }
 246
 247         switch (readl(&cap->func_id)) {
 248         case BPF_FUNC_map_lookup_elem:
 249                 bpf->helpers.map_lookup = readl(&cap->func_addr);
 250                 break;
 251         case BPF_FUNC_map_update_elem:
 252                 bpf->helpers.map_update = readl(&cap->func_addr);
 253                 break;
 254         case BPF_FUNC_map_delete_elem:
 255                 bpf->helpers.map_delete = readl(&cap->func_addr);
 256                 break;
 257         case BPF_FUNC_perf_event_output:
 258                 bpf->helpers.perf_event_output = readl(&cap->func_addr);
 259                 break;
 260         }
 261
 262         return 0;
 263 }
 264
 265 static int
 266 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
 267 {
 268         struct nfp_bpf_cap_tlv_maps __iomem *cap = value;
 269
 270         if (length < sizeof(*cap)) {
 271                 nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length);
 272                 return -EINVAL;
 273         }
 274
 275         bpf->maps.types = readl(&cap->types);
 276         bpf->maps.max_maps = readl(&cap->max_maps);
 277         bpf->maps.max_elems = readl(&cap->max_elems);
 278         bpf->maps.max_key_sz = readl(&cap->max_key_sz);
 279         bpf->maps.max_val_sz = readl(&cap->max_val_sz);
 280         bpf->maps.max_elem_sz = readl(&cap->max_elem_sz);
 281
 282         return 0;
 283 }
 284
 285 static int
 286 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
 287                          u32 length)
 288 {
 289         bpf->pseudo_random = true;
 290         return 0;
 291 }
 292
 293 static int
 294 nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
 295 {
 296         bpf->queue_select = true;
 297         return 0;
 298 }
 299
 300 static int
 301 nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value,
 302                               u32 length)
 303 {
 304         bpf->adjust_tail = true;
 305         return 0;
 306 }
 307
 308 static int
 309 nfp_bpf_parse_cap_cmsg_multi_ent(struct nfp_app_bpf *bpf, void __iomem *value,
 310                                  u32 length)
 311 {
 312         bpf->cmsg_multi_ent = true;
 313         return 0;
 314 }
 315
 316 static int
 317 nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value,
 318                               u32 length)
 319 {
 320         if (length < 4) {
 321                 nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n",
 322                         length);
 323                 return -EINVAL;
 324         }
 325
 326         bpf->abi_version = readl(value);
 327         if (bpf->abi_version < 2 || bpf->abi_version > 3) {
 328                 nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n",
 329                          bpf->abi_version);
 330                 bpf->abi_version = 0;
 331         }
 332
 333         return 0;
 334 }
 335
 336 static int nfp_bpf_parse_capabilities(struct nfp_app *app)
 337 {
 338         struct nfp_cpp *cpp = app->pf->cpp;
 339         struct nfp_cpp_area *area;
 340         u8 __iomem *mem, *start;
 341
 342         mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
 343                             8, &area);
 344         if (IS_ERR(mem))
 345                 return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);
 346
 347         start = mem;
 348         while (mem - start + 8 <= nfp_cpp_area_size(area)) {
 349                 u8 __iomem *value;
 350                 u32 type, length;
 351
 352                 type = readl(mem);
 353                 length = readl(mem + 4);
 354                 value = mem + 8;
 355
 356                 mem += 8 + length;
 357                 if (mem - start > nfp_cpp_area_size(area))
 358                         goto err_release_free;
 359
 360                 switch (type) {
 361                 case NFP_BPF_CAP_TYPE_FUNC:
 362                         if (nfp_bpf_parse_cap_func(app->priv, value, length))
 363                                 goto err_release_free;
 364                         break;
 365                 case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
 366                         if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
 367                                                           length))
 368                                 goto err_release_free;
 369                         break;
 370                 case NFP_BPF_CAP_TYPE_MAPS:
 371                         if (nfp_bpf_parse_cap_maps(app->priv, value, length))
 372                                 goto err_release_free;
 373                         break;
 374                 case NFP_BPF_CAP_TYPE_RANDOM:
 375                         if (nfp_bpf_parse_cap_random(app->priv, value, length))
 376                                 goto err_release_free;
 377                         break;
 378                 case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
 379                         if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
 380                                 goto err_release_free;
 381                         break;
 382                 case NFP_BPF_CAP_TYPE_ADJUST_TAIL:
 383                         if (nfp_bpf_parse_cap_adjust_tail(app->priv, value,
 384                                                           length))
 385                                 goto err_release_free;
 386                         break;
 387                 case NFP_BPF_CAP_TYPE_ABI_VERSION:
 388                         if (nfp_bpf_parse_cap_abi_version(app->priv, value,
 389                                                           length))
 390                                 goto err_release_free;
 391                         break;
 392                 case NFP_BPF_CAP_TYPE_CMSG_MULTI_ENT:
 393                         if (nfp_bpf_parse_cap_cmsg_multi_ent(app->priv, value,
 394                                                              length))
 395                                 goto err_release_free;
 396                         break;
 397                 default:
 398                         nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
 399                         break;
 400                 }
 401         }
 402         if (mem - start != nfp_cpp_area_size(area)) {
 403                 nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n",
 404                         mem - start, nfp_cpp_area_size(area));
 405                 goto err_release_free;
 406         }
 407
 408         nfp_cpp_area_release_free(area);
 409
 410         return 0;
 411
 412 err_release_free:
 413         nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start);
 414         nfp_cpp_area_release_free(area);
 415         return -EINVAL;
 416 }
 417
 418 static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf)
 419 {
 420         bpf->abi_version = 2; /* Original BPF ABI version */
 421 }
 422
 423 static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev)
 424 {
 425         struct nfp_app_bpf *bpf = app->priv;
 426
 427         return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev);
 428 }
 429
 430 static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev)
 431 {
 432         struct nfp_app_bpf *bpf = app->priv;
 433
 434         bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev);
 435 }
 436
 437 static int nfp_bpf_start(struct nfp_app *app)
 438 {
 439         struct nfp_app_bpf *bpf = app->priv;
 440
 441         if (app->ctrl->dp.mtu < nfp_bpf_ctrl_cmsg_min_mtu(bpf)) {
 442                 nfp_err(bpf->app->cpp,
 443                         "ctrl channel MTU below min required %u < %u\n",
 444                         app->ctrl->dp.mtu, nfp_bpf_ctrl_cmsg_min_mtu(bpf));
 445                 return -EINVAL;
 446         }
 447
 448         if (bpf->cmsg_multi_ent)
 449                 bpf->cmsg_cache_cnt = nfp_bpf_ctrl_cmsg_cache_cnt(bpf);
 450         else
 451                 bpf->cmsg_cache_cnt = 1;
 452
 453         return 0;
 454 }
 455
 456 static int nfp_bpf_init(struct nfp_app *app)
 457 {
 458         struct nfp_app_bpf *bpf;
 459         int err;
 460
 461         bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
 462         if (!bpf)
 463                 return -ENOMEM;
 464         bpf->app = app;
 465         app->priv = bpf;
 466
 467         INIT_LIST_HEAD(&bpf->map_list);
 468
 469         err = nfp_ccm_init(&bpf->ccm, app);
 470         if (err)
 471                 goto err_free_bpf;
 472
 473         err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params);
 474         if (err)
 475                 goto err_clean_ccm;
 476
 477         nfp_bpf_init_capabilities(bpf);
 478
 479         err = nfp_bpf_parse_capabilities(app);
 480         if (err)
 481                 goto err_free_neutral_maps;
 482
 483         if (bpf->abi_version < 3) {
 484                 bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4;
 485                 bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4;
 486         } else {
 487                 bpf->cmsg_key_sz = bpf->maps.max_key_sz;
 488                 bpf->cmsg_val_sz = bpf->maps.max_val_sz;
 489                 app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf);
 490         }
 491
 492         bpf->bpf_dev = bpf_offload_dev_create(&nfp_bpf_dev_ops, bpf);
 493         err = PTR_ERR_OR_ZERO(bpf->bpf_dev);
 494         if (err)
 495                 goto err_free_neutral_maps;
 496
 497         return 0;
 498
 499 err_free_neutral_maps:
 500         rhashtable_destroy(&bpf->maps_neutral);
 501 err_clean_ccm:
 502         nfp_ccm_clean(&bpf->ccm);
 503 err_free_bpf:
 504         kfree(bpf);
 505         return err;
 506 }
 507
 508 static void nfp_bpf_clean(struct nfp_app *app)
 509 {
 510         struct nfp_app_bpf *bpf = app->priv;
 511
 512         bpf_offload_dev_destroy(bpf->bpf_dev);
 513         nfp_ccm_clean(&bpf->ccm);
 514         WARN_ON(!list_empty(&bpf->map_list));
 515         WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
 516         rhashtable_free_and_destroy(&bpf->maps_neutral,
 517                                     nfp_check_rhashtable_empty, NULL);
 518         kfree(bpf);
 519 }
 520
 521 const struct nfp_app_type app_bpf = {
 522         .id             = NFP_APP_BPF_NIC,
 523         .name           = "ebpf",
 524
 525         .ctrl_cap_mask  = 0,
 526
 527         .init           = nfp_bpf_init,
 528         .clean          = nfp_bpf_clean,
 529         .start          = nfp_bpf_start,
 530
 531         .check_mtu      = nfp_bpf_check_mtu,
 532
 533         .extra_cap      = nfp_bpf_extra_cap,
 534
 535         .ndo_init       = nfp_bpf_ndo_init,
 536         .ndo_uninit     = nfp_bpf_ndo_uninit,
 537
 538         .vnic_alloc     = nfp_bpf_vnic_alloc,
 539         .vnic_free      = nfp_bpf_vnic_free,
 540
 541         .ctrl_msg_rx    = nfp_bpf_ctrl_msg_rx,
 542         .ctrl_msg_rx_raw        = nfp_bpf_ctrl_msg_rx_raw,
 543
 544         .setup_tc       = nfp_bpf_setup_tc,
 545         .bpf            = nfp_ndo_bpf,
 546         .xdp_offload    = nfp_bpf_xdp_offload,
 547 };