drivers/net/ethernet/netronome/nfp/bpf/offload.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 /* Copyright (C) 2016-2018 Netronome Systems, Inc. */
   3
   4 /*
   5  * nfp_net_offload.c
   6  * Netronome network device driver: TC offload functions for PF and VF
   7  */
   8
   9 #define pr_fmt(fmt)     "NFP net bpf: " fmt
  10
  11 #include <linux/bpf.h>
  12 #include <linux/kernel.h>
  13 #include <linux/netdevice.h>
  14 #include <linux/pci.h>
  15 #include <linux/jiffies.h>
  16 #include <linux/timer.h>
  17 #include <linux/list.h>
  18 #include <linux/mm.h>
  19
  20 #include <net/pkt_cls.h>
  21 #include <net/tc_act/tc_gact.h>
  22 #include <net/tc_act/tc_mirred.h>
  23
  24 #include "main.h"
  25 #include "../nfp_app.h"
  26 #include "../nfp_net_ctrl.h"
  27 #include "../nfp_net.h"
  28
  29 static int
  30 nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
  31                    struct bpf_map *map)
  32 {
  33         struct nfp_bpf_neutral_map *record;
  34         int err;
  35
  36         /* Reuse path - other offloaded program is already tracking this map. */
  37         record = rhashtable_lookup_fast(&bpf->maps_neutral, &map->id,
  38                                         nfp_bpf_maps_neutral_params);
  39         if (record) {
  40                 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
  41                 record->count++;
  42                 return 0;
  43         }
  44
  45         /* Grab a single ref to the map for our record.  The prog destroy ndo
  46          * happens after free_used_maps().
  47          */
  48         map = bpf_map_inc(map, false);
  49         if (IS_ERR(map))
  50                 return PTR_ERR(map);
  51
  52         record = kmalloc(sizeof(*record), GFP_KERNEL);
  53         if (!record) {
  54                 err = -ENOMEM;
  55                 goto err_map_put;
  56         }
  57
  58         record->ptr = map;
  59         record->map_id = map->id;
  60         record->count = 1;
  61
  62         err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l,
  63                                      nfp_bpf_maps_neutral_params);
  64         if (err)
  65                 goto err_free_rec;
  66
  67         nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
  68
  69         return 0;
  70
  71 err_free_rec:
  72         kfree(record);
  73 err_map_put:
  74         bpf_map_put(map);
  75         return err;
  76 }
  77
  78 static void
  79 nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog)
  80 {
  81         bool freed = false;
  82         int i;
  83
  84         for (i = 0; i < nfp_prog->map_records_cnt; i++) {
  85                 if (--nfp_prog->map_records[i]->count) {
  86                         nfp_prog->map_records[i] = NULL;
  87                         continue;
  88                 }
  89
  90                 WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral,
  91                                                &nfp_prog->map_records[i]->l,
  92                                                nfp_bpf_maps_neutral_params));
  93                 freed = true;
  94         }
  95
  96         if (freed) {
  97                 synchronize_rcu();
  98
  99                 for (i = 0; i < nfp_prog->map_records_cnt; i++)
 100                         if (nfp_prog->map_records[i]) {
 101                                 bpf_map_put(nfp_prog->map_records[i]->ptr);
 102                                 kfree(nfp_prog->map_records[i]);
 103                         }
 104         }
 105
 106         kfree(nfp_prog->map_records);
 107         nfp_prog->map_records = NULL;
 108         nfp_prog->map_records_cnt = 0;
 109 }
 110
 111 static int
 112 nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
 113                     struct bpf_prog *prog)
 114 {
 115         int i, cnt, err;
 116
 117         /* Quickly count the maps we will have to remember */
 118         cnt = 0;
 119         for (i = 0; i < prog->aux->used_map_cnt; i++)
 120                 if (bpf_map_offload_neutral(prog->aux->used_maps[i]))
 121                         cnt++;
 122         if (!cnt)
 123                 return 0;
 124
 125         nfp_prog->map_records = kmalloc_array(cnt,
 126                                               sizeof(nfp_prog->map_records[0]),
 127                                               GFP_KERNEL);
 128         if (!nfp_prog->map_records)
 129                 return -ENOMEM;
 130
 131         for (i = 0; i < prog->aux->used_map_cnt; i++)
 132                 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) {
 133                         err = nfp_map_ptr_record(bpf, nfp_prog,
 134                                                  prog->aux->used_maps[i]);
 135                         if (err) {
 136                                 nfp_map_ptrs_forget(bpf, nfp_prog);
 137                                 return err;
 138                         }
 139                 }
 140         WARN_ON(cnt != nfp_prog->map_records_cnt);
 141
 142         return 0;
 143 }
 144
 145 static int
 146 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
 147                  unsigned int cnt)
 148 {
 149         struct nfp_insn_meta *meta;
 150         unsigned int i;
 151
 152         for (i = 0; i < cnt; i++) {
 153                 meta = kzalloc(sizeof(*meta), GFP_KERNEL);
 154                 if (!meta)
 155                         return -ENOMEM;
 156
 157                 meta->insn = prog[i];
 158                 meta->n = i;
 159                 if (is_mbpf_alu(meta)) {
 160                         meta->umin_src = U64_MAX;
 161                         meta->umin_dst = U64_MAX;
 162                 }
 163
 164                 list_add_tail(&meta->l, &nfp_prog->insns);
 165         }
 166
 167         nfp_bpf_jit_prepare(nfp_prog, cnt);
 168
 169         return 0;
 170 }
 171
 172 static void nfp_prog_free(struct nfp_prog *nfp_prog)
 173 {
 174         struct nfp_insn_meta *meta, *tmp;
 175
 176         kfree(nfp_prog->subprog);
 177
 178         list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
 179                 list_del(&meta->l);
 180                 kfree(meta);
 181         }
 182         kfree(nfp_prog);
 183 }
 184
 185 static int
 186 nfp_bpf_verifier_prep(struct net_device *netdev, struct bpf_verifier_env *env)
 187 {
 188         struct nfp_net *nn = netdev_priv(netdev);
 189         struct bpf_prog *prog = env->prog;
 190         struct nfp_app *app = nn->app;
 191         struct nfp_prog *nfp_prog;
 192         int ret;
 193
 194         nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
 195         if (!nfp_prog)
 196                 return -ENOMEM;
 197         prog->aux->offload->dev_priv = nfp_prog;
 198
 199         INIT_LIST_HEAD(&nfp_prog->insns);
 200         nfp_prog->type = prog->type;
 201         nfp_prog->bpf = app->priv;
 202
 203         ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
 204         if (ret)
 205                 goto err_free;
 206
 207         nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
 208
 209         return 0;
 210
 211 err_free:
 212         nfp_prog_free(nfp_prog);
 213
 214         return ret;
 215 }
 216
 217 static int nfp_bpf_translate(struct net_device *netdev, struct bpf_prog *prog)
 218 {
 219         struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
 220         struct nfp_net *nn = netdev_priv(netdev);
 221         unsigned int max_instr;
 222         int err;
 223
 224         max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
 225         nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
 226
 227         nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
 228         if (!nfp_prog->prog)
 229                 return -ENOMEM;
 230
 231         err = nfp_bpf_jit(nfp_prog);
 232         if (err)
 233                 return err;
 234
 235         prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
 236         prog->aux->offload->jited_image = nfp_prog->prog;
 237
 238         return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog);
 239 }
 240
 241 static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog)
 242 {
 243         struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
 244
 245         kvfree(nfp_prog->prog);
 246         nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog);
 247         nfp_prog_free(nfp_prog);
 248
 249         return 0;
 250 }
 251
 252 /* Atomic engine requires values to be in big endian, we need to byte swap
 253  * the value words used with xadd.
 254  */
 255 static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value)
 256 {
 257         u32 *word = value;
 258         unsigned int i;
 259
 260         for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
 261                 if (nfp_map->use_map[i].type == NFP_MAP_USE_ATOMIC_CNT)
 262                         word[i] = (__force u32)cpu_to_be32(word[i]);
 263 }
 264
 265 /* Mark value as unsafely initialized in case it becomes atomic later
 266  * and we didn't byte swap something non-byte swap neutral.
 267  */
 268 static void
 269 nfp_map_bpf_byte_swap_record(struct nfp_bpf_map *nfp_map, void *value)
 270 {
 271         u32 *word = value;
 272         unsigned int i;
 273
 274         for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
 275                 if (nfp_map->use_map[i].type == NFP_MAP_UNUSED &&
 276                     word[i] != (__force u32)cpu_to_be32(word[i]))
 277                         nfp_map->use_map[i].non_zero_update = 1;
 278 }
 279
 280 static int
 281 nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap,
 282                          void *key, void *value)
 283 {
 284         int err;
 285
 286         err = nfp_bpf_ctrl_lookup_entry(offmap, key, value);
 287         if (err)
 288                 return err;
 289
 290         nfp_map_bpf_byte_swap(offmap->dev_priv, value);
 291         return 0;
 292 }
 293
 294 static int
 295 nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap,
 296                          void *key, void *value, u64 flags)
 297 {
 298         nfp_map_bpf_byte_swap(offmap->dev_priv, value);
 299         nfp_map_bpf_byte_swap_record(offmap->dev_priv, value);
 300         return nfp_bpf_ctrl_update_entry(offmap, key, value, flags);
 301 }
 302
 303 static int
 304 nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
 305                          void *key, void *next_key)
 306 {
 307         if (!key)
 308                 return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
 309         return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
 310 }
 311
 312 static int
 313 nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
 314 {
 315         if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
 316                 return -EINVAL;
 317         return nfp_bpf_ctrl_del_entry(offmap, key);
 318 }
 319
 320 static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
 321         .map_get_next_key       = nfp_bpf_map_get_next_key,
 322         .map_lookup_elem        = nfp_bpf_map_lookup_entry,
 323         .map_update_elem        = nfp_bpf_map_update_entry,
 324         .map_delete_elem        = nfp_bpf_map_delete_elem,
 325 };
 326
 327 static int
 328 nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
 329 {
 330         struct nfp_bpf_map *nfp_map;
 331         unsigned int use_map_size;
 332         long long int res;
 333
 334         if (!bpf->maps.types)
 335                 return -EOPNOTSUPP;
 336
 337         if (offmap->map.map_flags ||
 338             offmap->map.numa_node != NUMA_NO_NODE) {
 339                 pr_info("map flags are not supported\n");
 340                 return -EINVAL;
 341         }
 342
 343         if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
 344                 pr_info("map type not supported\n");
 345                 return -EOPNOTSUPP;
 346         }
 347         if (bpf->maps.max_maps == bpf->maps_in_use) {
 348                 pr_info("too many maps for a device\n");
 349                 return -ENOMEM;
 350         }
 351         if (bpf->maps.max_elems - bpf->map_elems_in_use <
 352             offmap->map.max_entries) {
 353                 pr_info("map with too many elements: %u, left: %u\n",
 354                         offmap->map.max_entries,
 355                         bpf->maps.max_elems - bpf->map_elems_in_use);
 356                 return -ENOMEM;
 357         }
 358
 359         if (round_up(offmap->map.key_size, 8) +
 360             round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
 361                 pr_info("map elements too large: %u, FW max element size (key+value): %u\n",
 362                         round_up(offmap->map.key_size, 8) +
 363                         round_up(offmap->map.value_size, 8),
 364                         bpf->maps.max_elem_sz);
 365                 return -ENOMEM;
 366         }
 367         if (offmap->map.key_size > bpf->maps.max_key_sz) {
 368                 pr_info("map key size %u, FW max is %u\n",
 369                         offmap->map.key_size, bpf->maps.max_key_sz);
 370                 return -ENOMEM;
 371         }
 372         if (offmap->map.value_size > bpf->maps.max_val_sz) {
 373                 pr_info("map value size %u, FW max is %u\n",
 374                         offmap->map.value_size, bpf->maps.max_val_sz);
 375                 return -ENOMEM;
 376         }
 377
 378         use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) *
 379                        FIELD_SIZEOF(struct nfp_bpf_map, use_map[0]);
 380
 381         nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER);
 382         if (!nfp_map)
 383                 return -ENOMEM;
 384
 385         offmap->dev_priv = nfp_map;
 386         nfp_map->offmap = offmap;
 387         nfp_map->bpf = bpf;
 388
 389         res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
 390         if (res < 0) {
 391                 kfree(nfp_map);
 392                 return res;
 393         }
 394
 395         nfp_map->tid = res;
 396         offmap->dev_ops = &nfp_bpf_map_ops;
 397         bpf->maps_in_use++;
 398         bpf->map_elems_in_use += offmap->map.max_entries;
 399         list_add_tail(&nfp_map->l, &bpf->map_list);
 400
 401         return 0;
 402 }
 403
 404 static int
 405 nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
 406 {
 407         struct nfp_bpf_map *nfp_map = offmap->dev_priv;
 408
 409         nfp_bpf_ctrl_free_map(bpf, nfp_map);
 410         list_del_init(&nfp_map->l);
 411         bpf->map_elems_in_use -= offmap->map.max_entries;
 412         bpf->maps_in_use--;
 413         kfree(nfp_map);
 414
 415         return 0;
 416 }
 417
 418 int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
 419 {
 420         switch (bpf->command) {
 421         case BPF_OFFLOAD_DESTROY:
 422                 return nfp_bpf_destroy(nn, bpf->offload.prog);
 423         case BPF_OFFLOAD_MAP_ALLOC:
 424                 return nfp_bpf_map_alloc(app->priv, bpf->offmap);
 425         case BPF_OFFLOAD_MAP_FREE:
 426                 return nfp_bpf_map_free(app->priv, bpf->offmap);
 427         default:
 428                 return -EINVAL;
 429         }
 430 }
 431
 432 static unsigned long
 433 nfp_bpf_perf_event_copy(void *dst, const void *src,
 434                         unsigned long off, unsigned long len)
 435 {
 436         memcpy(dst, src + off, len);
 437         return 0;
 438 }
 439
 440 int nfp_bpf_event_output(struct nfp_app_bpf *bpf, const void *data,
 441                          unsigned int len)
 442 {
 443         struct cmsg_bpf_event *cbe = (void *)data;
 444         struct nfp_bpf_neutral_map *record;
 445         u32 pkt_size, data_size, map_id;
 446         u64 map_id_full;
 447
 448         if (len < sizeof(struct cmsg_bpf_event))
 449                 return -EINVAL;
 450
 451         pkt_size = be32_to_cpu(cbe->pkt_size);
 452         data_size = be32_to_cpu(cbe->data_size);
 453         map_id_full = be64_to_cpu(cbe->map_ptr);
 454         map_id = map_id_full;
 455
 456         if (len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size)
 457                 return -EINVAL;
 458         if (cbe->hdr.ver != CMSG_MAP_ABI_VERSION)
 459                 return -EINVAL;
 460
 461         rcu_read_lock();
 462         record = rhashtable_lookup_fast(&bpf->maps_neutral, &map_id,
 463                                         nfp_bpf_maps_neutral_params);
 464         if (!record || map_id_full > U32_MAX) {
 465                 rcu_read_unlock();
 466                 cmsg_warn(bpf, "perf event: map id %lld (0x%llx) not recognized, dropping event\n",
 467                           map_id_full, map_id_full);
 468                 return -EINVAL;
 469         }
 470
 471         bpf_event_output(record->ptr, be32_to_cpu(cbe->cpu_id),
 472                          &cbe->data[round_up(pkt_size, 4)], data_size,
 473                          cbe->data, pkt_size, nfp_bpf_perf_event_copy);
 474         rcu_read_unlock();
 475
 476         return 0;
 477 }
 478
 479 static int
 480 nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog,
 481                  struct netlink_ext_ack *extack)
 482 {
 483         struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
 484         unsigned int fw_mtu, pkt_off, max_stack, max_prog_len;
 485         dma_addr_t dma_addr;
 486         void *img;
 487         int err;
 488
 489         fw_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
 490         pkt_off = min(prog->aux->max_pkt_offset, nn->dp.netdev->mtu);
 491         if (fw_mtu < pkt_off) {
 492                 NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with potential packet access beyond HW packet split boundary");
 493                 return -EOPNOTSUPP;
 494         }
 495
 496         max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
 497         if (nfp_prog->stack_size > max_stack) {
 498                 NL_SET_ERR_MSG_MOD(extack, "stack too large");
 499                 return -EOPNOTSUPP;
 500         }
 501
 502         max_prog_len = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
 503         if (nfp_prog->prog_len > max_prog_len) {
 504                 NL_SET_ERR_MSG_MOD(extack, "program too long");
 505                 return -EOPNOTSUPP;
 506         }
 507
 508         img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
 509         if (IS_ERR(img))
 510                 return PTR_ERR(img);
 511
 512         dma_addr = dma_map_single(nn->dp.dev, img,
 513                                   nfp_prog->prog_len * sizeof(u64),
 514                                   DMA_TO_DEVICE);
 515         if (dma_mapping_error(nn->dp.dev, dma_addr)) {
 516                 kfree(img);
 517                 return -ENOMEM;
 518         }
 519
 520         nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
 521         nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
 522
 523         /* Load up the JITed code */
 524         err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
 525         if (err)
 526                 NL_SET_ERR_MSG_MOD(extack,
 527                                    "FW command error while loading BPF");
 528
 529         dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
 530                          DMA_TO_DEVICE);
 531         kfree(img);
 532
 533         return err;
 534 }
 535
 536 static void
 537 nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack)
 538 {
 539         int err;
 540
 541         /* Enable passing packets through BPF function */
 542         nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
 543         nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
 544         err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
 545         if (err)
 546                 NL_SET_ERR_MSG_MOD(extack,
 547                                    "FW command error while enabling BPF");
 548 }
 549
 550 static int nfp_net_bpf_stop(struct nfp_net *nn)
 551 {
 552         if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
 553                 return 0;
 554
 555         nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
 556         nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
 557
 558         return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
 559 }
 560
 561 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
 562                         bool old_prog, struct netlink_ext_ack *extack)
 563 {
 564         int err;
 565
 566         if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev))
 567                 return -EINVAL;
 568
 569         if (prog && old_prog) {
 570                 u8 cap;
 571
 572                 cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
 573                 if (!(cap & NFP_NET_BPF_CAP_RELO)) {
 574                         NL_SET_ERR_MSG_MOD(extack,
 575                                            "FW does not support live reload");
 576                         return -EBUSY;
 577                 }
 578         }
 579
 580         /* Something else is loaded, different program type? */
 581         if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
 582                 return -EBUSY;
 583
 584         if (old_prog && !prog)
 585                 return nfp_net_bpf_stop(nn);
 586
 587         err = nfp_net_bpf_load(nn, prog, extack);
 588         if (err)
 589                 return err;
 590
 591         if (!old_prog)
 592                 nfp_net_bpf_start(nn, extack);
 593
 594         return 0;
 595 }
 596
 597 const struct bpf_prog_offload_ops nfp_bpf_dev_ops = {
 598         .insn_hook      = nfp_verify_insn,
 599         .finalize       = nfp_bpf_finalize,
 600         .prepare        = nfp_bpf_verifier_prep,
 601         .translate      = nfp_bpf_translate,
 602 };