drivers/net/ethernet/netronome/nfp/bpf/cmsg.c

   1 /*
   2  * Copyright (C) 2017-2018 Netronome Systems, Inc.
   3  *
   4  * This software is dual licensed under the GNU General License Version 2,
   5  * June 1991 as shown in the file COPYING in the top-level directory of this
   6  * source tree or the BSD 2-Clause License provided below.  You have the
   7  * option to license this software under the complete terms of either license.
   8  *
   9  * The BSD 2-Clause License:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      1. Redistributions of source code must retain the above
  16  *         copyright notice, this list of conditions and the following
  17  *         disclaimer.
  18  *
  19  *      2. Redistributions in binary form must reproduce the above
  20  *         copyright notice, this list of conditions and the following
  21  *         disclaimer in the documentation and/or other materials
  22  *         provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/bpf.h>
  35 #include <linux/bitops.h>
  36 #include <linux/bug.h>
  37 #include <linux/jiffies.h>
  38 #include <linux/skbuff.h>
  39 #include <linux/wait.h>
  40
  41 #include "../nfp_app.h"
  42 #include "../nfp_net.h"
  43 #include "fw.h"
  44 #include "main.h"
  45
  46 #define NFP_BPF_TAG_ALLOC_SPAN  (U16_MAX / 4)
  47
  48 static bool nfp_bpf_all_tags_busy(struct nfp_app_bpf *bpf)
  49 {
  50         u16 used_tags;
  51
  52         used_tags = bpf->tag_alloc_next - bpf->tag_alloc_last;
  53
  54         return used_tags > NFP_BPF_TAG_ALLOC_SPAN;
  55 }
  56
  57 static int nfp_bpf_alloc_tag(struct nfp_app_bpf *bpf)
  58 {
  59         /* All FW communication for BPF is request-reply.  To make sure we
  60          * don't reuse the message ID too early after timeout - limit the
  61          * number of requests in flight.
  62          */
  63         if (nfp_bpf_all_tags_busy(bpf)) {
  64                 cmsg_warn(bpf, "all FW request contexts busy!\n");
  65                 return -EAGAIN;
  66         }
  67
  68         WARN_ON(__test_and_set_bit(bpf->tag_alloc_next, bpf->tag_allocator));
  69         return bpf->tag_alloc_next++;
  70 }
  71
  72 static void nfp_bpf_free_tag(struct nfp_app_bpf *bpf, u16 tag)
  73 {
  74         WARN_ON(!__test_and_clear_bit(tag, bpf->tag_allocator));
  75
  76         while (!test_bit(bpf->tag_alloc_last, bpf->tag_allocator) &&
  77                bpf->tag_alloc_last != bpf->tag_alloc_next)
  78                 bpf->tag_alloc_last++;
  79 }
  80
  81 static struct sk_buff *
  82 nfp_bpf_cmsg_alloc(struct nfp_app_bpf *bpf, unsigned int size)
  83 {
  84         struct sk_buff *skb;
  85
  86         skb = nfp_app_ctrl_msg_alloc(bpf->app, size, GFP_KERNEL);
  87         skb_put(skb, size);
  88
  89         return skb;
  90 }
  91
  92 static unsigned int
  93 nfp_bpf_cmsg_map_req_size(struct nfp_app_bpf *bpf, unsigned int n)
  94 {
  95         unsigned int size;
  96
  97         size = sizeof(struct cmsg_req_map_op);
  98         size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n;
  99
 100         return size;
 101 }
 102
 103 static struct sk_buff *
 104 nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n)
 105 {
 106         return nfp_bpf_cmsg_alloc(bpf, nfp_bpf_cmsg_map_req_size(bpf, n));
 107 }
 108
 109 static unsigned int
 110 nfp_bpf_cmsg_map_reply_size(struct nfp_app_bpf *bpf, unsigned int n)
 111 {
 112         unsigned int size;
 113
 114         size = sizeof(struct cmsg_reply_map_op);
 115         size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n;
 116
 117         return size;
 118 }
 119
 120 static u8 nfp_bpf_cmsg_get_type(struct sk_buff *skb)
 121 {
 122         struct cmsg_hdr *hdr;
 123
 124         hdr = (struct cmsg_hdr *)skb->data;
 125
 126         return hdr->type;
 127 }
 128
 129 static unsigned int nfp_bpf_cmsg_get_tag(struct sk_buff *skb)
 130 {
 131         struct cmsg_hdr *hdr;
 132
 133         hdr = (struct cmsg_hdr *)skb->data;
 134
 135         return be16_to_cpu(hdr->tag);
 136 }
 137
 138 static struct sk_buff *__nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag)
 139 {
 140         unsigned int msg_tag;
 141         struct sk_buff *skb;
 142
 143         skb_queue_walk(&bpf->cmsg_replies, skb) {
 144                 msg_tag = nfp_bpf_cmsg_get_tag(skb);
 145                 if (msg_tag == tag) {
 146                         nfp_bpf_free_tag(bpf, tag);
 147                         __skb_unlink(skb, &bpf->cmsg_replies);
 148                         return skb;
 149                 }
 150         }
 151
 152         return NULL;
 153 }
 154
 155 static struct sk_buff *nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag)
 156 {
 157         struct sk_buff *skb;
 158
 159         nfp_ctrl_lock(bpf->app->ctrl);
 160         skb = __nfp_bpf_reply(bpf, tag);
 161         nfp_ctrl_unlock(bpf->app->ctrl);
 162
 163         return skb;
 164 }
 165
 166 static struct sk_buff *nfp_bpf_reply_drop_tag(struct nfp_app_bpf *bpf, u16 tag)
 167 {
 168         struct sk_buff *skb;
 169
 170         nfp_ctrl_lock(bpf->app->ctrl);
 171         skb = __nfp_bpf_reply(bpf, tag);
 172         if (!skb)
 173                 nfp_bpf_free_tag(bpf, tag);
 174         nfp_ctrl_unlock(bpf->app->ctrl);
 175
 176         return skb;
 177 }
 178
 179 static struct sk_buff *
 180 nfp_bpf_cmsg_wait_reply(struct nfp_app_bpf *bpf, enum nfp_bpf_cmsg_type type,
 181                         int tag)
 182 {
 183         struct sk_buff *skb;
 184         int i, err;
 185
 186         for (i = 0; i < 50; i++) {
 187                 udelay(4);
 188                 skb = nfp_bpf_reply(bpf, tag);
 189                 if (skb)
 190                         return skb;
 191         }
 192
 193         err = wait_event_interruptible_timeout(bpf->cmsg_wq,
 194                                                skb = nfp_bpf_reply(bpf, tag),
 195                                                msecs_to_jiffies(5000));
 196         /* We didn't get a response - try last time and atomically drop
 197          * the tag even if no response is matched.
 198          */
 199         if (!skb)
 200                 skb = nfp_bpf_reply_drop_tag(bpf, tag);
 201         if (err < 0) {
 202                 cmsg_warn(bpf, "%s waiting for response to 0x%02x: %d\n",
 203                           err == ERESTARTSYS ? "interrupted" : "error",
 204                           type, err);
 205                 return ERR_PTR(err);
 206         }
 207         if (!skb) {
 208                 cmsg_warn(bpf, "timeout waiting for response to 0x%02x\n",
 209                           type);
 210                 return ERR_PTR(-ETIMEDOUT);
 211         }
 212
 213         return skb;
 214 }
 215
 216 static struct sk_buff *
 217 nfp_bpf_cmsg_communicate(struct nfp_app_bpf *bpf, struct sk_buff *skb,
 218                          enum nfp_bpf_cmsg_type type, unsigned int reply_size)
 219 {
 220         struct cmsg_hdr *hdr;
 221         int tag;
 222
 223         nfp_ctrl_lock(bpf->app->ctrl);
 224         tag = nfp_bpf_alloc_tag(bpf);
 225         if (tag < 0) {
 226                 nfp_ctrl_unlock(bpf->app->ctrl);
 227                 dev_kfree_skb_any(skb);
 228                 return ERR_PTR(tag);
 229         }
 230
 231         hdr = (void *)skb->data;
 232         hdr->ver = CMSG_MAP_ABI_VERSION;
 233         hdr->type = type;
 234         hdr->tag = cpu_to_be16(tag);
 235
 236         __nfp_app_ctrl_tx(bpf->app, skb);
 237
 238         nfp_ctrl_unlock(bpf->app->ctrl);
 239
 240         skb = nfp_bpf_cmsg_wait_reply(bpf, type, tag);
 241         if (IS_ERR(skb))
 242                 return skb;
 243
 244         hdr = (struct cmsg_hdr *)skb->data;
 245         if (hdr->type != __CMSG_REPLY(type)) {
 246                 cmsg_warn(bpf, "cmsg drop - wrong type 0x%02x != 0x%02lx!\n",
 247                           hdr->type, __CMSG_REPLY(type));
 248                 goto err_free;
 249         }
 250         /* 0 reply_size means caller will do the validation */
 251         if (reply_size && skb->len != reply_size) {
 252                 cmsg_warn(bpf, "cmsg drop - type 0x%02x wrong size %d != %d!\n",
 253                           type, skb->len, reply_size);
 254                 goto err_free;
 255         }
 256
 257         return skb;
 258 err_free:
 259         dev_kfree_skb_any(skb);
 260         return ERR_PTR(-EIO);
 261 }
 262
 263 static int
 264 nfp_bpf_ctrl_rc_to_errno(struct nfp_app_bpf *bpf,
 265                          struct cmsg_reply_map_simple *reply)
 266 {
 267         static const int res_table[] = {
 268                 [CMSG_RC_SUCCESS]       = 0,
 269                 [CMSG_RC_ERR_MAP_FD]    = -EBADFD,
 270                 [CMSG_RC_ERR_MAP_NOENT] = -ENOENT,
 271                 [CMSG_RC_ERR_MAP_ERR]   = -EINVAL,
 272                 [CMSG_RC_ERR_MAP_PARSE] = -EIO,
 273                 [CMSG_RC_ERR_MAP_EXIST] = -EEXIST,
 274                 [CMSG_RC_ERR_MAP_NOMEM] = -ENOMEM,
 275                 [CMSG_RC_ERR_MAP_E2BIG] = -E2BIG,
 276         };
 277         u32 rc;
 278
 279         rc = be32_to_cpu(reply->rc);
 280         if (rc >= ARRAY_SIZE(res_table)) {
 281                 cmsg_warn(bpf, "FW responded with invalid status: %u\n", rc);
 282                 return -EIO;
 283         }
 284
 285         return res_table[rc];
 286 }
 287
 288 long long int
 289 nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map)
 290 {
 291         struct cmsg_reply_map_alloc_tbl *reply;
 292         struct cmsg_req_map_alloc_tbl *req;
 293         struct sk_buff *skb;
 294         u32 tid;
 295         int err;
 296
 297         skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
 298         if (!skb)
 299                 return -ENOMEM;
 300
 301         req = (void *)skb->data;
 302         req->key_size = cpu_to_be32(map->key_size);
 303         req->value_size = cpu_to_be32(map->value_size);
 304         req->max_entries = cpu_to_be32(map->max_entries);
 305         req->map_type = cpu_to_be32(map->map_type);
 306         req->map_flags = 0;
 307
 308         skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_ALLOC,
 309                                        sizeof(*reply));
 310         if (IS_ERR(skb))
 311                 return PTR_ERR(skb);
 312
 313         reply = (void *)skb->data;
 314         err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
 315         if (err)
 316                 goto err_free;
 317
 318         tid = be32_to_cpu(reply->tid);
 319         dev_consume_skb_any(skb);
 320
 321         return tid;
 322 err_free:
 323         dev_kfree_skb_any(skb);
 324         return err;
 325 }
 326
 327 void nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map)
 328 {
 329         struct cmsg_reply_map_free_tbl *reply;
 330         struct cmsg_req_map_free_tbl *req;
 331         struct sk_buff *skb;
 332         int err;
 333
 334         skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
 335         if (!skb) {
 336                 cmsg_warn(bpf, "leaking map - failed to allocate msg\n");
 337                 return;
 338         }
 339
 340         req = (void *)skb->data;
 341         req->tid = cpu_to_be32(nfp_map->tid);
 342
 343         skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_FREE,
 344                                        sizeof(*reply));
 345         if (IS_ERR(skb)) {
 346                 cmsg_warn(bpf, "leaking map - I/O error\n");
 347                 return;
 348         }
 349
 350         reply = (void *)skb->data;
 351         err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
 352         if (err)
 353                 cmsg_warn(bpf, "leaking map - FW responded with: %d\n", err);
 354
 355         dev_consume_skb_any(skb);
 356 }
 357
 358 static void *
 359 nfp_bpf_ctrl_req_key(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req,
 360                      unsigned int n)
 361 {
 362         return &req->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n];
 363 }
 364
 365 static void *
 366 nfp_bpf_ctrl_req_val(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req,
 367                      unsigned int n)
 368 {
 369         return &req->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n];
 370 }
 371
 372 static void *
 373 nfp_bpf_ctrl_reply_key(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply,
 374                        unsigned int n)
 375 {
 376         return &reply->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n];
 377 }
 378
 379 static void *
 380 nfp_bpf_ctrl_reply_val(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply,
 381                        unsigned int n)
 382 {
 383         return &reply->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n];
 384 }
 385
 386 static int
 387 nfp_bpf_ctrl_entry_op(struct bpf_offloaded_map *offmap,
 388                       enum nfp_bpf_cmsg_type op,
 389                       u8 *key, u8 *value, u64 flags, u8 *out_key, u8 *out_value)
 390 {
 391         struct nfp_bpf_map *nfp_map = offmap->dev_priv;
 392         struct nfp_app_bpf *bpf = nfp_map->bpf;
 393         struct bpf_map *map = &offmap->map;
 394         struct cmsg_reply_map_op *reply;
 395         struct cmsg_req_map_op *req;
 396         struct sk_buff *skb;
 397         int err;
 398
 399         /* FW messages have no space for more than 32 bits of flags */
 400         if (flags >> 32)
 401                 return -EOPNOTSUPP;
 402
 403         skb = nfp_bpf_cmsg_map_req_alloc(bpf, 1);
 404         if (!skb)
 405                 return -ENOMEM;
 406
 407         req = (void *)skb->data;
 408         req->tid = cpu_to_be32(nfp_map->tid);
 409         req->count = cpu_to_be32(1);
 410         req->flags = cpu_to_be32(flags);
 411
 412         /* Copy inputs */
 413         if (key)
 414                 memcpy(nfp_bpf_ctrl_req_key(bpf, req, 0), key, map->key_size);
 415         if (value)
 416                 memcpy(nfp_bpf_ctrl_req_val(bpf, req, 0), value,
 417                        map->value_size);
 418
 419         skb = nfp_bpf_cmsg_communicate(bpf, skb, op,
 420                                        nfp_bpf_cmsg_map_reply_size(bpf, 1));
 421         if (IS_ERR(skb))
 422                 return PTR_ERR(skb);
 423
 424         reply = (void *)skb->data;
 425         err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
 426         if (err)
 427                 goto err_free;
 428
 429         /* Copy outputs */
 430         if (out_key)
 431                 memcpy(out_key, nfp_bpf_ctrl_reply_key(bpf, reply, 0),
 432                        map->key_size);
 433         if (out_value)
 434                 memcpy(out_value, nfp_bpf_ctrl_reply_val(bpf, reply, 0),
 435                        map->value_size);
 436
 437         dev_consume_skb_any(skb);
 438
 439         return 0;
 440 err_free:
 441         dev_kfree_skb_any(skb);
 442         return err;
 443 }
 444
 445 int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap,
 446                               void *key, void *value, u64 flags)
 447 {
 448         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_UPDATE,
 449                                      key, value, flags, NULL, NULL);
 450 }
 451
 452 int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key)
 453 {
 454         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_DELETE,
 455                                      key, NULL, 0, NULL, NULL);
 456 }
 457
 458 int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap,
 459                               void *key, void *value)
 460 {
 461         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_LOOKUP,
 462                                      key, NULL, 0, NULL, value);
 463 }
 464
 465 int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap,
 466                                 void *next_key)
 467 {
 468         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETFIRST,
 469                                      NULL, NULL, 0, next_key, NULL);
 470 }
 471
 472 int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap,
 473                                void *key, void *next_key)
 474 {
 475         return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETNEXT,
 476                                      key, NULL, 0, next_key, NULL);
 477 }
 478
 479 unsigned int nfp_bpf_ctrl_cmsg_mtu(struct nfp_app_bpf *bpf)
 480 {
 481         return max3((unsigned int)NFP_NET_DEFAULT_MTU,
 482                     nfp_bpf_cmsg_map_req_size(bpf, 1),
 483                     nfp_bpf_cmsg_map_reply_size(bpf, 1));
 484 }
 485
 486 void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb)
 487 {
 488         struct nfp_app_bpf *bpf = app->priv;
 489         unsigned int tag;
 490
 491         if (unlikely(skb->len < sizeof(struct cmsg_reply_map_simple))) {
 492                 cmsg_warn(bpf, "cmsg drop - too short %d!\n", skb->len);
 493                 goto err_free;
 494         }
 495
 496         if (nfp_bpf_cmsg_get_type(skb) == CMSG_TYPE_BPF_EVENT) {
 497                 if (!nfp_bpf_event_output(bpf, skb->data, skb->len))
 498                         dev_consume_skb_any(skb);
 499                 else
 500                         dev_kfree_skb_any(skb);
 501                 return;
 502         }
 503
 504         nfp_ctrl_lock(bpf->app->ctrl);
 505
 506         tag = nfp_bpf_cmsg_get_tag(skb);
 507         if (unlikely(!test_bit(tag, bpf->tag_allocator))) {
 508                 cmsg_warn(bpf, "cmsg drop - no one is waiting for tag %u!\n",
 509                           tag);
 510                 goto err_unlock;
 511         }
 512
 513         __skb_queue_tail(&bpf->cmsg_replies, skb);
 514         wake_up_interruptible_all(&bpf->cmsg_wq);
 515
 516         nfp_ctrl_unlock(bpf->app->ctrl);
 517
 518         return;
 519 err_unlock:
 520         nfp_ctrl_unlock(bpf->app->ctrl);
 521 err_free:
 522         dev_kfree_skb_any(skb);
 523 }
 524
 525 void
 526 nfp_bpf_ctrl_msg_rx_raw(struct nfp_app *app, const void *data, unsigned int len)
 527 {
 528         struct nfp_app_bpf *bpf = app->priv;
 529         const struct cmsg_hdr *hdr = data;
 530
 531         if (unlikely(len < sizeof(struct cmsg_reply_map_simple))) {
 532                 cmsg_warn(bpf, "cmsg drop - too short %d!\n", len);
 533                 return;
 534         }
 535
 536         if (hdr->type == CMSG_TYPE_BPF_EVENT)
 537                 nfp_bpf_event_output(bpf, data, len);
 538         else
 539                 cmsg_warn(bpf, "cmsg drop - msg type %d with raw buffer!\n",
 540                           hdr->type);
 541 }