tools/lib/bpf/libbpf.c

   1 // SPDX-License-Identifier: LGPL-2.1
   2
   3 /*
   4  * Common eBPF ELF object loading operations.
   5  *
   6  * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
   7  * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
   8  * Copyright (C) 2015 Huawei Inc.
   9  * Copyright (C) 2017 Nicira, Inc.
  10  *
  11  * This program is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation;
  14  * version 2.1 of the License (not later!)
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19  * GNU Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this program; if not,  see <http://www.gnu.org/licenses>
  23  */
  24
  25 #define _GNU_SOURCE
  26 #include <stdlib.h>
  27 #include <stdio.h>
  28 #include <stdarg.h>
  29 #include <libgen.h>
  30 #include <inttypes.h>
  31 #include <string.h>
  32 #include <unistd.h>
  33 #include <fcntl.h>
  34 #include <errno.h>
  35 #include <perf-sys.h>
  36 #include <asm/unistd.h>
  37 #include <linux/err.h>
  38 #include <linux/kernel.h>
  39 #include <linux/bpf.h>
  40 #include <linux/btf.h>
  41 #include <linux/list.h>
  42 #include <linux/limits.h>
  43 #include <sys/stat.h>
  44 #include <sys/types.h>
  45 #include <sys/vfs.h>
  46 #include <tools/libc_compat.h>
  47 #include <libelf.h>
  48 #include <gelf.h>
  49
  50 #include "libbpf.h"
  51 #include "bpf.h"
  52 #include "btf.h"
  53
  54 #ifndef EM_BPF
  55 #define EM_BPF 247
  56 #endif
  57
  58 #ifndef BPF_FS_MAGIC
  59 #define BPF_FS_MAGIC            0xcafe4a11
  60 #endif
  61
  62 #define __printf(a, b)  __attribute__((format(printf, a, b)))
  63
  64 __printf(1, 2)
  65 static int __base_pr(const char *format, ...)
  66 {
  67         va_list args;
  68         int err;
  69
  70         va_start(args, format);
  71         err = vfprintf(stderr, format, args);
  72         va_end(args);
  73         return err;
  74 }
  75
  76 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
  77 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
  78 static __printf(1, 2) libbpf_print_fn_t __pr_debug;
  79
  80 #define __pr(func, fmt, ...)    \
  81 do {                            \
  82         if ((func))             \
  83                 (func)("libbpf: " fmt, ##__VA_ARGS__); \
  84 } while (0)
  85
  86 #define pr_warning(fmt, ...)    __pr(__pr_warning, fmt, ##__VA_ARGS__)
  87 #define pr_info(fmt, ...)       __pr(__pr_info, fmt, ##__VA_ARGS__)
  88 #define pr_debug(fmt, ...)      __pr(__pr_debug, fmt, ##__VA_ARGS__)
  89
  90 void libbpf_set_print(libbpf_print_fn_t warn,
  91                       libbpf_print_fn_t info,
  92                       libbpf_print_fn_t debug)
  93 {
  94         __pr_warning = warn;
  95         __pr_info = info;
  96         __pr_debug = debug;
  97 }
  98
  99 #define STRERR_BUFSIZE  128
 100
 101 #define CHECK_ERR(action, err, out) do {        \
 102         err = action;                   \
 103         if (err)                        \
 104                 goto out;               \
 105 } while(0)
 106
 107
 108 /* Copied from tools/perf/util/util.h */
 109 #ifndef zfree
 110 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
 111 #endif
 112
 113 #ifndef zclose
 114 # define zclose(fd) ({                  \
 115         int ___err = 0;                 \
 116         if ((fd) >= 0)                  \
 117                 ___err = close((fd));   \
 118         fd = -1;                        \
 119         ___err; })
 120 #endif
 121
 122 #ifdef HAVE_LIBELF_MMAP_SUPPORT
 123 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
 124 #else
 125 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
 126 #endif
 127
 128 /*
 129  * bpf_prog should be a better name but it has been used in
 130  * linux/filter.h.
 131  */
 132 struct bpf_program {
 133         /* Index in elf obj file, for relocation use. */
 134         int idx;
 135         char *name;
 136         int prog_ifindex;
 137         char *section_name;
 138         struct bpf_insn *insns;
 139         size_t insns_cnt, main_prog_cnt;
 140         enum bpf_prog_type type;
 141
 142         struct reloc_desc {
 143                 enum {
 144                         RELO_LD64,
 145                         RELO_CALL,
 146                 } type;
 147                 int insn_idx;
 148                 union {
 149                         int map_idx;
 150                         int text_off;
 151                 };
 152         } *reloc_desc;
 153         int nr_reloc;
 154
 155         struct {
 156                 int nr;
 157                 int *fds;
 158         } instances;
 159         bpf_program_prep_t preprocessor;
 160
 161         struct bpf_object *obj;
 162         void *priv;
 163         bpf_program_clear_priv_t clear_priv;
 164
 165         enum bpf_attach_type expected_attach_type;
 166 };
 167
 168 struct bpf_map {
 169         int fd;
 170         char *name;
 171         size_t offset;
 172         int map_ifindex;
 173         struct bpf_map_def def;
 174         __u32 btf_key_type_id;
 175         __u32 btf_value_type_id;
 176         void *priv;
 177         bpf_map_clear_priv_t clear_priv;
 178 };
 179
 180 static LIST_HEAD(bpf_objects_list);
 181
 182 struct bpf_object {
 183         char license[64];
 184         u32 kern_version;
 185
 186         struct bpf_program *programs;
 187         size_t nr_programs;
 188         struct bpf_map *maps;
 189         size_t nr_maps;
 190
 191         bool loaded;
 192         bool has_pseudo_calls;
 193
 194         /*
 195          * Information when doing elf related work. Only valid if fd
 196          * is valid.
 197          */
 198         struct {
 199                 int fd;
 200                 void *obj_buf;
 201                 size_t obj_buf_sz;
 202                 Elf *elf;
 203                 GElf_Ehdr ehdr;
 204                 Elf_Data *symbols;
 205                 size_t strtabidx;
 206                 struct {
 207                         GElf_Shdr shdr;
 208                         Elf_Data *data;
 209                 } *reloc;
 210                 int nr_reloc;
 211                 int maps_shndx;
 212                 int text_shndx;
 213         } efile;
 214         /*
 215          * All loaded bpf_object is linked in a list, which is
 216          * hidden to caller. bpf_objects__<func> handlers deal with
 217          * all objects.
 218          */
 219         struct list_head list;
 220
 221         struct btf *btf;
 222
 223         void *priv;
 224         bpf_object_clear_priv_t clear_priv;
 225
 226         char path[];
 227 };
 228 #define obj_elf_valid(o)        ((o)->efile.elf)
 229
 230 static void bpf_program__unload(struct bpf_program *prog)
 231 {
 232         int i;
 233
 234         if (!prog)
 235                 return;
 236
 237         /*
 238          * If the object is opened but the program was never loaded,
 239          * it is possible that prog->instances.nr == -1.
 240          */
 241         if (prog->instances.nr > 0) {
 242                 for (i = 0; i < prog->instances.nr; i++)
 243                         zclose(prog->instances.fds[i]);
 244         } else if (prog->instances.nr != -1) {
 245                 pr_warning("Internal error: instances.nr is %d\n",
 246                            prog->instances.nr);
 247         }
 248
 249         prog->instances.nr = -1;
 250         zfree(&prog->instances.fds);
 251 }
 252
 253 static void bpf_program__exit(struct bpf_program *prog)
 254 {
 255         if (!prog)
 256                 return;
 257
 258         if (prog->clear_priv)
 259                 prog->clear_priv(prog, prog->priv);
 260
 261         prog->priv = NULL;
 262         prog->clear_priv = NULL;
 263
 264         bpf_program__unload(prog);
 265         zfree(&prog->name);
 266         zfree(&prog->section_name);
 267         zfree(&prog->insns);
 268         zfree(&prog->reloc_desc);
 269
 270         prog->nr_reloc = 0;
 271         prog->insns_cnt = 0;
 272         prog->idx = -1;
 273 }
 274
 275 static int
 276 bpf_program__init(void *data, size_t size, char *section_name, int idx,
 277                   struct bpf_program *prog)
 278 {
 279         if (size < sizeof(struct bpf_insn)) {
 280                 pr_warning("corrupted section '%s'\n", section_name);
 281                 return -EINVAL;
 282         }
 283
 284         bzero(prog, sizeof(*prog));
 285
 286         prog->section_name = strdup(section_name);
 287         if (!prog->section_name) {
 288                 pr_warning("failed to alloc name for prog under section(%d) %s\n",
 289                            idx, section_name);
 290                 goto errout;
 291         }
 292
 293         prog->insns = malloc(size);
 294         if (!prog->insns) {
 295                 pr_warning("failed to alloc insns for prog under section %s\n",
 296                            section_name);
 297                 goto errout;
 298         }
 299         prog->insns_cnt = size / sizeof(struct bpf_insn);
 300         memcpy(prog->insns, data,
 301                prog->insns_cnt * sizeof(struct bpf_insn));
 302         prog->idx = idx;
 303         prog->instances.fds = NULL;
 304         prog->instances.nr = -1;
 305         prog->type = BPF_PROG_TYPE_KPROBE;
 306
 307         return 0;
 308 errout:
 309         bpf_program__exit(prog);
 310         return -ENOMEM;
 311 }
 312
 313 static int
 314 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
 315                         char *section_name, int idx)
 316 {
 317         struct bpf_program prog, *progs;
 318         int nr_progs, err;
 319
 320         err = bpf_program__init(data, size, section_name, idx, &prog);
 321         if (err)
 322                 return err;
 323
 324         progs = obj->programs;
 325         nr_progs = obj->nr_programs;
 326
 327         progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
 328         if (!progs) {
 329                 /*
 330                  * In this case the original obj->programs
 331                  * is still valid, so don't need special treat for
 332                  * bpf_close_object().
 333                  */
 334                 pr_warning("failed to alloc a new program under section '%s'\n",
 335                            section_name);
 336                 bpf_program__exit(&prog);
 337                 return -ENOMEM;
 338         }
 339
 340         pr_debug("found program %s\n", prog.section_name);
 341         obj->programs = progs;
 342         obj->nr_programs = nr_progs + 1;
 343         prog.obj = obj;
 344         progs[nr_progs] = prog;
 345         return 0;
 346 }
 347
 348 static int
 349 bpf_object__init_prog_names(struct bpf_object *obj)
 350 {
 351         Elf_Data *symbols = obj->efile.symbols;
 352         struct bpf_program *prog;
 353         size_t pi, si;
 354
 355         for (pi = 0; pi < obj->nr_programs; pi++) {
 356                 const char *name = NULL;
 357
 358                 prog = &obj->programs[pi];
 359
 360                 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
 361                      si++) {
 362                         GElf_Sym sym;
 363
 364                         if (!gelf_getsym(symbols, si, &sym))
 365                                 continue;
 366                         if (sym.st_shndx != prog->idx)
 367                                 continue;
 368                         if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
 369                                 continue;
 370
 371                         name = elf_strptr(obj->efile.elf,
 372                                           obj->efile.strtabidx,
 373                                           sym.st_name);
 374                         if (!name) {
 375                                 pr_warning("failed to get sym name string for prog %s\n",
 376                                            prog->section_name);
 377                                 return -LIBBPF_ERRNO__LIBELF;
 378                         }
 379                 }
 380
 381                 if (!name && prog->idx == obj->efile.text_shndx)
 382                         name = ".text";
 383
 384                 if (!name) {
 385                         pr_warning("failed to find sym for prog %s\n",
 386                                    prog->section_name);
 387                         return -EINVAL;
 388                 }
 389
 390                 prog->name = strdup(name);
 391                 if (!prog->name) {
 392                         pr_warning("failed to allocate memory for prog sym %s\n",
 393                                    name);
 394                         return -ENOMEM;
 395                 }
 396         }
 397
 398         return 0;
 399 }
 400
 401 static struct bpf_object *bpf_object__new(const char *path,
 402                                           void *obj_buf,
 403                                           size_t obj_buf_sz)
 404 {
 405         struct bpf_object *obj;
 406
 407         obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
 408         if (!obj) {
 409                 pr_warning("alloc memory failed for %s\n", path);
 410                 return ERR_PTR(-ENOMEM);
 411         }
 412
 413         strcpy(obj->path, path);
 414         obj->efile.fd = -1;
 415
 416         /*
 417          * Caller of this function should also calls
 418          * bpf_object__elf_finish() after data collection to return
 419          * obj_buf to user. If not, we should duplicate the buffer to
 420          * avoid user freeing them before elf finish.
 421          */
 422         obj->efile.obj_buf = obj_buf;
 423         obj->efile.obj_buf_sz = obj_buf_sz;
 424         obj->efile.maps_shndx = -1;
 425
 426         obj->loaded = false;
 427
 428         INIT_LIST_HEAD(&obj->list);
 429         list_add(&obj->list, &bpf_objects_list);
 430         return obj;
 431 }
 432
 433 static void bpf_object__elf_finish(struct bpf_object *obj)
 434 {
 435         if (!obj_elf_valid(obj))
 436                 return;
 437
 438         if (obj->efile.elf) {
 439                 elf_end(obj->efile.elf);
 440                 obj->efile.elf = NULL;
 441         }
 442         obj->efile.symbols = NULL;
 443
 444         zfree(&obj->efile.reloc);
 445         obj->efile.nr_reloc = 0;
 446         zclose(obj->efile.fd);
 447         obj->efile.obj_buf = NULL;
 448         obj->efile.obj_buf_sz = 0;
 449 }
 450
 451 static int bpf_object__elf_init(struct bpf_object *obj)
 452 {
 453         int err = 0;
 454         GElf_Ehdr *ep;
 455
 456         if (obj_elf_valid(obj)) {
 457                 pr_warning("elf init: internal error\n");
 458                 return -LIBBPF_ERRNO__LIBELF;
 459         }
 460
 461         if (obj->efile.obj_buf_sz > 0) {
 462                 /*
 463                  * obj_buf should have been validated by
 464                  * bpf_object__open_buffer().
 465                  */
 466                 obj->efile.elf = elf_memory(obj->efile.obj_buf,
 467                                             obj->efile.obj_buf_sz);
 468         } else {
 469                 obj->efile.fd = open(obj->path, O_RDONLY);
 470                 if (obj->efile.fd < 0) {
 471                         char errmsg[STRERR_BUFSIZE];
 472                         char *cp = strerror_r(errno, errmsg, sizeof(errmsg));
 473
 474                         pr_warning("failed to open %s: %s\n", obj->path, cp);
 475                         return -errno;
 476                 }
 477
 478                 obj->efile.elf = elf_begin(obj->efile.fd,
 479                                 LIBBPF_ELF_C_READ_MMAP,
 480                                 NULL);
 481         }
 482
 483         if (!obj->efile.elf) {
 484                 pr_warning("failed to open %s as ELF file\n",
 485                                 obj->path);
 486                 err = -LIBBPF_ERRNO__LIBELF;
 487                 goto errout;
 488         }
 489
 490         if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
 491                 pr_warning("failed to get EHDR from %s\n",
 492                                 obj->path);
 493                 err = -LIBBPF_ERRNO__FORMAT;
 494                 goto errout;
 495         }
 496         ep = &obj->efile.ehdr;
 497
 498         /* Old LLVM set e_machine to EM_NONE */
 499         if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
 500                 pr_warning("%s is not an eBPF object file\n",
 501                         obj->path);
 502                 err = -LIBBPF_ERRNO__FORMAT;
 503                 goto errout;
 504         }
 505
 506         return 0;
 507 errout:
 508         bpf_object__elf_finish(obj);
 509         return err;
 510 }
 511
 512 static int
 513 bpf_object__check_endianness(struct bpf_object *obj)
 514 {
 515         static unsigned int const endian = 1;
 516
 517         switch (obj->efile.ehdr.e_ident[EI_DATA]) {
 518         case ELFDATA2LSB:
 519                 /* We are big endian, BPF obj is little endian. */
 520                 if (*(unsigned char const *)&endian != 1)
 521                         goto mismatch;
 522                 break;
 523
 524         case ELFDATA2MSB:
 525                 /* We are little endian, BPF obj is big endian. */
 526                 if (*(unsigned char const *)&endian != 0)
 527                         goto mismatch;
 528                 break;
 529         default:
 530                 return -LIBBPF_ERRNO__ENDIAN;
 531         }
 532
 533         return 0;
 534
 535 mismatch:
 536         pr_warning("Error: endianness mismatch.\n");
 537         return -LIBBPF_ERRNO__ENDIAN;
 538 }
 539
 540 static int
 541 bpf_object__init_license(struct bpf_object *obj,
 542                          void *data, size_t size)
 543 {
 544         memcpy(obj->license, data,
 545                min(size, sizeof(obj->license) - 1));
 546         pr_debug("license of %s is %s\n", obj->path, obj->license);
 547         return 0;
 548 }
 549
 550 static int
 551 bpf_object__init_kversion(struct bpf_object *obj,
 552                           void *data, size_t size)
 553 {
 554         u32 kver;
 555
 556         if (size != sizeof(kver)) {
 557                 pr_warning("invalid kver section in %s\n", obj->path);
 558                 return -LIBBPF_ERRNO__FORMAT;
 559         }
 560         memcpy(&kver, data, sizeof(kver));
 561         obj->kern_version = kver;
 562         pr_debug("kernel version of %s is %x\n", obj->path,
 563                  obj->kern_version);
 564         return 0;
 565 }
 566
 567 static int compare_bpf_map(const void *_a, const void *_b)
 568 {
 569         const struct bpf_map *a = _a;
 570         const struct bpf_map *b = _b;
 571
 572         return a->offset - b->offset;
 573 }
 574
 575 static int
 576 bpf_object__init_maps(struct bpf_object *obj)
 577 {
 578         int i, map_idx, map_def_sz, nr_maps = 0;
 579         Elf_Scn *scn;
 580         Elf_Data *data;
 581         Elf_Data *symbols = obj->efile.symbols;
 582
 583         if (obj->efile.maps_shndx < 0)
 584                 return -EINVAL;
 585         if (!symbols)
 586                 return -EINVAL;
 587
 588         scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx);
 589         if (scn)
 590                 data = elf_getdata(scn, NULL);
 591         if (!scn || !data) {
 592                 pr_warning("failed to get Elf_Data from map section %d\n",
 593                            obj->efile.maps_shndx);
 594                 return -EINVAL;
 595         }
 596
 597         /*
 598          * Count number of maps. Each map has a name.
 599          * Array of maps is not supported: only the first element is
 600          * considered.
 601          *
 602          * TODO: Detect array of map and report error.
 603          */
 604         for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 605                 GElf_Sym sym;
 606
 607                 if (!gelf_getsym(symbols, i, &sym))
 608                         continue;
 609                 if (sym.st_shndx != obj->efile.maps_shndx)
 610                         continue;
 611                 nr_maps++;
 612         }
 613
 614         /* Alloc obj->maps and fill nr_maps. */
 615         pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
 616                  nr_maps, data->d_size);
 617
 618         if (!nr_maps)
 619                 return 0;
 620
 621         /* Assume equally sized map definitions */
 622         map_def_sz = data->d_size / nr_maps;
 623         if (!data->d_size || (data->d_size % nr_maps) != 0) {
 624                 pr_warning("unable to determine map definition size "
 625                            "section %s, %d maps in %zd bytes\n",
 626                            obj->path, nr_maps, data->d_size);
 627                 return -EINVAL;
 628         }
 629
 630         obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
 631         if (!obj->maps) {
 632                 pr_warning("alloc maps for object failed\n");
 633                 return -ENOMEM;
 634         }
 635         obj->nr_maps = nr_maps;
 636
 637         /*
 638          * fill all fd with -1 so won't close incorrect
 639          * fd (fd=0 is stdin) when failure (zclose won't close
 640          * negative fd)).
 641          */
 642         for (i = 0; i < nr_maps; i++)
 643                 obj->maps[i].fd = -1;
 644
 645         /*
 646          * Fill obj->maps using data in "maps" section.
 647          */
 648         for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) {
 649                 GElf_Sym sym;
 650                 const char *map_name;
 651                 struct bpf_map_def *def;
 652
 653                 if (!gelf_getsym(symbols, i, &sym))
 654                         continue;
 655                 if (sym.st_shndx != obj->efile.maps_shndx)
 656                         continue;
 657
 658                 map_name = elf_strptr(obj->efile.elf,
 659                                       obj->efile.strtabidx,
 660                                       sym.st_name);
 661                 obj->maps[map_idx].offset = sym.st_value;
 662                 if (sym.st_value + map_def_sz > data->d_size) {
 663                         pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
 664                                    obj->path, map_name);
 665                         return -EINVAL;
 666                 }
 667
 668                 obj->maps[map_idx].name = strdup(map_name);
 669                 if (!obj->maps[map_idx].name) {
 670                         pr_warning("failed to alloc map name\n");
 671                         return -ENOMEM;
 672                 }
 673                 pr_debug("map %d is \"%s\"\n", map_idx,
 674                          obj->maps[map_idx].name);
 675                 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
 676                 /*
 677                  * If the definition of the map in the object file fits in
 678                  * bpf_map_def, copy it.  Any extra fields in our version
 679                  * of bpf_map_def will default to zero as a result of the
 680                  * calloc above.
 681                  */
 682                 if (map_def_sz <= sizeof(struct bpf_map_def)) {
 683                         memcpy(&obj->maps[map_idx].def, def, map_def_sz);
 684                 } else {
 685                         /*
 686                          * Here the map structure being read is bigger than what
 687                          * we expect, truncate if the excess bits are all zero.
 688                          * If they are not zero, reject this map as
 689                          * incompatible.
 690                          */
 691                         char *b;
 692                         for (b = ((char *)def) + sizeof(struct bpf_map_def);
 693                              b < ((char *)def) + map_def_sz; b++) {
 694                                 if (*b != 0) {
 695                                         pr_warning("maps section in %s: \"%s\" "
 696                                                    "has unrecognized, non-zero "
 697                                                    "options\n",
 698                                                    obj->path, map_name);
 699                                         return -EINVAL;
 700                                 }
 701                         }
 702                         memcpy(&obj->maps[map_idx].def, def,
 703                                sizeof(struct bpf_map_def));
 704                 }
 705                 map_idx++;
 706         }
 707
 708         qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map);
 709         return 0;
 710 }
 711
 712 static bool section_have_execinstr(struct bpf_object *obj, int idx)
 713 {
 714         Elf_Scn *scn;
 715         GElf_Shdr sh;
 716
 717         scn = elf_getscn(obj->efile.elf, idx);
 718         if (!scn)
 719                 return false;
 720
 721         if (gelf_getshdr(scn, &sh) != &sh)
 722                 return false;
 723
 724         if (sh.sh_flags & SHF_EXECINSTR)
 725                 return true;
 726
 727         return false;
 728 }
 729
 730 static int bpf_object__elf_collect(struct bpf_object *obj)
 731 {
 732         Elf *elf = obj->efile.elf;
 733         GElf_Ehdr *ep = &obj->efile.ehdr;
 734         Elf_Scn *scn = NULL;
 735         int idx = 0, err = 0;
 736
 737         /* Elf is corrupted/truncated, avoid calling elf_strptr. */
 738         if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
 739                 pr_warning("failed to get e_shstrndx from %s\n",
 740                            obj->path);
 741                 return -LIBBPF_ERRNO__FORMAT;
 742         }
 743
 744         while ((scn = elf_nextscn(elf, scn)) != NULL) {
 745                 char *name;
 746                 GElf_Shdr sh;
 747                 Elf_Data *data;
 748
 749                 idx++;
 750                 if (gelf_getshdr(scn, &sh) != &sh) {
 751                         pr_warning("failed to get section(%d) header from %s\n",
 752                                    idx, obj->path);
 753                         err = -LIBBPF_ERRNO__FORMAT;
 754                         goto out;
 755                 }
 756
 757                 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
 758                 if (!name) {
 759                         pr_warning("failed to get section(%d) name from %s\n",
 760                                    idx, obj->path);
 761                         err = -LIBBPF_ERRNO__FORMAT;
 762                         goto out;
 763                 }
 764
 765                 data = elf_getdata(scn, 0);
 766                 if (!data) {
 767                         pr_warning("failed to get section(%d) data from %s(%s)\n",
 768                                    idx, name, obj->path);
 769                         err = -LIBBPF_ERRNO__FORMAT;
 770                         goto out;
 771                 }
 772                 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
 773                          idx, name, (unsigned long)data->d_size,
 774                          (int)sh.sh_link, (unsigned long)sh.sh_flags,
 775                          (int)sh.sh_type);
 776
 777                 if (strcmp(name, "license") == 0)
 778                         err = bpf_object__init_license(obj,
 779                                                        data->d_buf,
 780                                                        data->d_size);
 781                 else if (strcmp(name, "version") == 0)
 782                         err = bpf_object__init_kversion(obj,
 783                                                         data->d_buf,
 784                                                         data->d_size);
 785                 else if (strcmp(name, "maps") == 0)
 786                         obj->efile.maps_shndx = idx;
 787                 else if (strcmp(name, BTF_ELF_SEC) == 0) {
 788                         obj->btf = btf__new(data->d_buf, data->d_size,
 789                                             __pr_debug);
 790                         if (IS_ERR(obj->btf)) {
 791                                 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
 792                                            BTF_ELF_SEC, PTR_ERR(obj->btf));
 793                                 obj->btf = NULL;
 794                         }
 795                 } else if (sh.sh_type == SHT_SYMTAB) {
 796                         if (obj->efile.symbols) {
 797                                 pr_warning("bpf: multiple SYMTAB in %s\n",
 798                                            obj->path);
 799                                 err = -LIBBPF_ERRNO__FORMAT;
 800                         } else {
 801                                 obj->efile.symbols = data;
 802                                 obj->efile.strtabidx = sh.sh_link;
 803                         }
 804                 } else if ((sh.sh_type == SHT_PROGBITS) &&
 805                            (sh.sh_flags & SHF_EXECINSTR) &&
 806                            (data->d_size > 0)) {
 807                         if (strcmp(name, ".text") == 0)
 808                                 obj->efile.text_shndx = idx;
 809                         err = bpf_object__add_program(obj, data->d_buf,
 810                                                       data->d_size, name, idx);
 811                         if (err) {
 812                                 char errmsg[STRERR_BUFSIZE];
 813                                 char *cp = strerror_r(-err, errmsg,
 814                                                       sizeof(errmsg));
 815
 816                                 pr_warning("failed to alloc program %s (%s): %s",
 817                                            name, obj->path, cp);
 818                         }
 819                 } else if (sh.sh_type == SHT_REL) {
 820                         void *reloc = obj->efile.reloc;
 821                         int nr_reloc = obj->efile.nr_reloc + 1;
 822                         int sec = sh.sh_info; /* points to other section */
 823
 824                         /* Only do relo for section with exec instructions */
 825                         if (!section_have_execinstr(obj, sec)) {
 826                                 pr_debug("skip relo %s(%d) for section(%d)\n",
 827                                          name, idx, sec);
 828                                 continue;
 829                         }
 830
 831                         reloc = reallocarray(reloc, nr_reloc,
 832                                              sizeof(*obj->efile.reloc));
 833                         if (!reloc) {
 834                                 pr_warning("realloc failed\n");
 835                                 err = -ENOMEM;
 836                         } else {
 837                                 int n = nr_reloc - 1;
 838
 839                                 obj->efile.reloc = reloc;
 840                                 obj->efile.nr_reloc = nr_reloc;
 841
 842                                 obj->efile.reloc[n].shdr = sh;
 843                                 obj->efile.reloc[n].data = data;
 844                         }
 845                 } else {
 846                         pr_debug("skip section(%d) %s\n", idx, name);
 847                 }
 848                 if (err)
 849                         goto out;
 850         }
 851
 852         if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
 853                 pr_warning("Corrupted ELF file: index of strtab invalid\n");
 854                 return LIBBPF_ERRNO__FORMAT;
 855         }
 856         if (obj->efile.maps_shndx >= 0) {
 857                 err = bpf_object__init_maps(obj);
 858                 if (err)
 859                         goto out;
 860         }
 861         err = bpf_object__init_prog_names(obj);
 862 out:
 863         return err;
 864 }
 865
 866 static struct bpf_program *
 867 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
 868 {
 869         struct bpf_program *prog;
 870         size_t i;
 871
 872         for (i = 0; i < obj->nr_programs; i++) {
 873                 prog = &obj->programs[i];
 874                 if (prog->idx == idx)
 875                         return prog;
 876         }
 877         return NULL;
 878 }
 879
 880 struct bpf_program *
 881 bpf_object__find_program_by_title(struct bpf_object *obj, const char *title)
 882 {
 883         struct bpf_program *pos;
 884
 885         bpf_object__for_each_program(pos, obj) {
 886                 if (pos->section_name && !strcmp(pos->section_name, title))
 887                         return pos;
 888         }
 889         return NULL;
 890 }
 891
 892 static int
 893 bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
 894                            Elf_Data *data, struct bpf_object *obj)
 895 {
 896         Elf_Data *symbols = obj->efile.symbols;
 897         int text_shndx = obj->efile.text_shndx;
 898         int maps_shndx = obj->efile.maps_shndx;
 899         struct bpf_map *maps = obj->maps;
 900         size_t nr_maps = obj->nr_maps;
 901         int i, nrels;
 902
 903         pr_debug("collecting relocating info for: '%s'\n",
 904                  prog->section_name);
 905         nrels = shdr->sh_size / shdr->sh_entsize;
 906
 907         prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
 908         if (!prog->reloc_desc) {
 909                 pr_warning("failed to alloc memory in relocation\n");
 910                 return -ENOMEM;
 911         }
 912         prog->nr_reloc = nrels;
 913
 914         for (i = 0; i < nrels; i++) {
 915                 GElf_Sym sym;
 916                 GElf_Rel rel;
 917                 unsigned int insn_idx;
 918                 struct bpf_insn *insns = prog->insns;
 919                 size_t map_idx;
 920
 921                 if (!gelf_getrel(data, i, &rel)) {
 922                         pr_warning("relocation: failed to get %d reloc\n", i);
 923                         return -LIBBPF_ERRNO__FORMAT;
 924                 }
 925
 926                 if (!gelf_getsym(symbols,
 927                                  GELF_R_SYM(rel.r_info),
 928                                  &sym)) {
 929                         pr_warning("relocation: symbol %"PRIx64" not found\n",
 930                                    GELF_R_SYM(rel.r_info));
 931                         return -LIBBPF_ERRNO__FORMAT;
 932                 }
 933                 pr_debug("relo for %lld value %lld name %d\n",
 934                          (long long) (rel.r_info >> 32),
 935                          (long long) sym.st_value, sym.st_name);
 936
 937                 if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) {
 938                         pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n",
 939                                    prog->section_name, sym.st_shndx);
 940                         return -LIBBPF_ERRNO__RELOC;
 941                 }
 942
 943                 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
 944                 pr_debug("relocation: insn_idx=%u\n", insn_idx);
 945
 946                 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
 947                         if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
 948                                 pr_warning("incorrect bpf_call opcode\n");
 949                                 return -LIBBPF_ERRNO__RELOC;
 950                         }
 951                         prog->reloc_desc[i].type = RELO_CALL;
 952                         prog->reloc_desc[i].insn_idx = insn_idx;
 953                         prog->reloc_desc[i].text_off = sym.st_value;
 954                         obj->has_pseudo_calls = true;
 955                         continue;
 956                 }
 957
 958                 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
 959                         pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
 960                                    insn_idx, insns[insn_idx].code);
 961                         return -LIBBPF_ERRNO__RELOC;
 962                 }
 963
 964                 /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */
 965                 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
 966                         if (maps[map_idx].offset == sym.st_value) {
 967                                 pr_debug("relocation: find map %zd (%s) for insn %u\n",
 968                                          map_idx, maps[map_idx].name, insn_idx);
 969                                 break;
 970                         }
 971                 }
 972
 973                 if (map_idx >= nr_maps) {
 974                         pr_warning("bpf relocation: map_idx %d large than %d\n",
 975                                    (int)map_idx, (int)nr_maps - 1);
 976                         return -LIBBPF_ERRNO__RELOC;
 977                 }
 978
 979                 prog->reloc_desc[i].type = RELO_LD64;
 980                 prog->reloc_desc[i].insn_idx = insn_idx;
 981                 prog->reloc_desc[i].map_idx = map_idx;
 982         }
 983         return 0;
 984 }
 985
 986 static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
 987 {
 988         const struct btf_type *container_type;
 989         const struct btf_member *key, *value;
 990         struct bpf_map_def *def = &map->def;
 991         const size_t max_name = 256;
 992         char container_name[max_name];
 993         __s64 key_size, value_size;
 994         __s32 container_id;
 995
 996         if (snprintf(container_name, max_name, "____btf_map_%s", map->name) ==
 997             max_name) {
 998                 pr_warning("map:%s length of '____btf_map_%s' is too long\n",
 999                            map->name, map->name);
1000                 return -EINVAL;
1001         }
1002
1003         container_id = btf__find_by_name(btf, container_name);
1004         if (container_id < 0) {
1005                 pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n",
1006                          map->name, container_name);
1007                 return container_id;
1008         }
1009
1010         container_type = btf__type_by_id(btf, container_id);
1011         if (!container_type) {
1012                 pr_warning("map:%s cannot find BTF type for container_id:%u\n",
1013                            map->name, container_id);
1014                 return -EINVAL;
1015         }
1016
1017         if (BTF_INFO_KIND(container_type->info) != BTF_KIND_STRUCT ||
1018             BTF_INFO_VLEN(container_type->info) < 2) {
1019                 pr_warning("map:%s container_name:%s is an invalid container struct\n",
1020                            map->name, container_name);
1021                 return -EINVAL;
1022         }
1023
1024         key = (struct btf_member *)(container_type + 1);
1025         value = key + 1;
1026
1027         key_size = btf__resolve_size(btf, key->type);
1028         if (key_size < 0) {
1029                 pr_warning("map:%s invalid BTF key_type_size\n",
1030                            map->name);
1031                 return key_size;
1032         }
1033
1034         if (def->key_size != key_size) {
1035                 pr_warning("map:%s btf_key_type_size:%u != map_def_key_size:%u\n",
1036                            map->name, (__u32)key_size, def->key_size);
1037                 return -EINVAL;
1038         }
1039
1040         value_size = btf__resolve_size(btf, value->type);
1041         if (value_size < 0) {
1042                 pr_warning("map:%s invalid BTF value_type_size\n", map->name);
1043                 return value_size;
1044         }
1045
1046         if (def->value_size != value_size) {
1047                 pr_warning("map:%s btf_value_type_size:%u != map_def_value_size:%u\n",
1048                            map->name, (__u32)value_size, def->value_size);
1049                 return -EINVAL;
1050         }
1051
1052         map->btf_key_type_id = key->type;
1053         map->btf_value_type_id = value->type;
1054
1055         return 0;
1056 }
1057
1058 int bpf_map__reuse_fd(struct bpf_map *map, int fd)
1059 {
1060         struct bpf_map_info info = {};
1061         __u32 len = sizeof(info);
1062         int new_fd, err;
1063         char *new_name;
1064
1065         err = bpf_obj_get_info_by_fd(fd, &info, &len);
1066         if (err)
1067                 return err;
1068
1069         new_name = strdup(info.name);
1070         if (!new_name)
1071                 return -errno;
1072
1073         new_fd = open("/", O_RDONLY | O_CLOEXEC);
1074         if (new_fd < 0)
1075                 goto err_free_new_name;
1076
1077         new_fd = dup3(fd, new_fd, O_CLOEXEC);
1078         if (new_fd < 0)
1079                 goto err_close_new_fd;
1080
1081         err = zclose(map->fd);
1082         if (err)
1083                 goto err_close_new_fd;
1084         free(map->name);
1085
1086         map->fd = new_fd;
1087         map->name = new_name;
1088         map->def.type = info.type;
1089         map->def.key_size = info.key_size;
1090         map->def.value_size = info.value_size;
1091         map->def.max_entries = info.max_entries;
1092         map->def.map_flags = info.map_flags;
1093         map->btf_key_type_id = info.btf_key_type_id;
1094         map->btf_value_type_id = info.btf_value_type_id;
1095
1096         return 0;
1097
1098 err_close_new_fd:
1099         close(new_fd);
1100 err_free_new_name:
1101         free(new_name);
1102         return -errno;
1103 }
1104
1105 static int
1106 bpf_object__create_maps(struct bpf_object *obj)
1107 {
1108         struct bpf_create_map_attr create_attr = {};
1109         unsigned int i;
1110         int err;
1111
1112         for (i = 0; i < obj->nr_maps; i++) {
1113                 struct bpf_map *map = &obj->maps[i];
1114                 struct bpf_map_def *def = &map->def;
1115                 char *cp, errmsg[STRERR_BUFSIZE];
1116                 int *pfd = &map->fd;
1117
1118                 if (map->fd >= 0) {
1119                         pr_debug("skip map create (preset) %s: fd=%d\n",
1120                                  map->name, map->fd);
1121                         continue;
1122                 }
1123
1124                 create_attr.name = map->name;
1125                 create_attr.map_ifindex = map->map_ifindex;
1126                 create_attr.map_type = def->type;
1127                 create_attr.map_flags = def->map_flags;
1128                 create_attr.key_size = def->key_size;
1129                 create_attr.value_size = def->value_size;
1130                 create_attr.max_entries = def->max_entries;
1131                 create_attr.btf_fd = 0;
1132                 create_attr.btf_key_type_id = 0;
1133                 create_attr.btf_value_type_id = 0;
1134
1135                 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
1136                         create_attr.btf_fd = btf__fd(obj->btf);
1137                         create_attr.btf_key_type_id = map->btf_key_type_id;
1138                         create_attr.btf_value_type_id = map->btf_value_type_id;
1139                 }
1140
1141                 *pfd = bpf_create_map_xattr(&create_attr);
1142                 if (*pfd < 0 && create_attr.btf_key_type_id) {
1143                         cp = strerror_r(errno, errmsg, sizeof(errmsg));
1144                         pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
1145                                    map->name, cp, errno);
1146                         create_attr.btf_fd = 0;
1147                         create_attr.btf_key_type_id = 0;
1148                         create_attr.btf_value_type_id = 0;
1149                         map->btf_key_type_id = 0;
1150                         map->btf_value_type_id = 0;
1151                         *pfd = bpf_create_map_xattr(&create_attr);
1152                 }
1153
1154                 if (*pfd < 0) {
1155                         size_t j;
1156
1157                         err = *pfd;
1158                         cp = strerror_r(errno, errmsg, sizeof(errmsg));
1159                         pr_warning("failed to create map (name: '%s'): %s\n",
1160                                    map->name, cp);
1161                         for (j = 0; j < i; j++)
1162                                 zclose(obj->maps[j].fd);
1163                         return err;
1164                 }
1165                 pr_debug("create map %s: fd=%d\n", map->name, *pfd);
1166         }
1167
1168         return 0;
1169 }
1170
1171 static int
1172 bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
1173                         struct reloc_desc *relo)
1174 {
1175         struct bpf_insn *insn, *new_insn;
1176         struct bpf_program *text;
1177         size_t new_cnt;
1178
1179         if (relo->type != RELO_CALL)
1180                 return -LIBBPF_ERRNO__RELOC;
1181
1182         if (prog->idx == obj->efile.text_shndx) {
1183                 pr_warning("relo in .text insn %d into off %d\n",
1184                            relo->insn_idx, relo->text_off);
1185                 return -LIBBPF_ERRNO__RELOC;
1186         }
1187
1188         if (prog->main_prog_cnt == 0) {
1189                 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
1190                 if (!text) {
1191                         pr_warning("no .text section found yet relo into text exist\n");
1192                         return -LIBBPF_ERRNO__RELOC;
1193                 }
1194                 new_cnt = prog->insns_cnt + text->insns_cnt;
1195                 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
1196                 if (!new_insn) {
1197                         pr_warning("oom in prog realloc\n");
1198                         return -ENOMEM;
1199                 }
1200                 memcpy(new_insn + prog->insns_cnt, text->insns,
1201                        text->insns_cnt * sizeof(*insn));
1202                 prog->insns = new_insn;
1203                 prog->main_prog_cnt = prog->insns_cnt;
1204                 prog->insns_cnt = new_cnt;
1205                 pr_debug("added %zd insn from %s to prog %s\n",
1206                          text->insns_cnt, text->section_name,
1207                          prog->section_name);
1208         }
1209         insn = &prog->insns[relo->insn_idx];
1210         insn->imm += prog->main_prog_cnt - relo->insn_idx;
1211         return 0;
1212 }
1213
1214 static int
1215 bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
1216 {
1217         int i, err;
1218
1219         if (!prog || !prog->reloc_desc)
1220                 return 0;
1221
1222         for (i = 0; i < prog->nr_reloc; i++) {
1223                 if (prog->reloc_desc[i].type == RELO_LD64) {
1224                         struct bpf_insn *insns = prog->insns;
1225                         int insn_idx, map_idx;
1226
1227                         insn_idx = prog->reloc_desc[i].insn_idx;
1228                         map_idx = prog->reloc_desc[i].map_idx;
1229
1230                         if (insn_idx >= (int)prog->insns_cnt) {
1231                                 pr_warning("relocation out of range: '%s'\n",
1232                                            prog->section_name);
1233                                 return -LIBBPF_ERRNO__RELOC;
1234                         }
1235                         insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1236                         insns[insn_idx].imm = obj->maps[map_idx].fd;
1237                 } else {
1238                         err = bpf_program__reloc_text(prog, obj,
1239                                                       &prog->reloc_desc[i]);
1240                         if (err)
1241                                 return err;
1242                 }
1243         }
1244
1245         zfree(&prog->reloc_desc);
1246         prog->nr_reloc = 0;
1247         return 0;
1248 }
1249
1250
1251 static int
1252 bpf_object__relocate(struct bpf_object *obj)
1253 {
1254         struct bpf_program *prog;
1255         size_t i;
1256         int err;
1257
1258         for (i = 0; i < obj->nr_programs; i++) {
1259                 prog = &obj->programs[i];
1260
1261                 err = bpf_program__relocate(prog, obj);
1262                 if (err) {
1263                         pr_warning("failed to relocate '%s'\n",
1264                                    prog->section_name);
1265                         return err;
1266                 }
1267         }
1268         return 0;
1269 }
1270
1271 static int bpf_object__collect_reloc(struct bpf_object *obj)
1272 {
1273         int i, err;
1274
1275         if (!obj_elf_valid(obj)) {
1276                 pr_warning("Internal error: elf object is closed\n");
1277                 return -LIBBPF_ERRNO__INTERNAL;
1278         }
1279
1280         for (i = 0; i < obj->efile.nr_reloc; i++) {
1281                 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
1282                 Elf_Data *data = obj->efile.reloc[i].data;
1283                 int idx = shdr->sh_info;
1284                 struct bpf_program *prog;
1285
1286                 if (shdr->sh_type != SHT_REL) {
1287                         pr_warning("internal error at %d\n", __LINE__);
1288                         return -LIBBPF_ERRNO__INTERNAL;
1289                 }
1290
1291                 prog = bpf_object__find_prog_by_idx(obj, idx);
1292                 if (!prog) {
1293                         pr_warning("relocation failed: no section(%d)\n", idx);
1294                         return -LIBBPF_ERRNO__RELOC;
1295                 }
1296
1297                 err = bpf_program__collect_reloc(prog,
1298                                                  shdr, data,
1299                                                  obj);
1300                 if (err)
1301                         return err;
1302         }
1303         return 0;
1304 }
1305
1306 static int
1307 load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type,
1308              const char *name, struct bpf_insn *insns, int insns_cnt,
1309              char *license, u32 kern_version, int *pfd, int prog_ifindex)
1310 {
1311         struct bpf_load_program_attr load_attr;
1312         char *cp, errmsg[STRERR_BUFSIZE];
1313         char *log_buf;
1314         int ret;
1315
1316         memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
1317         load_attr.prog_type = type;
1318         load_attr.expected_attach_type = expected_attach_type;
1319         load_attr.name = name;
1320         load_attr.insns = insns;
1321         load_attr.insns_cnt = insns_cnt;
1322         load_attr.license = license;
1323         load_attr.kern_version = kern_version;
1324         load_attr.prog_ifindex = prog_ifindex;
1325
1326         if (!load_attr.insns || !load_attr.insns_cnt)
1327                 return -EINVAL;
1328
1329         log_buf = malloc(BPF_LOG_BUF_SIZE);
1330         if (!log_buf)
1331                 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
1332
1333         ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE);
1334
1335         if (ret >= 0) {
1336                 *pfd = ret;
1337                 ret = 0;
1338                 goto out;
1339         }
1340
1341         ret = -LIBBPF_ERRNO__LOAD;
1342         cp = strerror_r(errno, errmsg, sizeof(errmsg));
1343         pr_warning("load bpf program failed: %s\n", cp);
1344
1345         if (log_buf && log_buf[0] != '\0') {
1346                 ret = -LIBBPF_ERRNO__VERIFY;
1347                 pr_warning("-- BEGIN DUMP LOG ---\n");
1348                 pr_warning("\n%s\n", log_buf);
1349                 pr_warning("-- END LOG --\n");
1350         } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
1351                 pr_warning("Program too large (%zu insns), at most %d insns\n",
1352                            load_attr.insns_cnt, BPF_MAXINSNS);
1353                 ret = -LIBBPF_ERRNO__PROG2BIG;
1354         } else {
1355                 /* Wrong program type? */
1356                 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
1357                         int fd;
1358
1359                         load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
1360                         load_attr.expected_attach_type = 0;
1361                         fd = bpf_load_program_xattr(&load_attr, NULL, 0);
1362                         if (fd >= 0) {
1363                                 close(fd);
1364                                 ret = -LIBBPF_ERRNO__PROGTYPE;
1365                                 goto out;
1366                         }
1367                 }
1368
1369                 if (log_buf)
1370                         ret = -LIBBPF_ERRNO__KVER;
1371         }
1372
1373 out:
1374         free(log_buf);
1375         return ret;
1376 }
1377
1378 static int
1379 bpf_program__load(struct bpf_program *prog,
1380                   char *license, u32 kern_version)
1381 {
1382         int err = 0, fd, i;
1383
1384         if (prog->instances.nr < 0 || !prog->instances.fds) {
1385                 if (prog->preprocessor) {
1386                         pr_warning("Internal error: can't load program '%s'\n",
1387                                    prog->section_name);
1388                         return -LIBBPF_ERRNO__INTERNAL;
1389                 }
1390
1391                 prog->instances.fds = malloc(sizeof(int));
1392                 if (!prog->instances.fds) {
1393                         pr_warning("Not enough memory for BPF fds\n");
1394                         return -ENOMEM;
1395                 }
1396                 prog->instances.nr = 1;
1397                 prog->instances.fds[0] = -1;
1398         }
1399
1400         if (!prog->preprocessor) {
1401                 if (prog->instances.nr != 1) {
1402                         pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
1403                                    prog->section_name, prog->instances.nr);
1404                 }
1405                 err = load_program(prog->type, prog->expected_attach_type,
1406                                    prog->name, prog->insns, prog->insns_cnt,
1407                                    license, kern_version, &fd,
1408                                    prog->prog_ifindex);
1409                 if (!err)
1410                         prog->instances.fds[0] = fd;
1411                 goto out;
1412         }
1413
1414         for (i = 0; i < prog->instances.nr; i++) {
1415                 struct bpf_prog_prep_result result;
1416                 bpf_program_prep_t preprocessor = prog->preprocessor;
1417
1418                 bzero(&result, sizeof(result));
1419                 err = preprocessor(prog, i, prog->insns,
1420                                    prog->insns_cnt, &result);
1421                 if (err) {
1422                         pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
1423                                    i, prog->section_name);
1424                         goto out;
1425                 }
1426
1427                 if (!result.new_insn_ptr || !result.new_insn_cnt) {
1428                         pr_debug("Skip loading the %dth instance of program '%s'\n",
1429                                  i, prog->section_name);
1430                         prog->instances.fds[i] = -1;
1431                         if (result.pfd)
1432                                 *result.pfd = -1;
1433                         continue;
1434                 }
1435
1436                 err = load_program(prog->type, prog->expected_attach_type,
1437                                    prog->name, result.new_insn_ptr,
1438                                    result.new_insn_cnt,
1439                                    license, kern_version, &fd,
1440                                    prog->prog_ifindex);
1441
1442                 if (err) {
1443                         pr_warning("Loading the %dth instance of program '%s' failed\n",
1444                                         i, prog->section_name);
1445                         goto out;
1446                 }
1447
1448                 if (result.pfd)
1449                         *result.pfd = fd;
1450                 prog->instances.fds[i] = fd;
1451         }
1452 out:
1453         if (err)
1454                 pr_warning("failed to load program '%s'\n",
1455                            prog->section_name);
1456         zfree(&prog->insns);
1457         prog->insns_cnt = 0;
1458         return err;
1459 }
1460
1461 static bool bpf_program__is_function_storage(struct bpf_program *prog,
1462                                              struct bpf_object *obj)
1463 {
1464         return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
1465 }
1466
1467 static int
1468 bpf_object__load_progs(struct bpf_object *obj)
1469 {
1470         size_t i;
1471         int err;
1472
1473         for (i = 0; i < obj->nr_programs; i++) {
1474                 if (bpf_program__is_function_storage(&obj->programs[i], obj))
1475                         continue;
1476                 err = bpf_program__load(&obj->programs[i],
1477                                         obj->license,
1478                                         obj->kern_version);
1479                 if (err)
1480                         return err;
1481         }
1482         return 0;
1483 }
1484
1485 static bool bpf_prog_type__needs_kver(enum bpf_prog_type type)
1486 {
1487         switch (type) {
1488         case BPF_PROG_TYPE_SOCKET_FILTER:
1489         case BPF_PROG_TYPE_SCHED_CLS:
1490         case BPF_PROG_TYPE_SCHED_ACT:
1491         case BPF_PROG_TYPE_XDP:
1492         case BPF_PROG_TYPE_CGROUP_SKB:
1493         case BPF_PROG_TYPE_CGROUP_SOCK:
1494         case BPF_PROG_TYPE_LWT_IN:
1495         case BPF_PROG_TYPE_LWT_OUT:
1496         case BPF_PROG_TYPE_LWT_XMIT:
1497         case BPF_PROG_TYPE_LWT_SEG6LOCAL:
1498         case BPF_PROG_TYPE_SOCK_OPS:
1499         case BPF_PROG_TYPE_SK_SKB:
1500         case BPF_PROG_TYPE_CGROUP_DEVICE:
1501         case BPF_PROG_TYPE_SK_MSG:
1502         case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
1503         case BPF_PROG_TYPE_LIRC_MODE2:
1504         case BPF_PROG_TYPE_SK_REUSEPORT:
1505                 return false;
1506         case BPF_PROG_TYPE_UNSPEC:
1507         case BPF_PROG_TYPE_KPROBE:
1508         case BPF_PROG_TYPE_TRACEPOINT:
1509         case BPF_PROG_TYPE_PERF_EVENT:
1510         case BPF_PROG_TYPE_RAW_TRACEPOINT:
1511         default:
1512                 return true;
1513         }
1514 }
1515
1516 static int bpf_object__validate(struct bpf_object *obj, bool needs_kver)
1517 {
1518         if (needs_kver && obj->kern_version == 0) {
1519                 pr_warning("%s doesn't provide kernel version\n",
1520                            obj->path);
1521                 return -LIBBPF_ERRNO__KVERSION;
1522         }
1523         return 0;
1524 }
1525
1526 static struct bpf_object *
1527 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz,
1528                    bool needs_kver)
1529 {
1530         struct bpf_object *obj;
1531         int err;
1532
1533         if (elf_version(EV_CURRENT) == EV_NONE) {
1534                 pr_warning("failed to init libelf for %s\n", path);
1535                 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1536         }
1537
1538         obj = bpf_object__new(path, obj_buf, obj_buf_sz);
1539         if (IS_ERR(obj))
1540                 return obj;
1541
1542         CHECK_ERR(bpf_object__elf_init(obj), err, out);
1543         CHECK_ERR(bpf_object__check_endianness(obj), err, out);
1544         CHECK_ERR(bpf_object__elf_collect(obj), err, out);
1545         CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
1546         CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out);
1547
1548         bpf_object__elf_finish(obj);
1549         return obj;
1550 out:
1551         bpf_object__close(obj);
1552         return ERR_PTR(err);
1553 }
1554
1555 struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
1556 {
1557         /* param validation */
1558         if (!attr->file)
1559                 return NULL;
1560
1561         pr_debug("loading %s\n", attr->file);
1562
1563         return __bpf_object__open(attr->file, NULL, 0,
1564                                   bpf_prog_type__needs_kver(attr->prog_type));
1565 }
1566
1567 struct bpf_object *bpf_object__open(const char *path)
1568 {
1569         struct bpf_object_open_attr attr = {
1570                 .file           = path,
1571                 .prog_type      = BPF_PROG_TYPE_UNSPEC,
1572         };
1573
1574         return bpf_object__open_xattr(&attr);
1575 }
1576
1577 struct bpf_object *bpf_object__open_buffer(void *obj_buf,
1578                                            size_t obj_buf_sz,
1579                                            const char *name)
1580 {
1581         char tmp_name[64];
1582
1583         /* param validation */
1584         if (!obj_buf || obj_buf_sz <= 0)
1585                 return NULL;
1586
1587         if (!name) {
1588                 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
1589                          (unsigned long)obj_buf,
1590                          (unsigned long)obj_buf_sz);
1591                 tmp_name[sizeof(tmp_name) - 1] = '\0';
1592                 name = tmp_name;
1593         }
1594         pr_debug("loading object '%s' from buffer\n",
1595                  name);
1596
1597         return __bpf_object__open(name, obj_buf, obj_buf_sz, true);
1598 }
1599
1600 int bpf_object__unload(struct bpf_object *obj)
1601 {
1602         size_t i;
1603
1604         if (!obj)
1605                 return -EINVAL;
1606
1607         for (i = 0; i < obj->nr_maps; i++)
1608                 zclose(obj->maps[i].fd);
1609
1610         for (i = 0; i < obj->nr_programs; i++)
1611                 bpf_program__unload(&obj->programs[i]);
1612
1613         return 0;
1614 }
1615
1616 int bpf_object__load(struct bpf_object *obj)
1617 {
1618         int err;
1619
1620         if (!obj)
1621                 return -EINVAL;
1622
1623         if (obj->loaded) {
1624                 pr_warning("object should not be loaded twice\n");
1625                 return -EINVAL;
1626         }
1627
1628         obj->loaded = true;
1629
1630         CHECK_ERR(bpf_object__create_maps(obj), err, out);
1631         CHECK_ERR(bpf_object__relocate(obj), err, out);
1632         CHECK_ERR(bpf_object__load_progs(obj), err, out);
1633
1634         return 0;
1635 out:
1636         bpf_object__unload(obj);
1637         pr_warning("failed to load object '%s'\n", obj->path);
1638         return err;
1639 }
1640
1641 static int check_path(const char *path)
1642 {
1643         char *cp, errmsg[STRERR_BUFSIZE];
1644         struct statfs st_fs;
1645         char *dname, *dir;
1646         int err = 0;
1647
1648         if (path == NULL)
1649                 return -EINVAL;
1650
1651         dname = strdup(path);
1652         if (dname == NULL)
1653                 return -ENOMEM;
1654
1655         dir = dirname(dname);
1656         if (statfs(dir, &st_fs)) {
1657                 cp = strerror_r(errno, errmsg, sizeof(errmsg));
1658                 pr_warning("failed to statfs %s: %s\n", dir, cp);
1659                 err = -errno;
1660         }
1661         free(dname);
1662
1663         if (!err && st_fs.f_type != BPF_FS_MAGIC) {
1664                 pr_warning("specified path %s is not on BPF FS\n", path);
1665                 err = -EINVAL;
1666         }
1667
1668         return err;
1669 }
1670
1671 int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
1672                               int instance)
1673 {
1674         char *cp, errmsg[STRERR_BUFSIZE];
1675         int err;
1676
1677         err = check_path(path);
1678         if (err)
1679                 return err;
1680
1681         if (prog == NULL) {
1682                 pr_warning("invalid program pointer\n");
1683                 return -EINVAL;
1684         }
1685
1686         if (instance < 0 || instance >= prog->instances.nr) {
1687                 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
1688                            instance, prog->section_name, prog->instances.nr);
1689                 return -EINVAL;
1690         }
1691
1692         if (bpf_obj_pin(prog->instances.fds[instance], path)) {
1693                 cp = strerror_r(errno, errmsg, sizeof(errmsg));
1694                 pr_warning("failed to pin program: %s\n", cp);
1695                 return -errno;
1696         }
1697         pr_debug("pinned program '%s'\n", path);
1698
1699         return 0;
1700 }
1701
1702 static int make_dir(const char *path)
1703 {
1704         char *cp, errmsg[STRERR_BUFSIZE];
1705         int err = 0;
1706
1707         if (mkdir(path, 0700) && errno != EEXIST)
1708                 err = -errno;
1709
1710         if (err) {
1711                 cp = strerror_r(-err, errmsg, sizeof(errmsg));
1712                 pr_warning("failed to mkdir %s: %s\n", path, cp);
1713         }
1714         return err;
1715 }
1716
1717 int bpf_program__pin(struct bpf_program *prog, const char *path)
1718 {
1719         int i, err;
1720
1721         err = check_path(path);
1722         if (err)
1723                 return err;
1724
1725         if (prog == NULL) {
1726                 pr_warning("invalid program pointer\n");
1727                 return -EINVAL;
1728         }
1729
1730         if (prog->instances.nr <= 0) {
1731                 pr_warning("no instances of prog %s to pin\n",
1732                            prog->section_name);
1733                 return -EINVAL;
1734         }
1735
1736         err = make_dir(path);
1737         if (err)
1738                 return err;
1739
1740         for (i = 0; i < prog->instances.nr; i++) {
1741                 char buf[PATH_MAX];
1742                 int len;
1743
1744                 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
1745                 if (len < 0)
1746                         return -EINVAL;
1747                 else if (len >= PATH_MAX)
1748                         return -ENAMETOOLONG;
1749
1750                 err = bpf_program__pin_instance(prog, buf, i);
1751                 if (err)
1752                         return err;
1753         }
1754
1755         return 0;
1756 }
1757
1758 int bpf_map__pin(struct bpf_map *map, const char *path)
1759 {
1760         char *cp, errmsg[STRERR_BUFSIZE];
1761         int err;
1762
1763         err = check_path(path);
1764         if (err)
1765                 return err;
1766
1767         if (map == NULL) {
1768                 pr_warning("invalid map pointer\n");
1769                 return -EINVAL;
1770         }
1771
1772         if (bpf_obj_pin(map->fd, path)) {
1773                 cp = strerror_r(errno, errmsg, sizeof(errmsg));
1774                 pr_warning("failed to pin map: %s\n", cp);
1775                 return -errno;
1776         }
1777
1778         pr_debug("pinned map '%s'\n", path);
1779         return 0;
1780 }
1781
1782 int bpf_object__pin(struct bpf_object *obj, const char *path)
1783 {
1784         struct bpf_program *prog;
1785         struct bpf_map *map;
1786         int err;
1787
1788         if (!obj)
1789                 return -ENOENT;
1790
1791         if (!obj->loaded) {
1792                 pr_warning("object not yet loaded; load it first\n");
1793                 return -ENOENT;
1794         }
1795
1796         err = make_dir(path);
1797         if (err)
1798                 return err;
1799
1800         bpf_map__for_each(map, obj) {
1801                 char buf[PATH_MAX];
1802                 int len;
1803
1804                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1805                                bpf_map__name(map));
1806                 if (len < 0)
1807                         return -EINVAL;
1808                 else if (len >= PATH_MAX)
1809                         return -ENAMETOOLONG;
1810
1811                 err = bpf_map__pin(map, buf);
1812                 if (err)
1813                         return err;
1814         }
1815
1816         bpf_object__for_each_program(prog, obj) {
1817                 char buf[PATH_MAX];
1818                 int len;
1819
1820                 len = snprintf(buf, PATH_MAX, "%s/%s", path,
1821                                prog->section_name);
1822                 if (len < 0)
1823                         return -EINVAL;
1824                 else if (len >= PATH_MAX)
1825                         return -ENAMETOOLONG;
1826
1827                 err = bpf_program__pin(prog, buf);
1828                 if (err)
1829                         return err;
1830         }
1831
1832         return 0;
1833 }
1834
1835 void bpf_object__close(struct bpf_object *obj)
1836 {
1837         size_t i;
1838
1839         if (!obj)
1840                 return;
1841
1842         if (obj->clear_priv)
1843                 obj->clear_priv(obj, obj->priv);
1844
1845         bpf_object__elf_finish(obj);
1846         bpf_object__unload(obj);
1847         btf__free(obj->btf);
1848
1849         for (i = 0; i < obj->nr_maps; i++) {
1850                 zfree(&obj->maps[i].name);
1851                 if (obj->maps[i].clear_priv)
1852                         obj->maps[i].clear_priv(&obj->maps[i],
1853                                                 obj->maps[i].priv);
1854                 obj->maps[i].priv = NULL;
1855                 obj->maps[i].clear_priv = NULL;
1856         }
1857         zfree(&obj->maps);
1858         obj->nr_maps = 0;
1859
1860         if (obj->programs && obj->nr_programs) {
1861                 for (i = 0; i < obj->nr_programs; i++)
1862                         bpf_program__exit(&obj->programs[i]);
1863         }
1864         zfree(&obj->programs);
1865
1866         list_del(&obj->list);
1867         free(obj);
1868 }
1869
1870 struct bpf_object *
1871 bpf_object__next(struct bpf_object *prev)
1872 {
1873         struct bpf_object *next;
1874
1875         if (!prev)
1876                 next = list_first_entry(&bpf_objects_list,
1877                                         struct bpf_object,
1878                                         list);
1879         else
1880                 next = list_next_entry(prev, list);
1881
1882         /* Empty list is noticed here so don't need checking on entry. */
1883         if (&next->list == &bpf_objects_list)
1884                 return NULL;
1885
1886         return next;
1887 }
1888
1889 const char *bpf_object__name(struct bpf_object *obj)
1890 {
1891         return obj ? obj->path : ERR_PTR(-EINVAL);
1892 }
1893
1894 unsigned int bpf_object__kversion(struct bpf_object *obj)
1895 {
1896         return obj ? obj->kern_version : 0;
1897 }
1898
1899 int bpf_object__btf_fd(const struct bpf_object *obj)
1900 {
1901         return obj->btf ? btf__fd(obj->btf) : -1;
1902 }
1903
1904 int bpf_object__set_priv(struct bpf_object *obj, void *priv,
1905                          bpf_object_clear_priv_t clear_priv)
1906 {
1907         if (obj->priv && obj->clear_priv)
1908                 obj->clear_priv(obj, obj->priv);
1909
1910         obj->priv = priv;
1911         obj->clear_priv = clear_priv;
1912         return 0;
1913 }
1914
1915 void *bpf_object__priv(struct bpf_object *obj)
1916 {
1917         return obj ? obj->priv : ERR_PTR(-EINVAL);
1918 }
1919
1920 static struct bpf_program *
1921 __bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1922 {
1923         size_t idx;
1924
1925         if (!obj->programs)
1926                 return NULL;
1927         /* First handler */
1928         if (prev == NULL)
1929                 return &obj->programs[0];
1930
1931         if (prev->obj != obj) {
1932                 pr_warning("error: program handler doesn't match object\n");
1933                 return NULL;
1934         }
1935
1936         idx = (prev - obj->programs) + 1;
1937         if (idx >= obj->nr_programs)
1938                 return NULL;
1939         return &obj->programs[idx];
1940 }
1941
1942 struct bpf_program *
1943 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
1944 {
1945         struct bpf_program *prog = prev;
1946
1947         do {
1948                 prog = __bpf_program__next(prog, obj);
1949         } while (prog && bpf_program__is_function_storage(prog, obj));
1950
1951         return prog;
1952 }
1953
1954 int bpf_program__set_priv(struct bpf_program *prog, void *priv,
1955                           bpf_program_clear_priv_t clear_priv)
1956 {
1957         if (prog->priv && prog->clear_priv)
1958                 prog->clear_priv(prog, prog->priv);
1959
1960         prog->priv = priv;
1961         prog->clear_priv = clear_priv;
1962         return 0;
1963 }
1964
1965 void *bpf_program__priv(struct bpf_program *prog)
1966 {
1967         return prog ? prog->priv : ERR_PTR(-EINVAL);
1968 }
1969
1970 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
1971 {
1972         prog->prog_ifindex = ifindex;
1973 }
1974
1975 const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
1976 {
1977         const char *title;
1978
1979         title = prog->section_name;
1980         if (needs_copy) {
1981                 title = strdup(title);
1982                 if (!title) {
1983                         pr_warning("failed to strdup program title\n");
1984                         return ERR_PTR(-ENOMEM);
1985                 }
1986         }
1987
1988         return title;
1989 }
1990
1991 int bpf_program__fd(struct bpf_program *prog)
1992 {
1993         return bpf_program__nth_fd(prog, 0);
1994 }
1995
1996 int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
1997                           bpf_program_prep_t prep)
1998 {
1999         int *instances_fds;
2000
2001         if (nr_instances <= 0 || !prep)
2002                 return -EINVAL;
2003
2004         if (prog->instances.nr > 0 || prog->instances.fds) {
2005                 pr_warning("Can't set pre-processor after loading\n");
2006                 return -EINVAL;
2007         }
2008
2009         instances_fds = malloc(sizeof(int) * nr_instances);
2010         if (!instances_fds) {
2011                 pr_warning("alloc memory failed for fds\n");
2012                 return -ENOMEM;
2013         }
2014
2015         /* fill all fd with -1 */
2016         memset(instances_fds, -1, sizeof(int) * nr_instances);
2017
2018         prog->instances.nr = nr_instances;
2019         prog->instances.fds = instances_fds;
2020         prog->preprocessor = prep;
2021         return 0;
2022 }
2023
2024 int bpf_program__nth_fd(struct bpf_program *prog, int n)
2025 {
2026         int fd;
2027
2028         if (!prog)
2029                 return -EINVAL;
2030
2031         if (n >= prog->instances.nr || n < 0) {
2032                 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
2033                            n, prog->section_name, prog->instances.nr);
2034                 return -EINVAL;
2035         }
2036
2037         fd = prog->instances.fds[n];
2038         if (fd < 0) {
2039                 pr_warning("%dth instance of program '%s' is invalid\n",
2040                            n, prog->section_name);
2041                 return -ENOENT;
2042         }
2043
2044         return fd;
2045 }
2046
2047 void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
2048 {
2049         prog->type = type;
2050 }
2051
2052 static bool bpf_program__is_type(struct bpf_program *prog,
2053                                  enum bpf_prog_type type)
2054 {
2055         return prog ? (prog->type == type) : false;
2056 }
2057
2058 #define BPF_PROG_TYPE_FNS(NAME, TYPE)                   \
2059 int bpf_program__set_##NAME(struct bpf_program *prog)   \
2060 {                                                       \
2061         if (!prog)                                      \
2062                 return -EINVAL;                         \
2063         bpf_program__set_type(prog, TYPE);              \
2064         return 0;                                       \
2065 }                                                       \
2066                                                         \
2067 bool bpf_program__is_##NAME(struct bpf_program *prog)   \
2068 {                                                       \
2069         return bpf_program__is_type(prog, TYPE);        \
2070 }                                                       \
2071
2072 BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
2073 BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
2074 BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
2075 BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
2076 BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
2077 BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
2078 BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
2079 BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
2080
2081 void bpf_program__set_expected_attach_type(struct bpf_program *prog,
2082                                            enum bpf_attach_type type)
2083 {
2084         prog->expected_attach_type = type;
2085 }
2086
2087 #define BPF_PROG_SEC_FULL(string, ptype, atype) \
2088         { string, sizeof(string) - 1, ptype, atype }
2089
2090 #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_FULL(string, ptype, 0)
2091
2092 #define BPF_S_PROG_SEC(string, ptype) \
2093         BPF_PROG_SEC_FULL(string, BPF_PROG_TYPE_CGROUP_SOCK, ptype)
2094
2095 #define BPF_SA_PROG_SEC(string, ptype) \
2096         BPF_PROG_SEC_FULL(string, BPF_PROG_TYPE_CGROUP_SOCK_ADDR, ptype)
2097
2098 static const struct {
2099         const char *sec;
2100         size_t len;
2101         enum bpf_prog_type prog_type;
2102         enum bpf_attach_type expected_attach_type;
2103 } section_names[] = {
2104         BPF_PROG_SEC("socket",          BPF_PROG_TYPE_SOCKET_FILTER),
2105         BPF_PROG_SEC("kprobe/",         BPF_PROG_TYPE_KPROBE),
2106         BPF_PROG_SEC("kretprobe/",      BPF_PROG_TYPE_KPROBE),
2107         BPF_PROG_SEC("classifier",      BPF_PROG_TYPE_SCHED_CLS),
2108         BPF_PROG_SEC("action",          BPF_PROG_TYPE_SCHED_ACT),
2109         BPF_PROG_SEC("tracepoint/",     BPF_PROG_TYPE_TRACEPOINT),
2110         BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT),
2111         BPF_PROG_SEC("xdp",             BPF_PROG_TYPE_XDP),
2112         BPF_PROG_SEC("perf_event",      BPF_PROG_TYPE_PERF_EVENT),
2113         BPF_PROG_SEC("cgroup/skb",      BPF_PROG_TYPE_CGROUP_SKB),
2114         BPF_PROG_SEC("cgroup/sock",     BPF_PROG_TYPE_CGROUP_SOCK),
2115         BPF_PROG_SEC("cgroup/dev",      BPF_PROG_TYPE_CGROUP_DEVICE),
2116         BPF_PROG_SEC("lwt_in",          BPF_PROG_TYPE_LWT_IN),
2117         BPF_PROG_SEC("lwt_out",         BPF_PROG_TYPE_LWT_OUT),
2118         BPF_PROG_SEC("lwt_xmit",        BPF_PROG_TYPE_LWT_XMIT),
2119         BPF_PROG_SEC("lwt_seg6local",   BPF_PROG_TYPE_LWT_SEG6LOCAL),
2120         BPF_PROG_SEC("sockops",         BPF_PROG_TYPE_SOCK_OPS),
2121         BPF_PROG_SEC("sk_skb",          BPF_PROG_TYPE_SK_SKB),
2122         BPF_PROG_SEC("sk_msg",          BPF_PROG_TYPE_SK_MSG),
2123         BPF_PROG_SEC("lirc_mode2",      BPF_PROG_TYPE_LIRC_MODE2),
2124         BPF_SA_PROG_SEC("cgroup/bind4", BPF_CGROUP_INET4_BIND),
2125         BPF_SA_PROG_SEC("cgroup/bind6", BPF_CGROUP_INET6_BIND),
2126         BPF_SA_PROG_SEC("cgroup/connect4", BPF_CGROUP_INET4_CONNECT),
2127         BPF_SA_PROG_SEC("cgroup/connect6", BPF_CGROUP_INET6_CONNECT),
2128         BPF_SA_PROG_SEC("cgroup/sendmsg4", BPF_CGROUP_UDP4_SENDMSG),
2129         BPF_SA_PROG_SEC("cgroup/sendmsg6", BPF_CGROUP_UDP6_SENDMSG),
2130         BPF_S_PROG_SEC("cgroup/post_bind4", BPF_CGROUP_INET4_POST_BIND),
2131         BPF_S_PROG_SEC("cgroup/post_bind6", BPF_CGROUP_INET6_POST_BIND),
2132 };
2133
2134 #undef BPF_PROG_SEC
2135 #undef BPF_PROG_SEC_FULL
2136 #undef BPF_S_PROG_SEC
2137 #undef BPF_SA_PROG_SEC
2138
2139 int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
2140                              enum bpf_attach_type *expected_attach_type)
2141 {
2142         int i;
2143
2144         if (!name)
2145                 return -EINVAL;
2146
2147         for (i = 0; i < ARRAY_SIZE(section_names); i++) {
2148                 if (strncmp(name, section_names[i].sec, section_names[i].len))
2149                         continue;
2150                 *prog_type = section_names[i].prog_type;
2151                 *expected_attach_type = section_names[i].expected_attach_type;
2152                 return 0;
2153         }
2154         return -EINVAL;
2155 }
2156
2157 static int
2158 bpf_program__identify_section(struct bpf_program *prog,
2159                               enum bpf_prog_type *prog_type,
2160                               enum bpf_attach_type *expected_attach_type)
2161 {
2162         return libbpf_prog_type_by_name(prog->section_name, prog_type,
2163                                         expected_attach_type);
2164 }
2165
2166 int bpf_map__fd(struct bpf_map *map)
2167 {
2168         return map ? map->fd : -EINVAL;
2169 }
2170
2171 const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
2172 {
2173         return map ? &map->def : ERR_PTR(-EINVAL);
2174 }
2175
2176 const char *bpf_map__name(struct bpf_map *map)
2177 {
2178         return map ? map->name : NULL;
2179 }
2180
2181 __u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
2182 {
2183         return map ? map->btf_key_type_id : 0;
2184 }
2185
2186 __u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
2187 {
2188         return map ? map->btf_value_type_id : 0;
2189 }
2190
2191 int bpf_map__set_priv(struct bpf_map *map, void *priv,
2192                      bpf_map_clear_priv_t clear_priv)
2193 {
2194         if (!map)
2195                 return -EINVAL;
2196
2197         if (map->priv) {
2198                 if (map->clear_priv)
2199                         map->clear_priv(map, map->priv);
2200         }
2201
2202         map->priv = priv;
2203         map->clear_priv = clear_priv;
2204         return 0;
2205 }
2206
2207 void *bpf_map__priv(struct bpf_map *map)
2208 {
2209         return map ? map->priv : ERR_PTR(-EINVAL);
2210 }
2211
2212 bool bpf_map__is_offload_neutral(struct bpf_map *map)
2213 {
2214         return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
2215 }
2216
2217 void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
2218 {
2219         map->map_ifindex = ifindex;
2220 }
2221
2222 struct bpf_map *
2223 bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
2224 {
2225         size_t idx;
2226         struct bpf_map *s, *e;
2227
2228         if (!obj || !obj->maps)
2229                 return NULL;
2230
2231         s = obj->maps;
2232         e = obj->maps + obj->nr_maps;
2233
2234         if (prev == NULL)
2235                 return s;
2236
2237         if ((prev < s) || (prev >= e)) {
2238                 pr_warning("error in %s: map handler doesn't belong to object\n",
2239                            __func__);
2240                 return NULL;
2241         }
2242
2243         idx = (prev - obj->maps) + 1;
2244         if (idx >= obj->nr_maps)
2245                 return NULL;
2246         return &obj->maps[idx];
2247 }
2248
2249 struct bpf_map *
2250 bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
2251 {
2252         struct bpf_map *pos;
2253
2254         bpf_map__for_each(pos, obj) {
2255                 if (pos->name && !strcmp(pos->name, name))
2256                         return pos;
2257         }
2258         return NULL;
2259 }
2260
2261 struct bpf_map *
2262 bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
2263 {
2264         int i;
2265
2266         for (i = 0; i < obj->nr_maps; i++) {
2267                 if (obj->maps[i].offset == offset)
2268                         return &obj->maps[i];
2269         }
2270         return ERR_PTR(-ENOENT);
2271 }
2272
2273 long libbpf_get_error(const void *ptr)
2274 {
2275         if (IS_ERR(ptr))
2276                 return PTR_ERR(ptr);
2277         return 0;
2278 }
2279
2280 int bpf_prog_load(const char *file, enum bpf_prog_type type,
2281                   struct bpf_object **pobj, int *prog_fd)
2282 {
2283         struct bpf_prog_load_attr attr;
2284
2285         memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
2286         attr.file = file;
2287         attr.prog_type = type;
2288         attr.expected_attach_type = 0;
2289
2290         return bpf_prog_load_xattr(&attr, pobj, prog_fd);
2291 }
2292
2293 int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
2294                         struct bpf_object **pobj, int *prog_fd)
2295 {
2296         struct bpf_object_open_attr open_attr = {
2297                 .file           = attr->file,
2298                 .prog_type      = attr->prog_type,
2299         };
2300         struct bpf_program *prog, *first_prog = NULL;
2301         enum bpf_attach_type expected_attach_type;
2302         enum bpf_prog_type prog_type;
2303         struct bpf_object *obj;
2304         struct bpf_map *map;
2305         int err;
2306
2307         if (!attr)
2308                 return -EINVAL;
2309         if (!attr->file)
2310                 return -EINVAL;
2311
2312         obj = bpf_object__open_xattr(&open_attr);
2313         if (IS_ERR_OR_NULL(obj))
2314                 return -ENOENT;
2315
2316         bpf_object__for_each_program(prog, obj) {
2317                 /*
2318                  * If type is not specified, try to guess it based on
2319                  * section name.
2320                  */
2321                 prog_type = attr->prog_type;
2322                 prog->prog_ifindex = attr->ifindex;
2323                 expected_attach_type = attr->expected_attach_type;
2324                 if (prog_type == BPF_PROG_TYPE_UNSPEC) {
2325                         err = bpf_program__identify_section(prog, &prog_type,
2326                                                             &expected_attach_type);
2327                         if (err < 0) {
2328                                 pr_warning("failed to guess program type based on section name %s\n",
2329                                            prog->section_name);
2330                                 bpf_object__close(obj);
2331                                 return -EINVAL;
2332                         }
2333                 }
2334
2335                 bpf_program__set_type(prog, prog_type);
2336                 bpf_program__set_expected_attach_type(prog,
2337                                                       expected_attach_type);
2338
2339                 if (!bpf_program__is_function_storage(prog, obj) && !first_prog)
2340                         first_prog = prog;
2341         }
2342
2343         bpf_map__for_each(map, obj) {
2344                 if (!bpf_map__is_offload_neutral(map))
2345                         map->map_ifindex = attr->ifindex;
2346         }
2347
2348         if (!first_prog) {
2349                 pr_warning("object file doesn't contain bpf program\n");
2350                 bpf_object__close(obj);
2351                 return -ENOENT;
2352         }
2353
2354         err = bpf_object__load(obj);
2355         if (err) {
2356                 bpf_object__close(obj);
2357                 return -EINVAL;
2358         }
2359
2360         *pobj = obj;
2361         *prog_fd = bpf_program__fd(first_prog);
2362         return 0;
2363 }
2364
2365 enum bpf_perf_event_ret
2366 bpf_perf_event_read_simple(void *mem, unsigned long size,
2367                            unsigned long page_size, void **buf, size_t *buf_len,
2368                            bpf_perf_event_print_t fn, void *priv)
2369 {
2370         volatile struct perf_event_mmap_page *header = mem;
2371         __u64 data_tail = header->data_tail;
2372         __u64 data_head = header->data_head;
2373         int ret = LIBBPF_PERF_EVENT_ERROR;
2374         void *base, *begin, *end;
2375
2376         asm volatile("" ::: "memory"); /* in real code it should be smp_rmb() */
2377         if (data_head == data_tail)
2378                 return LIBBPF_PERF_EVENT_CONT;
2379
2380         base = ((char *)header) + page_size;
2381
2382         begin = base + data_tail % size;
2383         end = base + data_head % size;
2384
2385         while (begin != end) {
2386                 struct perf_event_header *ehdr;
2387
2388                 ehdr = begin;
2389                 if (begin + ehdr->size > base + size) {
2390                         long len = base + size - begin;
2391
2392                         if (*buf_len < ehdr->size) {
2393                                 free(*buf);
2394                                 *buf = malloc(ehdr->size);
2395                                 if (!*buf) {
2396                                         ret = LIBBPF_PERF_EVENT_ERROR;
2397                                         break;
2398                                 }
2399                                 *buf_len = ehdr->size;
2400                         }
2401
2402                         memcpy(*buf, begin, len);
2403                         memcpy(*buf + len, base, ehdr->size - len);
2404                         ehdr = (void *)*buf;
2405                         begin = base + ehdr->size - len;
2406                 } else if (begin + ehdr->size == base + size) {
2407                         begin = base;
2408                 } else {
2409                         begin += ehdr->size;
2410                 }
2411
2412                 ret = fn(ehdr, priv);
2413                 if (ret != LIBBPF_PERF_EVENT_CONT)
2414                         break;
2415
2416                 data_tail += ehdr->size;
2417         }
2418
2419         __sync_synchronize(); /* smp_mb() */
2420         header->data_tail = data_tail;
2421
2422         return ret;
2423 }