projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
libbpf: check map name retrieved from ELF
[linux-block.git] / tools / lib / bpf / libbpf.c
CommitLineData
1bc38b8f 1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6061a3d6 2
1b76c13e
WN		 3/*
4 * Common eBPF ELF object loading operations.
5 *
6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8 * Copyright (C) 2015 Huawei Inc.
f367540c 9 * Copyright (C) 2017 Nicira, Inc.
d859900c 10 * Copyright (C) 2019 Isovalent, Inc.
1b76c13e
WN		 11 */
12
b4269954 13#ifndef _GNU_SOURCE
531b014e 14#define _GNU_SOURCE
b4269954 15#endif
1b76c13e 16#include <stdlib.h>
b3f59d66
WN		 17#include <stdio.h>
18#include <stdarg.h>
f367540c 19#include <libgen.h>
34090915 20#include <inttypes.h>
b3f59d66 21#include <string.h>
1b76c13e 22#include <unistd.h>
1a5e3fb1
WN		 23#include <fcntl.h>
24#include <errno.h>
1b76c13e 25#include <asm/unistd.h>
e28ff1a8 26#include <linux/err.h>
cb1e5e96 27#include <linux/kernel.h>
1b76c13e 28#include <linux/bpf.h>
38d5d3b3 29#include <linux/btf.h>
47eff617 30#include <linux/filter.h>
9a208eff 31#include <linux/list.h>
f367540c 32#include <linux/limits.h>
438363c0 33#include <linux/perf_event.h>
a64af0ef 34#include <linux/ring_buffer.h>
f367540c
JS		 35#include <sys/stat.h>
36#include <sys/types.h>
37#include <sys/vfs.h>
531b014e 38#include <tools/libc_compat.h>
1a5e3fb1
WN		 39#include <libelf.h>
40#include <gelf.h>
1b76c13e
WN		 41
42#include "libbpf.h"
52d3352e 43#include "bpf.h"
8a138aed 44#include "btf.h"
6d41907c 45#include "str_error.h"
d7c4b398 46#include "libbpf_internal.h"
b3f59d66 47
9b16137a
WN		 48#ifndef EM_BPF
49#define EM_BPF 247
50#endif
51
f367540c
JS		 52#ifndef BPF_FS_MAGIC
53#define BPF_FS_MAGIC 0xcafe4a11
54#endif
55
ff466b58
AI		 56/* vsprintf() in __base_pr() uses nonliteral format string. It may break
57 * compilation if user enables corresponding warning. Disable it explicitly.
58 */
59#pragma GCC diagnostic ignored "-Wformat-nonliteral"
60
b3f59d66
WN		 61#define __printf(a, b) __attribute__((format(printf, a, b)))
62
a8a1f7d0
SF		 63static int __base_pr(enum libbpf_print_level level, const char *format,
64 va_list args)
b3f59d66 65{
6f1ae8b6
YS		 66 if (level == LIBBPF_DEBUG)
67 return 0;
68
a8a1f7d0 69 return vfprintf(stderr, format, args);
b3f59d66
WN		 70}
71
a8a1f7d0 72static libbpf_print_fn_t __libbpf_pr = __base_pr;
b3f59d66 73
6f1ae8b6 74void libbpf_set_print(libbpf_print_fn_t fn)
b3f59d66 75{
6f1ae8b6 76 __libbpf_pr = fn;
b3f59d66 77}
1a5e3fb1 78
8461ef8b
YS		 79__printf(2, 3)
80void libbpf_print(enum libbpf_print_level level, const char *format, ...)
81{
82 va_list args;
83
6f1ae8b6
YS		 84 if (!__libbpf_pr)
85 return;
86
8461ef8b 87 va_start(args, format);
6f1ae8b6 88 __libbpf_pr(level, format, args);
8461ef8b
YS		 89 va_end(args);
90}
91
6371ca3b
WN		 92#define STRERR_BUFSIZE 128
93
6371ca3b
WN		 94#define CHECK_ERR(action, err, out) do { \
95 err = action; \
96 if (err) \
97 goto out; \
98} while(0)
99
100
1a5e3fb1
WN		 101/* Copied from tools/perf/util/util.h */
102#ifndef zfree
103# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
104#endif
105
106#ifndef zclose
107# define zclose(fd) ({ \
108 int ___err = 0; \
109 if ((fd) >= 0) \
110 ___err = close((fd)); \
111 fd = -1; \
112 ___err; })
113#endif
114
115#ifdef HAVE_LIBELF_MMAP_SUPPORT
116# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
117#else
118# define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
119#endif
120
34be1646
SL		 121static inline __u64 ptr_to_u64(const void *ptr)
122{
123 return (__u64) (unsigned long) ptr;
124}
125
47eff617
SF		 126struct bpf_capabilities {
127 /* v4.14: kernel support for program & map names. */
128 __u32 name:1;
8837fe5d
DB		 129 /* v5.2: kernel support for global data sections. */
130 __u32 global_data:1;
d7c4b398
AN		 131 /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */
132 __u32 btf_func:1;
133 /* BTF_KIND_VAR and BTF_KIND_DATASEC support */
134 __u32 btf_datasec:1;
47eff617
SF		 135};
136
a5b8bd47
WN		 137/*
138 * bpf_prog should be a better name but it has been used in
139 * linux/filter.h.
140 */
141struct bpf_program {
142 /* Index in elf obj file, for relocation use. */
143 int idx;
88cda1c9 144 char *name;
f0307a7e 145 int prog_ifindex;
a5b8bd47 146 char *section_name;
33a2c75c
SF		 147 /* section_name with / replaced by _; makes recursive pinning
148 * in bpf_object__pin_programs easier
149 */
150 char *pin_name;
a5b8bd47 151 struct bpf_insn *insns;
48cca7e4 152 size_t insns_cnt, main_prog_cnt;
5f44e4c8 153 enum bpf_prog_type type;
34090915 154
48cca7e4
AS		 155 struct reloc_desc {
156 enum {
157 RELO_LD64,
158 RELO_CALL,
d859900c 159 RELO_DATA,
48cca7e4 160 } type;
34090915 161 int insn_idx;
48cca7e4
AS		 162 union {
163 int map_idx;
164 int text_off;
165 };
34090915
WN		 166 } *reloc_desc;
167 int nr_reloc;
da11b417 168 int log_level;
55cffde2 169
b580563e
WN		 170 struct {
171 int nr;
172 int *fds;
173 } instances;
174 bpf_program_prep_t preprocessor;
aa9b1ac3
WN		 175
176 struct bpf_object *obj;
177 void *priv;
178 bpf_program_clear_priv_t clear_priv;
d7be143b
AI		 179
180 enum bpf_attach_type expected_attach_type;
2993e051
YS		 181 int btf_fd;
182 void *func_info;
183 __u32 func_info_rec_size;
f0187f0b 184 __u32 func_info_cnt;
47eff617
SF		 185
186 struct bpf_capabilities *caps;
3d650141
MKL		 187
188 void *line_info;
189 __u32 line_info_rec_size;
190 __u32 line_info_cnt;
04656198 191 __u32 prog_flags;
a5b8bd47
WN		 192};
193
d859900c
DB		 194enum libbpf_map_type {
195 LIBBPF_MAP_UNSPEC,
196 LIBBPF_MAP_DATA,
197 LIBBPF_MAP_BSS,
198 LIBBPF_MAP_RODATA,
199};
200
201static const char * const libbpf_type_to_btf_name[] = {
202 [LIBBPF_MAP_DATA] = ".data",
203 [LIBBPF_MAP_BSS] = ".bss",
204 [LIBBPF_MAP_RODATA] = ".rodata",
205};
206
9d759a9b
WN		 207struct bpf_map {
208 int fd;
561bbcca 209 char *name;
4708bbda 210 size_t offset;
f0307a7e 211 int map_ifindex;
addb9fc9 212 int inner_map_fd;
9d759a9b 213 struct bpf_map_def def;
5b891af7
MKL		 214 __u32 btf_key_type_id;
215 __u32 btf_value_type_id;
9d759a9b
WN		 216 void *priv;
217 bpf_map_clear_priv_t clear_priv;
d859900c
DB		 218 enum libbpf_map_type libbpf_type;
219};
220
221struct bpf_secdata {
222 void *rodata;
223 void *data;
9d759a9b
WN		 224};
225
9a208eff
WN		 226static LIST_HEAD(bpf_objects_list);
227
1a5e3fb1 228struct bpf_object {
d859900c 229 char name[BPF_OBJ_NAME_LEN];
cb1e5e96 230 char license[64];
438363c0 231 __u32 kern_version;
0b3d1efa 232
a5b8bd47
WN		 233 struct bpf_program *programs;
234 size_t nr_programs;
9d759a9b
WN		 235 struct bpf_map *maps;
236 size_t nr_maps;
d859900c 237 struct bpf_secdata sections;
9d759a9b 238
52d3352e 239 bool loaded;
9a94f277 240 bool has_pseudo_calls;
a5b8bd47 241
1a5e3fb1
WN		 242 /*
243 * Information when doing elf related work. Only valid if fd
244 * is valid.
245 */
246 struct {
247 int fd;
6c956392
WN		 248 void *obj_buf;
249 size_t obj_buf_sz;
1a5e3fb1
WN		 250 Elf *elf;
251 GElf_Ehdr ehdr;
bec7d68c 252 Elf_Data *symbols;
d859900c
DB		 253 Elf_Data *data;
254 Elf_Data *rodata;
255 Elf_Data *bss;
77ba9a5b 256 size_t strtabidx;
b62f06e8
WN		 257 struct {
258 GElf_Shdr shdr;
259 Elf_Data *data;
260 } *reloc;
261 int nr_reloc;
666810e8 262 int maps_shndx;
48cca7e4 263 int text_shndx;
d859900c
DB		 264 int data_shndx;
265 int rodata_shndx;
266 int bss_shndx;
1a5e3fb1 267 } efile;
9a208eff
WN		 268 /*
269 * All loaded bpf_object is linked in a list, which is
270 * hidden to caller. bpf_objects__<func> handlers deal with
271 * all objects.
272 */
273 struct list_head list;
10931d24 274
8a138aed 275 struct btf *btf;
2993e051 276 struct btf_ext *btf_ext;
8a138aed 277
10931d24
WN		 278 void *priv;
279 bpf_object_clear_priv_t clear_priv;
280
47eff617
SF		 281 struct bpf_capabilities caps;
282
1a5e3fb1
WN		 283 char path[];
284};
285#define obj_elf_valid(o) ((o)->efile.elf)
286
29cd77f4 287void bpf_program__unload(struct bpf_program *prog)
55cffde2 288{
b580563e
WN		 289 int i;
290
55cffde2
WN		 291 if (!prog)
292 return;
293
b580563e
WN		 294 /*
295 * If the object is opened but the program was never loaded,
296 * it is possible that prog->instances.nr == -1.
297 */
298 if (prog->instances.nr > 0) {
299 for (i = 0; i < prog->instances.nr; i++)
300 zclose(prog->instances.fds[i]);
301 } else if (prog->instances.nr != -1) {
302 pr_warning("Internal error: instances.nr is %d\n",
303 prog->instances.nr);
304 }
305
306 prog->instances.nr = -1;
307 zfree(&prog->instances.fds);
2993e051
YS		 308
309 zclose(prog->btf_fd);
310 zfree(&prog->func_info);
07a09d1b 311 zfree(&prog->line_info);
55cffde2
WN		 312}
313
a5b8bd47
WN		 314static void bpf_program__exit(struct bpf_program *prog)
315{
316 if (!prog)
317 return;
318
aa9b1ac3
WN		 319 if (prog->clear_priv)
320 prog->clear_priv(prog, prog->priv);
321
322 prog->priv = NULL;
323 prog->clear_priv = NULL;
324
55cffde2 325 bpf_program__unload(prog);
88cda1c9 326 zfree(&prog->name);
a5b8bd47 327 zfree(&prog->section_name);
33a2c75c 328 zfree(&prog->pin_name);
a5b8bd47 329 zfree(&prog->insns);
34090915
WN		 330 zfree(&prog->reloc_desc);
331
332 prog->nr_reloc = 0;
a5b8bd47
WN		 333 prog->insns_cnt = 0;
334 prog->idx = -1;
335}
336
33a2c75c
SF		 337static char *__bpf_program__pin_name(struct bpf_program *prog)
338{
339 char *name, *p;
340
341 name = p = strdup(prog->section_name);
342 while ((p = strchr(p, '/')))
343 *p = '_';
344
345 return name;
346}
347
a5b8bd47 348static int
88cda1c9
MKL		 349bpf_program__init(void *data, size_t size, char *section_name, int idx,
350 struct bpf_program *prog)
a5b8bd47 351{
8ca990ce
AN		 352 const size_t bpf_insn_sz = sizeof(struct bpf_insn);
353
354 if (size == 0 || size % bpf_insn_sz) {
355 pr_warning("corrupted section '%s', size: %zu\n",
356 section_name, size);
a5b8bd47
WN		 357 return -EINVAL;
358 }
359
1ad9cbb8 360 memset(prog, 0, sizeof(*prog));
a5b8bd47 361
88cda1c9 362 prog->section_name = strdup(section_name);
a5b8bd47 363 if (!prog->section_name) {
077c066a
JDB		 364 pr_warning("failed to alloc name for prog under section(%d) %s\n",
365 idx, section_name);
a5b8bd47
WN		 366 goto errout;
367 }
368
33a2c75c
SF		 369 prog->pin_name = __bpf_program__pin_name(prog);
370 if (!prog->pin_name) {
371 pr_warning("failed to alloc pin name for prog under section(%d) %s\n",
372 idx, section_name);
373 goto errout;
374 }
375
a5b8bd47
WN		 376 prog->insns = malloc(size);
377 if (!prog->insns) {
88cda1c9
MKL		 378 pr_warning("failed to alloc insns for prog under section %s\n",
379 section_name);
a5b8bd47
WN		 380 goto errout;
381 }
8ca990ce
AN		 382 prog->insns_cnt = size / bpf_insn_sz;
383 memcpy(prog->insns, data, size);
a5b8bd47 384 prog->idx = idx;
b580563e
WN		 385 prog->instances.fds = NULL;
386 prog->instances.nr = -1;
47ae7e3d 387 prog->type = BPF_PROG_TYPE_UNSPEC;
2993e051 388 prog->btf_fd = -1;
a5b8bd47
WN		 389
390 return 0;
391errout:
392 bpf_program__exit(prog);
393 return -ENOMEM;
394}
395
396static int
397bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
88cda1c9 398 char *section_name, int idx)
a5b8bd47
WN		 399{
400 struct bpf_program prog, *progs;
401 int nr_progs, err;
402
88cda1c9 403 err = bpf_program__init(data, size, section_name, idx, &prog);
a5b8bd47
WN		 404 if (err)
405 return err;
406
47eff617 407 prog.caps = &obj->caps;
a5b8bd47
WN		 408 progs = obj->programs;
409 nr_progs = obj->nr_programs;
410
531b014e 411 progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0]));
a5b8bd47
WN		 412 if (!progs) {
413 /*
414 * In this case the original obj->programs
415 * is still valid, so don't need special treat for
416 * bpf_close_object().
417 */
88cda1c9
MKL		 418 pr_warning("failed to alloc a new program under section '%s'\n",
419 section_name);
a5b8bd47
WN		 420 bpf_program__exit(&prog);
421 return -ENOMEM;
422 }
423
424 pr_debug("found program %s\n", prog.section_name);
425 obj->programs = progs;
426 obj->nr_programs = nr_progs + 1;
aa9b1ac3 427 prog.obj = obj;
a5b8bd47
WN		 428 progs[nr_progs] = prog;
429 return 0;
430}
431
88cda1c9
MKL		 432static int
433bpf_object__init_prog_names(struct bpf_object *obj)
434{
435 Elf_Data *symbols = obj->efile.symbols;
436 struct bpf_program *prog;
437 size_t pi, si;
438
439 for (pi = 0; pi < obj->nr_programs; pi++) {
48cca7e4 440 const char *name = NULL;
88cda1c9
MKL		 441
442 prog = &obj->programs[pi];
443
444 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name;
445 si++) {
446 GElf_Sym sym;
447
448 if (!gelf_getsym(symbols, si, &sym))
449 continue;
450 if (sym.st_shndx != prog->idx)
451 continue;
fe4d44b2
RG		 452 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL)
453 continue;
88cda1c9
MKL		 454
455 name = elf_strptr(obj->efile.elf,
456 obj->efile.strtabidx,
457 sym.st_name);
458 if (!name) {
459 pr_warning("failed to get sym name string for prog %s\n",
460 prog->section_name);
461 return -LIBBPF_ERRNO__LIBELF;
462 }
463 }
464
9a94f277
JK		 465 if (!name && prog->idx == obj->efile.text_shndx)
466 name = ".text";
467
88cda1c9
MKL		 468 if (!name) {
469 pr_warning("failed to find sym for prog %s\n",
470 prog->section_name);
471 return -EINVAL;
472 }
9a94f277 473
88cda1c9
MKL		 474 prog->name = strdup(name);
475 if (!prog->name) {
476 pr_warning("failed to allocate memory for prog sym %s\n",
477 name);
478 return -ENOMEM;
479 }
480 }
481
482 return 0;
483}
484
6c956392
WN		 485static struct bpf_object *bpf_object__new(const char *path,
486 void *obj_buf,
487 size_t obj_buf_sz)
1a5e3fb1
WN		 488{
489 struct bpf_object *obj;
d859900c 490 char *end;
1a5e3fb1
WN		 491
492 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
493 if (!obj) {
494 pr_warning("alloc memory failed for %s\n", path);
6371ca3b 495 return ERR_PTR(-ENOMEM);
1a5e3fb1
WN		 496 }
497
498 strcpy(obj->path, path);
d859900c
DB		 499 /* Using basename() GNU version which doesn't modify arg. */
500 strncpy(obj->name, basename((void *)path),
501 sizeof(obj->name) - 1);
502 end = strchr(obj->name, '.');
503 if (end)
504 *end = 0;
6c956392 505
d859900c 506 obj->efile.fd = -1;
6c956392
WN		 507 /*
508 * Caller of this function should also calls
509 * bpf_object__elf_finish() after data collection to return
510 * obj_buf to user. If not, we should duplicate the buffer to
511 * avoid user freeing them before elf finish.
512 */
513 obj->efile.obj_buf = obj_buf;
514 obj->efile.obj_buf_sz = obj_buf_sz;
666810e8 515 obj->efile.maps_shndx = -1;
d859900c
DB		 516 obj->efile.data_shndx = -1;
517 obj->efile.rodata_shndx = -1;
518 obj->efile.bss_shndx = -1;
6c956392 519
52d3352e 520 obj->loaded = false;
9a208eff
WN		 521
522 INIT_LIST_HEAD(&obj->list);
523 list_add(&obj->list, &bpf_objects_list);
1a5e3fb1
WN		 524 return obj;
525}
526
527static void bpf_object__elf_finish(struct bpf_object *obj)
528{
529 if (!obj_elf_valid(obj))
530 return;
531
532 if (obj->efile.elf) {
533 elf_end(obj->efile.elf);
534 obj->efile.elf = NULL;
535 }
bec7d68c 536 obj->efile.symbols = NULL;
d859900c
DB		 537 obj->efile.data = NULL;
538 obj->efile.rodata = NULL;
539 obj->efile.bss = NULL;
b62f06e8
WN		 540
541 zfree(&obj->efile.reloc);
542 obj->efile.nr_reloc = 0;
1a5e3fb1 543 zclose(obj->efile.fd);
6c956392
WN		 544 obj->efile.obj_buf = NULL;
545 obj->efile.obj_buf_sz = 0;
1a5e3fb1
WN		 546}
547
548static int bpf_object__elf_init(struct bpf_object *obj)
549{
550 int err = 0;
551 GElf_Ehdr *ep;
552
553 if (obj_elf_valid(obj)) {
554 pr_warning("elf init: internal error\n");
6371ca3b 555 return -LIBBPF_ERRNO__LIBELF;
1a5e3fb1
WN		 556 }
557
6c956392
WN		 558 if (obj->efile.obj_buf_sz > 0) {
559 /*
560 * obj_buf should have been validated by
561 * bpf_object__open_buffer().
562 */
563 obj->efile.elf = elf_memory(obj->efile.obj_buf,
564 obj->efile.obj_buf_sz);
565 } else {
566 obj->efile.fd = open(obj->path, O_RDONLY);
567 if (obj->efile.fd < 0) {
be5c5d4e 568 char errmsg[STRERR_BUFSIZE], *cp;
1ce6a9fc 569
be5c5d4e
AN		 570 err = -errno;
571 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1ce6a9fc 572 pr_warning("failed to open %s: %s\n", obj->path, cp);
be5c5d4e 573 return err;
6c956392
WN		 574 }
575
576 obj->efile.elf = elf_begin(obj->efile.fd,
577 LIBBPF_ELF_C_READ_MMAP,
578 NULL);
1a5e3fb1
WN		 579 }
580
1a5e3fb1
WN		 581 if (!obj->efile.elf) {
582 pr_warning("failed to open %s as ELF file\n",
583 obj->path);
6371ca3b 584 err = -LIBBPF_ERRNO__LIBELF;
1a5e3fb1
WN		 585 goto errout;
586 }
587
588 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
589 pr_warning("failed to get EHDR from %s\n",
590 obj->path);
6371ca3b 591 err = -LIBBPF_ERRNO__FORMAT;
1a5e3fb1
WN		 592 goto errout;
593 }
594 ep = &obj->efile.ehdr;
595
9b16137a
WN		 596 /* Old LLVM set e_machine to EM_NONE */
597 if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) {
1a5e3fb1
WN		 598 pr_warning("%s is not an eBPF object file\n",
599 obj->path);
6371ca3b 600 err = -LIBBPF_ERRNO__FORMAT;
1a5e3fb1
WN		 601 goto errout;
602 }
603
604 return 0;
605errout:
606 bpf_object__elf_finish(obj);
607 return err;
608}
609
12ef5634 610static int bpf_object__check_endianness(struct bpf_object *obj)
cc4228d5 611{
12ef5634
AN		 612#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
613 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB)
614 return 0;
615#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
616 if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB)
617 return 0;
618#else
619# error "Unrecognized __BYTE_ORDER__"
620#endif
621 pr_warning("endianness mismatch.\n");
6371ca3b 622 return -LIBBPF_ERRNO__ENDIAN;
cc4228d5
WN		 623}
624
cb1e5e96
WN		 625static int
626bpf_object__init_license(struct bpf_object *obj,
627 void *data, size_t size)
628{
629 memcpy(obj->license, data,
630 min(size, sizeof(obj->license) - 1));
631 pr_debug("license of %s is %s\n", obj->path, obj->license);
632 return 0;
633}
634
635static int
636bpf_object__init_kversion(struct bpf_object *obj,
637 void *data, size_t size)
638{
438363c0 639 __u32 kver;
cb1e5e96
WN		 640
641 if (size != sizeof(kver)) {
642 pr_warning("invalid kver section in %s\n", obj->path);
6371ca3b 643 return -LIBBPF_ERRNO__FORMAT;
cb1e5e96
WN		 644 }
645 memcpy(&kver, data, sizeof(kver));
646 obj->kern_version = kver;
647 pr_debug("kernel version of %s is %x\n", obj->path,
648 obj->kern_version);
649 return 0;
650}
651
4708bbda
EL		 652static int compare_bpf_map(const void *_a, const void *_b)
653{
654 const struct bpf_map *a = _a;
655 const struct bpf_map *b = _b;
9d759a9b 656
4708bbda 657 return a->offset - b->offset;
0b3d1efa
WN		 658}
659
addb9fc9
NS		 660static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
661{
662 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
663 type == BPF_MAP_TYPE_HASH_OF_MAPS)
664 return true;
665 return false;
666}
667
1713d68b
DB		 668static int bpf_object_search_section_size(const struct bpf_object *obj,
669 const char *name, size_t *d_size)
670{
671 const GElf_Ehdr *ep = &obj->efile.ehdr;
672 Elf *elf = obj->efile.elf;
673 Elf_Scn *scn = NULL;
674 int idx = 0;
675
676 while ((scn = elf_nextscn(elf, scn)) != NULL) {
677 const char *sec_name;
678 Elf_Data *data;
679 GElf_Shdr sh;
680
681 idx++;
682 if (gelf_getshdr(scn, &sh) != &sh) {
683 pr_warning("failed to get section(%d) header from %s\n",
684 idx, obj->path);
685 return -EIO;
686 }
687
688 sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
689 if (!sec_name) {
690 pr_warning("failed to get section(%d) name from %s\n",
691 idx, obj->path);
692 return -EIO;
693 }
694
695 if (strcmp(name, sec_name))
696 continue;
697
698 data = elf_getdata(scn, 0);
699 if (!data) {
700 pr_warning("failed to get section(%d) data from %s(%s)\n",
701 idx, name, obj->path);
702 return -EIO;
703 }
704
705 *d_size = data->d_size;
706 return 0;
707 }
708
709 return -ENOENT;
710}
711
712int bpf_object__section_size(const struct bpf_object *obj, const char *name,
713 __u32 *size)
714{
715 int ret = -ENOENT;
716 size_t d_size;
717
718 *size = 0;
719 if (!name) {
720 return -EINVAL;
721 } else if (!strcmp(name, ".data")) {
722 if (obj->efile.data)
723 *size = obj->efile.data->d_size;
724 } else if (!strcmp(name, ".bss")) {
725 if (obj->efile.bss)
726 *size = obj->efile.bss->d_size;
727 } else if (!strcmp(name, ".rodata")) {
728 if (obj->efile.rodata)
729 *size = obj->efile.rodata->d_size;
730 } else {
731 ret = bpf_object_search_section_size(obj, name, &d_size);
732 if (!ret)
733 *size = d_size;
734 }
735
736 return *size ? 0 : ret;
737}
738
739int bpf_object__variable_offset(const struct bpf_object *obj, const char *name,
740 __u32 *off)
741{
742 Elf_Data *symbols = obj->efile.symbols;
743 const char *sname;
744 size_t si;
745
746 if (!name || !off)
747 return -EINVAL;
748
749 for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) {
750 GElf_Sym sym;
751
752 if (!gelf_getsym(symbols, si, &sym))
753 continue;
754 if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL ||
755 GELF_ST_TYPE(sym.st_info) != STT_OBJECT)
756 continue;
757
758 sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
759 sym.st_name);
760 if (!sname) {
761 pr_warning("failed to get sym name string for var %s\n",
762 name);
763 return -EIO;
764 }
765 if (strcmp(name, sname) == 0) {
766 *off = sym.st_value;
767 return 0;
768 }
769 }
770
771 return -ENOENT;
772}
773
d859900c
DB		 774static bool bpf_object__has_maps(const struct bpf_object *obj)
775{
776 return obj->efile.maps_shndx >= 0 ||
777 obj->efile.data_shndx >= 0 ||
778 obj->efile.rodata_shndx >= 0 ||
779 obj->efile.bss_shndx >= 0;
780}
781
782static int
783bpf_object__init_internal_map(struct bpf_object *obj, struct bpf_map *map,
784 enum libbpf_map_type type, Elf_Data *data,
785 void **data_buff)
786{
787 struct bpf_map_def *def = &map->def;
788 char map_name[BPF_OBJ_NAME_LEN];
789
790 map->libbpf_type = type;
791 map->offset = ~(typeof(map->offset))0;
792 snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name,
793 libbpf_type_to_btf_name[type]);
794 map->name = strdup(map_name);
795 if (!map->name) {
796 pr_warning("failed to alloc map name\n");
797 return -ENOMEM;
798 }
799
800 def->type = BPF_MAP_TYPE_ARRAY;
801 def->key_size = sizeof(int);
802 def->value_size = data->d_size;
803 def->max_entries = 1;
804 def->map_flags = type == LIBBPF_MAP_RODATA ?
805 BPF_F_RDONLY_PROG : 0;
806 if (data_buff) {
807 *data_buff = malloc(data->d_size);
808 if (!*data_buff) {
809 zfree(&map->name);
810 pr_warning("failed to alloc map content buffer\n");
811 return -ENOMEM;
812 }
813 memcpy(*data_buff, data->d_buf, data->d_size);
814 }
815
e1d1dc46 816 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
d859900c
DB		 817 return 0;
818}
819
973170e6 820static int
c034a177 821bpf_object__init_maps(struct bpf_object *obj, int flags)
561bbcca 822{
d5adbdd7 823 int i, map_idx, map_def_sz = 0, nr_syms, nr_maps = 0, nr_maps_glob = 0;
c034a177 824 bool strict = !(flags & MAPS_RELAX_COMPAT);
561bbcca 825 Elf_Data *symbols = obj->efile.symbols;
d859900c
DB		 826 Elf_Data *data = NULL;
827 int ret = 0;
561bbcca 828
4708bbda
EL		 829 if (!symbols)
830 return -EINVAL;
d859900c 831 nr_syms = symbols->d_size / sizeof(GElf_Sym);
4708bbda 832
d859900c
DB		 833 if (obj->efile.maps_shndx >= 0) {
834 Elf_Scn *scn = elf_getscn(obj->efile.elf,
835 obj->efile.maps_shndx);
836
837 if (scn)
838 data = elf_getdata(scn, NULL);
839 if (!scn || !data) {
840 pr_warning("failed to get Elf_Data from map section %d\n",
841 obj->efile.maps_shndx);
842 return -EINVAL;
843 }
4708bbda 844 }
561bbcca 845
4708bbda
EL		 846 /*
847 * Count number of maps. Each map has a name.
848 * Array of maps is not supported: only the first element is
849 * considered.
850 *
851 * TODO: Detect array of map and report error.
852 */
8837fe5d
DB		 853 if (obj->caps.global_data) {
854 if (obj->efile.data_shndx >= 0)
855 nr_maps_glob++;
856 if (obj->efile.rodata_shndx >= 0)
857 nr_maps_glob++;
858 if (obj->efile.bss_shndx >= 0)
859 nr_maps_glob++;
860 }
861
d859900c 862 for (i = 0; data && i < nr_syms; i++) {
561bbcca 863 GElf_Sym sym;
4708bbda
EL		 864
865 if (!gelf_getsym(symbols, i, &sym))
866 continue;
867 if (sym.st_shndx != obj->efile.maps_shndx)
868 continue;
869 nr_maps++;
870 }
871
d859900c 872 if (!nr_maps && !nr_maps_glob)
4708bbda
EL		 873 return 0;
874
b13c5c14 875 /* Assume equally sized map definitions */
d859900c 876 if (data) {
4f8827d2
DB		 877 pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path,
878 nr_maps, data->d_size);
879
d859900c
DB		 880 map_def_sz = data->d_size / nr_maps;
881 if (!data->d_size || (data->d_size % nr_maps) != 0) {
882 pr_warning("unable to determine map definition size "
883 "section %s, %d maps in %zd bytes\n",
884 obj->path, nr_maps, data->d_size);
885 return -EINVAL;
886 }
b13c5c14
CG		 887 }
888
d859900c 889 nr_maps += nr_maps_glob;
4708bbda
EL		 890 obj->maps = calloc(nr_maps, sizeof(obj->maps[0]));
891 if (!obj->maps) {
892 pr_warning("alloc maps for object failed\n");
893 return -ENOMEM;
894 }
895 obj->nr_maps = nr_maps;
896
addb9fc9
NS		 897 for (i = 0; i < nr_maps; i++) {
898 /*
899 * fill all fd with -1 so won't close incorrect
900 * fd (fd=0 is stdin) when failure (zclose won't close
901 * negative fd)).
902 */
4708bbda 903 obj->maps[i].fd = -1;
addb9fc9
NS		 904 obj->maps[i].inner_map_fd = -1;
905 }
4708bbda
EL		 906
907 /*
908 * Fill obj->maps using data in "maps" section.
909 */
d859900c 910 for (i = 0, map_idx = 0; data && i < nr_syms; i++) {
4708bbda 911 GElf_Sym sym;
561bbcca 912 const char *map_name;
4708bbda 913 struct bpf_map_def *def;
561bbcca
WN		 914
915 if (!gelf_getsym(symbols, i, &sym))
916 continue;
666810e8 917 if (sym.st_shndx != obj->efile.maps_shndx)
561bbcca
WN		 918 continue;
919
920 map_name = elf_strptr(obj->efile.elf,
77ba9a5b 921 obj->efile.strtabidx,
561bbcca 922 sym.st_name);
c51829bb
AN		 923 if (!map_name) {
924 pr_warning("failed to get map #%d name sym string for obj %s\n",
925 map_idx, obj->path);
926 return -LIBBPF_ERRNO__FORMAT;
927 }
d859900c
DB		 928
929 obj->maps[map_idx].libbpf_type = LIBBPF_MAP_UNSPEC;
4708bbda 930 obj->maps[map_idx].offset = sym.st_value;
b13c5c14 931 if (sym.st_value + map_def_sz > data->d_size) {
4708bbda
EL		 932 pr_warning("corrupted maps section in %s: last map \"%s\" too small\n",
933 obj->path, map_name);
934 return -EINVAL;
561bbcca 935 }
4708bbda 936
561bbcca 937 obj->maps[map_idx].name = strdup(map_name);
973170e6
WN		 938 if (!obj->maps[map_idx].name) {
939 pr_warning("failed to alloc map name\n");
940 return -ENOMEM;
941 }
4708bbda 942 pr_debug("map %d is \"%s\"\n", map_idx,
561bbcca 943 obj->maps[map_idx].name);
4708bbda 944 def = (struct bpf_map_def *)(data->d_buf + sym.st_value);
b13c5c14
CG		 945 /*
946 * If the definition of the map in the object file fits in
947 * bpf_map_def, copy it. Any extra fields in our version
948 * of bpf_map_def will default to zero as a result of the
949 * calloc above.
950 */
951 if (map_def_sz <= sizeof(struct bpf_map_def)) {
952 memcpy(&obj->maps[map_idx].def, def, map_def_sz);
953 } else {
954 /*
955 * Here the map structure being read is bigger than what
956 * we expect, truncate if the excess bits are all zero.
957 * If they are not zero, reject this map as
958 * incompatible.
959 */
960 char *b;
961 for (b = ((char *)def) + sizeof(struct bpf_map_def);
962 b < ((char *)def) + map_def_sz; b++) {
963 if (*b != 0) {
964 pr_warning("maps section in %s: \"%s\" "
965 "has unrecognized, non-zero "
966 "options\n",
967 obj->path, map_name);
c034a177
JF		 968 if (strict)
969 return -EINVAL;
b13c5c14
CG		 970 }
971 }
972 memcpy(&obj->maps[map_idx].def, def,
973 sizeof(struct bpf_map_def));
974 }
4708bbda 975 map_idx++;
561bbcca 976 }
4708bbda 977
8837fe5d
DB		 978 if (!obj->caps.global_data)
979 goto finalize;
980
d859900c
DB		 981 /*
982 * Populate rest of obj->maps with libbpf internal maps.
983 */
984 if (obj->efile.data_shndx >= 0)
985 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++],
986 LIBBPF_MAP_DATA,
987 obj->efile.data,
988 &obj->sections.data);
989 if (!ret && obj->efile.rodata_shndx >= 0)
990 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++],
991 LIBBPF_MAP_RODATA,
992 obj->efile.rodata,
993 &obj->sections.rodata);
994 if (!ret && obj->efile.bss_shndx >= 0)
995 ret = bpf_object__init_internal_map(obj, &obj->maps[map_idx++],
996 LIBBPF_MAP_BSS,
997 obj->efile.bss, NULL);
8837fe5d 998finalize:
d859900c
DB		 999 if (!ret)
1000 qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]),
1001 compare_bpf_map);
1002 return ret;
561bbcca
WN		 1003}
1004
e3d91b0c
JDB		 1005static bool section_have_execinstr(struct bpf_object *obj, int idx)
1006{
1007 Elf_Scn *scn;
1008 GElf_Shdr sh;
1009
1010 scn = elf_getscn(obj->efile.elf, idx);
1011 if (!scn)
1012 return false;
1013
1014 if (gelf_getshdr(scn, &sh) != &sh)
1015 return false;
1016
1017 if (sh.sh_flags & SHF_EXECINSTR)
1018 return true;
1019
1020 return false;
1021}
1022
d7c4b398
AN		 1023static void bpf_object__sanitize_btf(struct bpf_object *obj)
1024{
1025 bool has_datasec = obj->caps.btf_datasec;
1026 bool has_func = obj->caps.btf_func;
1027 struct btf *btf = obj->btf;
1028 struct btf_type *t;
1029 int i, j, vlen;
1030 __u16 kind;
1031
1032 if (!obj->btf || (has_func && has_datasec))
1033 return;
1034
1035 for (i = 1; i <= btf__get_nr_types(btf); i++) {
1036 t = (struct btf_type *)btf__type_by_id(btf, i);
1037 kind = BTF_INFO_KIND(t->info);
1038
1039 if (!has_datasec && kind == BTF_KIND_VAR) {
1040 /* replace VAR with INT */
1041 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
1042 t->size = sizeof(int);
1043 *(int *)(t+1) = BTF_INT_ENC(0, 0, 32);
1044 } else if (!has_datasec && kind == BTF_KIND_DATASEC) {
1045 /* replace DATASEC with STRUCT */
1046 struct btf_var_secinfo *v = (void *)(t + 1);
1047 struct btf_member *m = (void *)(t + 1);
1048 struct btf_type *vt;
1049 char *name;
1050
1051 name = (char *)btf__name_by_offset(btf, t->name_off);
1052 while (*name) {
1053 if (*name == '.')
1054 *name = '_';
1055 name++;
1056 }
1057
1058 vlen = BTF_INFO_VLEN(t->info);
1059 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
1060 for (j = 0; j < vlen; j++, v++, m++) {
1061 /* order of field assignments is important */
1062 m->offset = v->offset * 8;
1063 m->type = v->type;
1064 /* preserve variable name as member name */
1065 vt = (void *)btf__type_by_id(btf, v->type);
1066 m->name_off = vt->name_off;
1067 }
1068 } else if (!has_func && kind == BTF_KIND_FUNC_PROTO) {
1069 /* replace FUNC_PROTO with ENUM */
1070 vlen = BTF_INFO_VLEN(t->info);
1071 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
1072 t->size = sizeof(__u32); /* kernel enforced */
1073 } else if (!has_func && kind == BTF_KIND_FUNC) {
1074 /* replace FUNC with TYPEDEF */
1075 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
1076 }
1077 }
1078}
1079
1080static void bpf_object__sanitize_btf_ext(struct bpf_object *obj)
1081{
1082 if (!obj->btf_ext)
1083 return;
1084
1085 if (!obj->caps.btf_func) {
1086 btf_ext__free(obj->btf_ext);
1087 obj->btf_ext = NULL;
1088 }
1089}
1090
c034a177 1091static int bpf_object__elf_collect(struct bpf_object *obj, int flags)
29603665
WN		 1092{
1093 Elf *elf = obj->efile.elf;
1094 GElf_Ehdr *ep = &obj->efile.ehdr;
f0187f0b 1095 Elf_Data *btf_ext_data = NULL;
1713d68b 1096 Elf_Data *btf_data = NULL;
29603665 1097 Elf_Scn *scn = NULL;
666810e8 1098 int idx = 0, err = 0;
29603665
WN		 1099
1100 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
1101 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
1102 pr_warning("failed to get e_shstrndx from %s\n",
1103 obj->path);
6371ca3b 1104 return -LIBBPF_ERRNO__FORMAT;
29603665
WN		 1105 }
1106
1107 while ((scn = elf_nextscn(elf, scn)) != NULL) {
1108 char *name;
1109 GElf_Shdr sh;
1110 Elf_Data *data;
1111
1112 idx++;
1113 if (gelf_getshdr(scn, &sh) != &sh) {
077c066a
JDB		 1114 pr_warning("failed to get section(%d) header from %s\n",
1115 idx, obj->path);
6371ca3b 1116 err = -LIBBPF_ERRNO__FORMAT;
29603665
WN		 1117 goto out;
1118 }
1119
1120 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
1121 if (!name) {
077c066a
JDB		 1122 pr_warning("failed to get section(%d) name from %s\n",
1123 idx, obj->path);
6371ca3b 1124 err = -LIBBPF_ERRNO__FORMAT;
29603665
WN		 1125 goto out;
1126 }
1127
1128 data = elf_getdata(scn, 0);
1129 if (!data) {
077c066a
JDB		 1130 pr_warning("failed to get section(%d) data from %s(%s)\n",
1131 idx, name, obj->path);
6371ca3b 1132 err = -LIBBPF_ERRNO__FORMAT;
29603665
WN		 1133 goto out;
1134 }
077c066a
JDB		 1135 pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
1136 idx, name, (unsigned long)data->d_size,
29603665
WN		 1137 (int)sh.sh_link, (unsigned long)sh.sh_flags,
1138 (int)sh.sh_type);
cb1e5e96 1139
1713d68b 1140 if (strcmp(name, "license") == 0) {
cb1e5e96
WN		 1141 err = bpf_object__init_license(obj,
1142 data->d_buf,
1143 data->d_size);
1713d68b 1144 } else if (strcmp(name, "version") == 0) {
cb1e5e96
WN		 1145 err = bpf_object__init_kversion(obj,
1146 data->d_buf,
1147 data->d_size);
1713d68b 1148 } else if (strcmp(name, "maps") == 0) {
666810e8 1149 obj->efile.maps_shndx = idx;
1713d68b
DB		 1150 } else if (strcmp(name, BTF_ELF_SEC) == 0) {
1151 btf_data = data;
2993e051 1152 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
f0187f0b 1153 btf_ext_data = data;
8a138aed 1154 } else if (sh.sh_type == SHT_SYMTAB) {
bec7d68c
WN		 1155 if (obj->efile.symbols) {
1156 pr_warning("bpf: multiple SYMTAB in %s\n",
1157 obj->path);
6371ca3b 1158 err = -LIBBPF_ERRNO__FORMAT;
77ba9a5b 1159 } else {
bec7d68c 1160 obj->efile.symbols = data;
77ba9a5b
WN		 1161 obj->efile.strtabidx = sh.sh_link;
1162 }
f8c7a4d4
JS		 1163 } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) {
1164 if (sh.sh_flags & SHF_EXECINSTR) {
1165 if (strcmp(name, ".text") == 0)
1166 obj->efile.text_shndx = idx;
1167 err = bpf_object__add_program(obj, data->d_buf,
1168 data->d_size, name, idx);
1169 if (err) {
1170 char errmsg[STRERR_BUFSIZE];
1171 char *cp = libbpf_strerror_r(-err, errmsg,
1172 sizeof(errmsg));
1173
1174 pr_warning("failed to alloc program %s (%s): %s",
1175 name, obj->path, cp);
1176 }
d859900c
DB		 1177 } else if (strcmp(name, ".data") == 0) {
1178 obj->efile.data = data;
1179 obj->efile.data_shndx = idx;
1180 } else if (strcmp(name, ".rodata") == 0) {
1181 obj->efile.rodata = data;
1182 obj->efile.rodata_shndx = idx;
1183 } else {
1184 pr_debug("skip section(%d) %s\n", idx, name);
a5b8bd47 1185 }
b62f06e8
WN		 1186 } else if (sh.sh_type == SHT_REL) {
1187 void *reloc = obj->efile.reloc;
1188 int nr_reloc = obj->efile.nr_reloc + 1;
e3d91b0c
JDB		 1189 int sec = sh.sh_info; /* points to other section */
1190
1191 /* Only do relo for section with exec instructions */
1192 if (!section_have_execinstr(obj, sec)) {
1193 pr_debug("skip relo %s(%d) for section(%d)\n",
1194 name, idx, sec);
1195 continue;
1196 }
b62f06e8 1197
531b014e
JK		 1198 reloc = reallocarray(reloc, nr_reloc,
1199 sizeof(*obj->efile.reloc));
b62f06e8
WN		 1200 if (!reloc) {
1201 pr_warning("realloc failed\n");
1202 err = -ENOMEM;
1203 } else {
1204 int n = nr_reloc - 1;
1205
1206 obj->efile.reloc = reloc;
1207 obj->efile.nr_reloc = nr_reloc;
1208
1209 obj->efile.reloc[n].shdr = sh;
1210 obj->efile.reloc[n].data = data;
1211 }
d859900c
DB		 1212 } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) {
1213 obj->efile.bss = data;
1214 obj->efile.bss_shndx = idx;
077c066a
JDB		 1215 } else {
1216 pr_debug("skip section(%d) %s\n", idx, name);
bec7d68c 1217 }
cb1e5e96
WN		 1218 if (err)
1219 goto out;
29603665 1220 }
561bbcca 1221
77ba9a5b
WN		 1222 if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) {
1223 pr_warning("Corrupted ELF file: index of strtab invalid\n");
1224 return LIBBPF_ERRNO__FORMAT;
1225 }
1713d68b
DB		 1226 if (btf_data) {
1227 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
1228 if (IS_ERR(obj->btf)) {
1229 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
1230 BTF_ELF_SEC, PTR_ERR(obj->btf));
1231 obj->btf = NULL;
1232 } else {
1233 err = btf__finalize_data(obj, obj->btf);
d7c4b398
AN		 1234 if (!err) {
1235 bpf_object__sanitize_btf(obj);
1713d68b 1236 err = btf__load(obj->btf);
d7c4b398 1237 }
1713d68b
DB		 1238 if (err) {
1239 pr_warning("Error finalizing and loading %s into kernel: %d. Ignored and continue.\n",
1240 BTF_ELF_SEC, err);
1241 btf__free(obj->btf);
1242 obj->btf = NULL;
1243 err = 0;
1244 }
1245 }
1246 }
f0187f0b
MKL		 1247 if (btf_ext_data) {
1248 if (!obj->btf) {
1249 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
1250 BTF_EXT_ELF_SEC, BTF_ELF_SEC);
1251 } else {
1252 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf,
8461ef8b 1253 btf_ext_data->d_size);
f0187f0b
MKL		 1254 if (IS_ERR(obj->btf_ext)) {
1255 pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n",
1256 BTF_EXT_ELF_SEC,
1257 PTR_ERR(obj->btf_ext));
1258 obj->btf_ext = NULL;
d7c4b398
AN		 1259 } else {
1260 bpf_object__sanitize_btf_ext(obj);
f0187f0b
MKL		 1261 }
1262 }
1263 }
d859900c 1264 if (bpf_object__has_maps(obj)) {
c034a177 1265 err = bpf_object__init_maps(obj, flags);
88cda1c9
MKL		 1266 if (err)
1267 goto out;
1268 }
1269 err = bpf_object__init_prog_names(obj);
29603665
WN		 1270out:
1271 return err;
1272}
1273
34090915
WN		 1274static struct bpf_program *
1275bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
1276{
1277 struct bpf_program *prog;
1278 size_t i;
1279
1280 for (i = 0; i < obj->nr_programs; i++) {
1281 prog = &obj->programs[i];
1282 if (prog->idx == idx)
1283 return prog;
1284 }
1285 return NULL;
1286}
1287
6d4b198b
JK		 1288struct bpf_program *
1289bpf_object__find_program_by_title(struct bpf_object *obj, const char *title)
1290{
1291 struct bpf_program *pos;
1292
1293 bpf_object__for_each_program(pos, obj) {
1294 if (pos->section_name && !strcmp(pos->section_name, title))
1295 return pos;
1296 }
1297 return NULL;
1298}
1299
d859900c
DB		 1300static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
1301 int shndx)
1302{
1303 return shndx == obj->efile.data_shndx ||
1304 shndx == obj->efile.bss_shndx ||
1305 shndx == obj->efile.rodata_shndx;
1306}
1307
1308static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
1309 int shndx)
1310{
1311 return shndx == obj->efile.maps_shndx;
1312}
1313
1314static bool bpf_object__relo_in_known_section(const struct bpf_object *obj,
1315 int shndx)
1316{
1317 return shndx == obj->efile.text_shndx ||
1318 bpf_object__shndx_is_maps(obj, shndx) ||
1319 bpf_object__shndx_is_data(obj, shndx);
1320}
1321
1322static enum libbpf_map_type
1323bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
1324{
1325 if (shndx == obj->efile.data_shndx)
1326 return LIBBPF_MAP_DATA;
1327 else if (shndx == obj->efile.bss_shndx)
1328 return LIBBPF_MAP_BSS;
1329 else if (shndx == obj->efile.rodata_shndx)
1330 return LIBBPF_MAP_RODATA;
1331 else
1332 return LIBBPF_MAP_UNSPEC;
1333}
1334
34090915 1335static int
48cca7e4
AS		 1336bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr,
1337 Elf_Data *data, struct bpf_object *obj)
34090915 1338{
48cca7e4 1339 Elf_Data *symbols = obj->efile.symbols;
48cca7e4
AS		 1340 struct bpf_map *maps = obj->maps;
1341 size_t nr_maps = obj->nr_maps;
34090915
WN		 1342 int i, nrels;
1343
1344 pr_debug("collecting relocating info for: '%s'\n",
1345 prog->section_name);
1346 nrels = shdr->sh_size / shdr->sh_entsize;
1347
1348 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
1349 if (!prog->reloc_desc) {
1350 pr_warning("failed to alloc memory in relocation\n");
1351 return -ENOMEM;
1352 }
1353 prog->nr_reloc = nrels;
1354
1355 for (i = 0; i < nrels; i++) {
1356 GElf_Sym sym;
1357 GElf_Rel rel;
1358 unsigned int insn_idx;
d859900c 1359 unsigned int shdr_idx;
34090915 1360 struct bpf_insn *insns = prog->insns;
d859900c
DB		 1361 enum libbpf_map_type type;
1362 const char *name;
34090915
WN		 1363 size_t map_idx;
1364
1365 if (!gelf_getrel(data, i, &rel)) {
1366 pr_warning("relocation: failed to get %d reloc\n", i);
6371ca3b 1367 return -LIBBPF_ERRNO__FORMAT;
34090915
WN		 1368 }
1369
34090915
WN		 1370 if (!gelf_getsym(symbols,
1371 GELF_R_SYM(rel.r_info),
1372 &sym)) {
1373 pr_warning("relocation: symbol %"PRIx64" not found\n",
1374 GELF_R_SYM(rel.r_info));
6371ca3b 1375 return -LIBBPF_ERRNO__FORMAT;
34090915 1376 }
d859900c
DB		 1377
1378 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx,
1379 sym.st_name) ? : "<?>";
1380
1381 pr_debug("relo for %lld value %lld name %d (\'%s\')\n",
7d9890ef 1382 (long long) (rel.r_info >> 32),
d859900c 1383 (long long) sym.st_value, sym.st_name, name);
34090915 1384
d859900c
DB		 1385 shdr_idx = sym.st_shndx;
1386 if (!bpf_object__relo_in_known_section(obj, shdr_idx)) {
1387 pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n",
1388 prog->section_name, shdr_idx);
666810e8
WN		 1389 return -LIBBPF_ERRNO__RELOC;
1390 }
1391
1392 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
1393 pr_debug("relocation: insn_idx=%u\n", insn_idx);
1394
48cca7e4
AS		 1395 if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) {
1396 if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) {
1397 pr_warning("incorrect bpf_call opcode\n");
1398 return -LIBBPF_ERRNO__RELOC;
1399 }
1400 prog->reloc_desc[i].type = RELO_CALL;
1401 prog->reloc_desc[i].insn_idx = insn_idx;
1402 prog->reloc_desc[i].text_off = sym.st_value;
9a94f277 1403 obj->has_pseudo_calls = true;
48cca7e4
AS		 1404 continue;
1405 }
1406
34090915
WN		 1407 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
1408 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
1409 insn_idx, insns[insn_idx].code);
6371ca3b 1410 return -LIBBPF_ERRNO__RELOC;
34090915
WN		 1411 }
1412
d859900c
DB		 1413 if (bpf_object__shndx_is_maps(obj, shdr_idx) ||
1414 bpf_object__shndx_is_data(obj, shdr_idx)) {
1415 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
8837fe5d
DB		 1416 if (type != LIBBPF_MAP_UNSPEC) {
1417 if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) {
1418 pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n",
1419 name, insn_idx, insns[insn_idx].code);
1420 return -LIBBPF_ERRNO__RELOC;
1421 }
1422 if (!obj->caps.global_data) {
1423 pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n",
1424 name, insn_idx);
1425 return -LIBBPF_ERRNO__RELOC;
1426 }
d859900c
DB		 1427 }
1428
f8c7a4d4 1429 for (map_idx = 0; map_idx < nr_maps; map_idx++) {
d859900c
DB		 1430 if (maps[map_idx].libbpf_type != type)
1431 continue;
1432 if (type != LIBBPF_MAP_UNSPEC ||
1433 (type == LIBBPF_MAP_UNSPEC &&
1434 maps[map_idx].offset == sym.st_value)) {
f8c7a4d4
JS		 1435 pr_debug("relocation: find map %zd (%s) for insn %u\n",
1436 map_idx, maps[map_idx].name, insn_idx);
1437 break;
1438 }
94e5adec 1439 }
94e5adec 1440
f8c7a4d4
JS		 1441 if (map_idx >= nr_maps) {
1442 pr_warning("bpf relocation: map_idx %d large than %d\n",
1443 (int)map_idx, (int)nr_maps - 1);
1444 return -LIBBPF_ERRNO__RELOC;
1445 }
34090915 1446
d859900c
DB		 1447 prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ?
1448 RELO_DATA : RELO_LD64;
f8c7a4d4
JS		 1449 prog->reloc_desc[i].insn_idx = insn_idx;
1450 prog->reloc_desc[i].map_idx = map_idx;
1451 }
34090915
WN		 1452 }
1453 return 0;
1454}
1455
8a138aed
MKL		 1456static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf)
1457{
1458 struct bpf_map_def *def = &map->def;
d859900c 1459 __u32 key_type_id = 0, value_type_id = 0;
96408c43 1460 int ret;
8a138aed 1461
d859900c
DB		 1462 if (!bpf_map__is_internal(map)) {
1463 ret = btf__get_map_kv_tids(btf, map->name, def->key_size,
1464 def->value_size, &key_type_id,
1465 &value_type_id);
1466 } else {
1467 /*
1468 * LLVM annotates global data differently in BTF, that is,
1469 * only as '.data', '.bss' or '.rodata'.
1470 */
1471 ret = btf__find_by_name(btf,
1472 libbpf_type_to_btf_name[map->libbpf_type]);
1473 }
1474 if (ret < 0)
96408c43 1475 return ret;
8a138aed 1476
96408c43 1477 map->btf_key_type_id = key_type_id;
d859900c
DB		 1478 map->btf_value_type_id = bpf_map__is_internal(map) ?
1479 ret : value_type_id;
8a138aed
MKL		 1480 return 0;
1481}
1482
26736eb9
JK		 1483int bpf_map__reuse_fd(struct bpf_map *map, int fd)
1484{
1485 struct bpf_map_info info = {};
1486 __u32 len = sizeof(info);
1487 int new_fd, err;
1488 char *new_name;
1489
1490 err = bpf_obj_get_info_by_fd(fd, &info, &len);
1491 if (err)
1492 return err;
1493
1494 new_name = strdup(info.name);
1495 if (!new_name)
1496 return -errno;
1497
1498 new_fd = open("/", O_RDONLY | O_CLOEXEC);
1499 if (new_fd < 0)
1500 goto err_free_new_name;
1501
1502 new_fd = dup3(fd, new_fd, O_CLOEXEC);
1503 if (new_fd < 0)
1504 goto err_close_new_fd;
1505
1506 err = zclose(map->fd);
1507 if (err)
1508 goto err_close_new_fd;
1509 free(map->name);
1510
1511 map->fd = new_fd;
1512 map->name = new_name;
1513 map->def.type = info.type;
1514 map->def.key_size = info.key_size;
1515 map->def.value_size = info.value_size;
1516 map->def.max_entries = info.max_entries;
1517 map->def.map_flags = info.map_flags;
1518 map->btf_key_type_id = info.btf_key_type_id;
1519 map->btf_value_type_id = info.btf_value_type_id;
1520
1521 return 0;
1522
1523err_close_new_fd:
1524 close(new_fd);
1525err_free_new_name:
1526 free(new_name);
1527 return -errno;
1528}
1529
1a11a4c7
AI		 1530int bpf_map__resize(struct bpf_map *map, __u32 max_entries)
1531{
1532 if (!map || !max_entries)
1533 return -EINVAL;
1534
1535 /* If map already created, its attributes can't be changed. */
1536 if (map->fd >= 0)
1537 return -EBUSY;
1538
1539 map->def.max_entries = max_entries;
1540
1541 return 0;
1542}
1543
47eff617
SF		 1544static int
1545bpf_object__probe_name(struct bpf_object *obj)
1546{
1547 struct bpf_load_program_attr attr;
1548 char *cp, errmsg[STRERR_BUFSIZE];
1549 struct bpf_insn insns[] = {
1550 BPF_MOV64_IMM(BPF_REG_0, 0),
1551 BPF_EXIT_INSN(),
1552 };
1553 int ret;
1554
1555 /* make sure basic loading works */
1556
1557 memset(&attr, 0, sizeof(attr));
1558 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
1559 attr.insns = insns;
1560 attr.insns_cnt = ARRAY_SIZE(insns);
1561 attr.license = "GPL";
1562
1563 ret = bpf_load_program_xattr(&attr, NULL, 0);
1564 if (ret < 0) {
1565 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1566 pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n",
1567 __func__, cp, errno);
1568 return -errno;
1569 }
1570 close(ret);
1571
1572 /* now try the same program, but with the name */
1573
1574 attr.name = "test";
1575 ret = bpf_load_program_xattr(&attr, NULL, 0);
1576 if (ret >= 0) {
1577 obj->caps.name = 1;
1578 close(ret);
1579 }
1580
1581 return 0;
1582}
1583
8837fe5d
DB		 1584static int
1585bpf_object__probe_global_data(struct bpf_object *obj)
1586{
1587 struct bpf_load_program_attr prg_attr;
1588 struct bpf_create_map_attr map_attr;
1589 char *cp, errmsg[STRERR_BUFSIZE];
1590 struct bpf_insn insns[] = {
1591 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
1592 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
1593 BPF_MOV64_IMM(BPF_REG_0, 0),
1594 BPF_EXIT_INSN(),
1595 };
1596 int ret, map;
1597
1598 memset(&map_attr, 0, sizeof(map_attr));
1599 map_attr.map_type = BPF_MAP_TYPE_ARRAY;
1600 map_attr.key_size = sizeof(int);
1601 map_attr.value_size = 32;
1602 map_attr.max_entries = 1;
1603
1604 map = bpf_create_map_xattr(&map_attr);
1605 if (map < 0) {
1606 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1607 pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n",
1608 __func__, cp, errno);
1609 return -errno;
1610 }
1611
1612 insns[0].imm = map;
1613
1614 memset(&prg_attr, 0, sizeof(prg_attr));
1615 prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
1616 prg_attr.insns = insns;
1617 prg_attr.insns_cnt = ARRAY_SIZE(insns);
1618 prg_attr.license = "GPL";
1619
1620 ret = bpf_load_program_xattr(&prg_attr, NULL, 0);
1621 if (ret >= 0) {
1622 obj->caps.global_data = 1;
1623 close(ret);
1624 }
1625
1626 close(map);
1627 return 0;
1628}
1629
d7c4b398
AN		 1630static int bpf_object__probe_btf_func(struct bpf_object *obj)
1631{
1632 const char strs[] = "\0int\0x\0a";
1633 /* void x(int a) {} */
1634 __u32 types[] = {
1635 /* int */
1636 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
1637 /* FUNC_PROTO */ /* [2] */
1638 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
1639 BTF_PARAM_ENC(7, 1),
1640 /* FUNC x */ /* [3] */
1641 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
1642 };
1643 int res;
1644
1645 res = libbpf__probe_raw_btf((char *)types, sizeof(types),
1646 strs, sizeof(strs));
1647 if (res < 0)
1648 return res;
1649 if (res > 0)
1650 obj->caps.btf_func = 1;
1651 return 0;
1652}
1653
1654static int bpf_object__probe_btf_datasec(struct bpf_object *obj)
1655{
1656 const char strs[] = "\0x\0.data";
1657 /* static int a; */
1658 __u32 types[] = {
1659 /* int */
1660 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */
1661 /* VAR x */ /* [2] */
1662 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
1663 BTF_VAR_STATIC,
1664 /* DATASEC val */ /* [3] */
1665 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
1666 BTF_VAR_SECINFO_ENC(2, 0, 4),
1667 };
1668 int res;
1669
1670 res = libbpf__probe_raw_btf((char *)types, sizeof(types),
1671 strs, sizeof(strs));
1672 if (res < 0)
1673 return res;
1674 if (res > 0)
1675 obj->caps.btf_datasec = 1;
1676 return 0;
1677}
1678
47eff617
SF		 1679static int
1680bpf_object__probe_caps(struct bpf_object *obj)
1681{
8837fe5d
DB		 1682 int (*probe_fn[])(struct bpf_object *obj) = {
1683 bpf_object__probe_name,
1684 bpf_object__probe_global_data,
d7c4b398
AN		 1685 bpf_object__probe_btf_func,
1686 bpf_object__probe_btf_datasec,
8837fe5d
DB		 1687 };
1688 int i, ret;
1689
1690 for (i = 0; i < ARRAY_SIZE(probe_fn); i++) {
1691 ret = probe_fn[i](obj);
1692 if (ret < 0)
15ea164e 1693 pr_debug("Probe #%d failed with %d.\n", i, ret);
8837fe5d
DB		 1694 }
1695
1696 return 0;
47eff617
SF		 1697}
1698
d859900c
DB		 1699static int
1700bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
1701{
1702 char *cp, errmsg[STRERR_BUFSIZE];
1703 int err, zero = 0;
1704 __u8 *data;
1705
1706 /* Nothing to do here since kernel already zero-initializes .bss map. */
1707 if (map->libbpf_type == LIBBPF_MAP_BSS)
1708 return 0;
1709
1710 data = map->libbpf_type == LIBBPF_MAP_DATA ?
1711 obj->sections.data : obj->sections.rodata;
1712
1713 err = bpf_map_update_elem(map->fd, &zero, data, 0);
1714 /* Freeze .rodata map as read-only from syscall side. */
1715 if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) {
1716 err = bpf_map_freeze(map->fd);
1717 if (err) {
1718 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1719 pr_warning("Error freezing map(%s) as read-only: %s\n",
1720 map->name, cp);
1721 err = 0;
1722 }
1723 }
1724 return err;
1725}
1726
52d3352e
WN		 1727static int
1728bpf_object__create_maps(struct bpf_object *obj)
1729{
8a138aed 1730 struct bpf_create_map_attr create_attr = {};
52d3352e 1731 unsigned int i;
8a138aed 1732 int err;
52d3352e 1733
9d759a9b 1734 for (i = 0; i < obj->nr_maps; i++) {
8a138aed
MKL		 1735 struct bpf_map *map = &obj->maps[i];
1736 struct bpf_map_def *def = &map->def;
1ce6a9fc 1737 char *cp, errmsg[STRERR_BUFSIZE];
8a138aed
MKL		 1738 int *pfd = &map->fd;
1739
26736eb9
JK		 1740 if (map->fd >= 0) {
1741 pr_debug("skip map create (preset) %s: fd=%d\n",
1742 map->name, map->fd);
1743 continue;
1744 }
1745
94cb310c
SF		 1746 if (obj->caps.name)
1747 create_attr.name = map->name;
f0307a7e 1748 create_attr.map_ifindex = map->map_ifindex;
8a138aed
MKL		 1749 create_attr.map_type = def->type;
1750 create_attr.map_flags = def->map_flags;
1751 create_attr.key_size = def->key_size;
1752 create_attr.value_size = def->value_size;
1753 create_attr.max_entries = def->max_entries;
1754 create_attr.btf_fd = 0;
61746dbe
MKL		 1755 create_attr.btf_key_type_id = 0;
1756 create_attr.btf_value_type_id = 0;
addb9fc9
NS		 1757 if (bpf_map_type__is_map_in_map(def->type) &&
1758 map->inner_map_fd >= 0)
1759 create_attr.inner_map_fd = map->inner_map_fd;
8a138aed
MKL		 1760
1761 if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) {
1762 create_attr.btf_fd = btf__fd(obj->btf);
61746dbe
MKL		 1763 create_attr.btf_key_type_id = map->btf_key_type_id;
1764 create_attr.btf_value_type_id = map->btf_value_type_id;
8a138aed
MKL		 1765 }
1766
1767 *pfd = bpf_create_map_xattr(&create_attr);
61746dbe 1768 if (*pfd < 0 && create_attr.btf_key_type_id) {
24d6a808 1769 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
8a138aed 1770 pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n",
1ce6a9fc 1771 map->name, cp, errno);
8a138aed 1772 create_attr.btf_fd = 0;
61746dbe
MKL		 1773 create_attr.btf_key_type_id = 0;
1774 create_attr.btf_value_type_id = 0;
1775 map->btf_key_type_id = 0;
1776 map->btf_value_type_id = 0;
8a138aed
MKL		 1777 *pfd = bpf_create_map_xattr(&create_attr);
1778 }
1779
52d3352e
WN		 1780 if (*pfd < 0) {
1781 size_t j;
52d3352e 1782
8a138aed 1783 err = *pfd;
d859900c 1784err_out:
24d6a808 1785 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
49bf4b36 1786 pr_warning("failed to create map (name: '%s'): %s\n",
1ce6a9fc 1787 map->name, cp);
52d3352e 1788 for (j = 0; j < i; j++)
9d759a9b 1789 zclose(obj->maps[j].fd);
52d3352e
WN		 1790 return err;
1791 }
d859900c
DB		 1792
1793 if (bpf_map__is_internal(map)) {
1794 err = bpf_object__populate_internal_map(obj, map);
1795 if (err < 0) {
1796 zclose(*pfd);
1797 goto err_out;
1798 }
1799 }
1800
8a138aed 1801 pr_debug("create map %s: fd=%d\n", map->name, *pfd);
52d3352e
WN		 1802 }
1803
52d3352e
WN		 1804 return 0;
1805}
1806
f0187f0b
MKL		 1807static int
1808check_btf_ext_reloc_err(struct bpf_program *prog, int err,
1809 void *btf_prog_info, const char *info_name)
1810{
1811 if (err != -ENOENT) {
1812 pr_warning("Error in loading %s for sec %s.\n",
1813 info_name, prog->section_name);
1814 return err;
1815 }
1816
1817 /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */
1818
1819 if (btf_prog_info) {
1820 /*
1821 * Some info has already been found but has problem
1822 * in the last btf_ext reloc. Must have to error
1823 * out.
1824 */
1825 pr_warning("Error in relocating %s for sec %s.\n",
1826 info_name, prog->section_name);
1827 return err;
1828 }
1829
1830 /*
1831 * Have problem loading the very first info. Ignore
1832 * the rest.
1833 */
1834 pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n",
1835 info_name, prog->section_name, info_name);
1836 return 0;
1837}
1838
1839static int
1840bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj,
1841 const char *section_name, __u32 insn_offset)
1842{
1843 int err;
1844
1845 if (!insn_offset || prog->func_info) {
1846 /*
1847 * !insn_offset => main program
1848 *
1849 * For sub prog, the main program's func_info has to
1850 * be loaded first (i.e. prog->func_info != NULL)
1851 */
1852 err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext,
1853 section_name, insn_offset,
1854 &prog->func_info,
1855 &prog->func_info_cnt);
1856 if (err)
1857 return check_btf_ext_reloc_err(prog, err,
1858 prog->func_info,
1859 "bpf_func_info");
1860
1861 prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext);
1862 }
1863
3d650141
MKL		 1864 if (!insn_offset || prog->line_info) {
1865 err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext,
1866 section_name, insn_offset,
1867 &prog->line_info,
1868 &prog->line_info_cnt);
1869 if (err)
1870 return check_btf_ext_reloc_err(prog, err,
1871 prog->line_info,
1872 "bpf_line_info");
1873
1874 prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext);
1875 }
1876
f0187f0b
MKL		 1877 if (!insn_offset)
1878 prog->btf_fd = btf__fd(obj->btf);
1879
1880 return 0;
1881}
1882
48cca7e4
AS		 1883static int
1884bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj,
1885 struct reloc_desc *relo)
1886{
1887 struct bpf_insn *insn, *new_insn;
1888 struct bpf_program *text;
1889 size_t new_cnt;
2993e051 1890 int err;
48cca7e4
AS		 1891
1892 if (relo->type != RELO_CALL)
1893 return -LIBBPF_ERRNO__RELOC;
1894
1895 if (prog->idx == obj->efile.text_shndx) {
1896 pr_warning("relo in .text insn %d into off %d\n",
1897 relo->insn_idx, relo->text_off);
1898 return -LIBBPF_ERRNO__RELOC;
1899 }
1900
1901 if (prog->main_prog_cnt == 0) {
1902 text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx);
1903 if (!text) {
1904 pr_warning("no .text section found yet relo into text exist\n");
1905 return -LIBBPF_ERRNO__RELOC;
1906 }
1907 new_cnt = prog->insns_cnt + text->insns_cnt;
531b014e 1908 new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn));
48cca7e4
AS		 1909 if (!new_insn) {
1910 pr_warning("oom in prog realloc\n");
1911 return -ENOMEM;
1912 }
2993e051 1913
f0187f0b
MKL		 1914 if (obj->btf_ext) {
1915 err = bpf_program_reloc_btf_ext(prog, obj,
1916 text->section_name,
1917 prog->insns_cnt);
1918 if (err)
2993e051 1919 return err;
2993e051
YS		 1920 }
1921
48cca7e4
AS		 1922 memcpy(new_insn + prog->insns_cnt, text->insns,
1923 text->insns_cnt * sizeof(*insn));
1924 prog->insns = new_insn;
1925 prog->main_prog_cnt = prog->insns_cnt;
1926 prog->insns_cnt = new_cnt;
b1a2ce82
JC		 1927 pr_debug("added %zd insn from %s to prog %s\n",
1928 text->insns_cnt, text->section_name,
1929 prog->section_name);
48cca7e4
AS		 1930 }
1931 insn = &prog->insns[relo->insn_idx];
1932 insn->imm += prog->main_prog_cnt - relo->insn_idx;
48cca7e4
AS		 1933 return 0;
1934}
1935
8a47a6c5 1936static int
9d759a9b 1937bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj)
8a47a6c5 1938{
48cca7e4 1939 int i, err;
8a47a6c5 1940
2993e051
YS		 1941 if (!prog)
1942 return 0;
1943
f0187f0b
MKL		 1944 if (obj->btf_ext) {
1945 err = bpf_program_reloc_btf_ext(prog, obj,
1946 prog->section_name, 0);
1947 if (err)
2993e051 1948 return err;
2993e051
YS		 1949 }
1950
1951 if (!prog->reloc_desc)
8a47a6c5
WN		 1952 return 0;
1953
1954 for (i = 0; i < prog->nr_reloc; i++) {
d859900c
DB		 1955 if (prog->reloc_desc[i].type == RELO_LD64 ||
1956 prog->reloc_desc[i].type == RELO_DATA) {
1957 bool relo_data = prog->reloc_desc[i].type == RELO_DATA;
48cca7e4
AS		 1958 struct bpf_insn *insns = prog->insns;
1959 int insn_idx, map_idx;
8a47a6c5 1960
48cca7e4
AS		 1961 insn_idx = prog->reloc_desc[i].insn_idx;
1962 map_idx = prog->reloc_desc[i].map_idx;
8a47a6c5 1963
d859900c 1964 if (insn_idx + 1 >= (int)prog->insns_cnt) {
48cca7e4
AS		 1965 pr_warning("relocation out of range: '%s'\n",
1966 prog->section_name);
1967 return -LIBBPF_ERRNO__RELOC;
1968 }
d859900c
DB		 1969
1970 if (!relo_data) {
1971 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
1972 } else {
1973 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE;
1974 insns[insn_idx + 1].imm = insns[insn_idx].imm;
1975 }
48cca7e4 1976 insns[insn_idx].imm = obj->maps[map_idx].fd;
f8c7a4d4 1977 } else if (prog->reloc_desc[i].type == RELO_CALL) {
48cca7e4
AS		 1978 err = bpf_program__reloc_text(prog, obj,
1979 &prog->reloc_desc[i]);
1980 if (err)
1981 return err;
8a47a6c5 1982 }
8a47a6c5
WN		 1983 }
1984
1985 zfree(&prog->reloc_desc);
1986 prog->nr_reloc = 0;
1987 return 0;
1988}
1989
1990
1991static int
1992bpf_object__relocate(struct bpf_object *obj)
1993{
1994 struct bpf_program *prog;
1995 size_t i;
1996 int err;
1997
1998 for (i = 0; i < obj->nr_programs; i++) {
1999 prog = &obj->programs[i];
2000
9d759a9b 2001 err = bpf_program__relocate(prog, obj);
8a47a6c5
WN		 2002 if (err) {
2003 pr_warning("failed to relocate '%s'\n",
2004 prog->section_name);
2005 return err;
2006 }
2007 }
2008 return 0;
2009}
2010
34090915
WN		 2011static int bpf_object__collect_reloc(struct bpf_object *obj)
2012{
2013 int i, err;
2014
2015 if (!obj_elf_valid(obj)) {
2016 pr_warning("Internal error: elf object is closed\n");
6371ca3b 2017 return -LIBBPF_ERRNO__INTERNAL;
34090915
WN		 2018 }
2019
2020 for (i = 0; i < obj->efile.nr_reloc; i++) {
2021 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
2022 Elf_Data *data = obj->efile.reloc[i].data;
2023 int idx = shdr->sh_info;
2024 struct bpf_program *prog;
34090915
WN		 2025
2026 if (shdr->sh_type != SHT_REL) {
2027 pr_warning("internal error at %d\n", __LINE__);
6371ca3b 2028 return -LIBBPF_ERRNO__INTERNAL;
34090915
WN		 2029 }
2030
2031 prog = bpf_object__find_prog_by_idx(obj, idx);
2032 if (!prog) {
077c066a 2033 pr_warning("relocation failed: no section(%d)\n", idx);
6371ca3b 2034 return -LIBBPF_ERRNO__RELOC;
34090915
WN		 2035 }
2036
48cca7e4 2037 err = bpf_program__collect_reloc(prog,
34090915 2038 shdr, data,
48cca7e4 2039 obj);
34090915 2040 if (err)
6371ca3b 2041 return err;
34090915
WN		 2042 }
2043 return 0;
2044}
2045
55cffde2 2046static int
2993e051 2047load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt,
f0187f0b 2048 char *license, __u32 kern_version, int *pfd)
55cffde2 2049{
d7be143b 2050 struct bpf_load_program_attr load_attr;
1ce6a9fc 2051 char *cp, errmsg[STRERR_BUFSIZE];
da11b417 2052 int log_buf_size = BPF_LOG_BUF_SIZE;
55cffde2 2053 char *log_buf;
d7be143b 2054 int ret;
55cffde2 2055
d7be143b 2056 memset(&load_attr, 0, sizeof(struct bpf_load_program_attr));
2993e051
YS		 2057 load_attr.prog_type = prog->type;
2058 load_attr.expected_attach_type = prog->expected_attach_type;
5b32a23e
SF		 2059 if (prog->caps->name)
2060 load_attr.name = prog->name;
d7be143b
AI		 2061 load_attr.insns = insns;
2062 load_attr.insns_cnt = insns_cnt;
2063 load_attr.license = license;
2064 load_attr.kern_version = kern_version;
2993e051 2065 load_attr.prog_ifindex = prog->prog_ifindex;
462c124c 2066 load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0;
2993e051
YS		 2067 load_attr.func_info = prog->func_info;
2068 load_attr.func_info_rec_size = prog->func_info_rec_size;
f0187f0b 2069 load_attr.func_info_cnt = prog->func_info_cnt;
3d650141
MKL		 2070 load_attr.line_info = prog->line_info;
2071 load_attr.line_info_rec_size = prog->line_info_rec_size;
2072 load_attr.line_info_cnt = prog->line_info_cnt;
da11b417 2073 load_attr.log_level = prog->log_level;
04656198 2074 load_attr.prog_flags = prog->prog_flags;
d7be143b 2075 if (!load_attr.insns || !load_attr.insns_cnt)
55cffde2
WN		 2076 return -EINVAL;
2077
da11b417
AS		 2078retry_load:
2079 log_buf = malloc(log_buf_size);
55cffde2
WN		 2080 if (!log_buf)
2081 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
2082
da11b417 2083 ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size);
55cffde2
WN		 2084
2085 if (ret >= 0) {
da11b417
AS		 2086 if (load_attr.log_level)
2087 pr_debug("verifier log:\n%s", log_buf);
55cffde2
WN		 2088 *pfd = ret;
2089 ret = 0;
2090 goto out;
2091 }
2092
da11b417
AS		 2093 if (errno == ENOSPC) {
2094 log_buf_size <<= 1;
2095 free(log_buf);
2096 goto retry_load;
2097 }
6371ca3b 2098 ret = -LIBBPF_ERRNO__LOAD;
24d6a808 2099 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1ce6a9fc 2100 pr_warning("load bpf program failed: %s\n", cp);
55cffde2 2101
6371ca3b
WN		 2102 if (log_buf && log_buf[0] != '\0') {
2103 ret = -LIBBPF_ERRNO__VERIFY;
55cffde2
WN		 2104 pr_warning("-- BEGIN DUMP LOG ---\n");
2105 pr_warning("\n%s\n", log_buf);
2106 pr_warning("-- END LOG --\n");
d7be143b
AI		 2107 } else if (load_attr.insns_cnt >= BPF_MAXINSNS) {
2108 pr_warning("Program too large (%zu insns), at most %d insns\n",
2109 load_attr.insns_cnt, BPF_MAXINSNS);
705fa219 2110 ret = -LIBBPF_ERRNO__PROG2BIG;
6371ca3b 2111 } else {
705fa219 2112 /* Wrong program type? */
d7be143b 2113 if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) {
705fa219
WN		 2114 int fd;
2115
d7be143b
AI		 2116 load_attr.prog_type = BPF_PROG_TYPE_KPROBE;
2117 load_attr.expected_attach_type = 0;
2118 fd = bpf_load_program_xattr(&load_attr, NULL, 0);
705fa219
WN		 2119 if (fd >= 0) {
2120 close(fd);
2121 ret = -LIBBPF_ERRNO__PROGTYPE;
2122 goto out;
2123 }
6371ca3b 2124 }
705fa219
WN		 2125
2126 if (log_buf)
2127 ret = -LIBBPF_ERRNO__KVER;
55cffde2
WN		 2128 }
2129
2130out:
2131 free(log_buf);
2132 return ret;
2133}
2134
29cd77f4 2135int
55cffde2 2136bpf_program__load(struct bpf_program *prog,
e5b0863c 2137 char *license, __u32 kern_version)
55cffde2 2138{
b580563e 2139 int err = 0, fd, i;
55cffde2 2140
b580563e
WN		 2141 if (prog->instances.nr < 0 || !prog->instances.fds) {
2142 if (prog->preprocessor) {
2143 pr_warning("Internal error: can't load program '%s'\n",
2144 prog->section_name);
2145 return -LIBBPF_ERRNO__INTERNAL;
2146 }
55cffde2 2147
b580563e
WN		 2148 prog->instances.fds = malloc(sizeof(int));
2149 if (!prog->instances.fds) {
2150 pr_warning("Not enough memory for BPF fds\n");
2151 return -ENOMEM;
2152 }
2153 prog->instances.nr = 1;
2154 prog->instances.fds[0] = -1;
2155 }
2156
2157 if (!prog->preprocessor) {
2158 if (prog->instances.nr != 1) {
2159 pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n",
2160 prog->section_name, prog->instances.nr);
2161 }
2993e051 2162 err = load_program(prog, prog->insns, prog->insns_cnt,
f0187f0b 2163 license, kern_version, &fd);
b580563e
WN		 2164 if (!err)
2165 prog->instances.fds[0] = fd;
2166 goto out;
2167 }
2168
2169 for (i = 0; i < prog->instances.nr; i++) {
2170 struct bpf_prog_prep_result result;
2171 bpf_program_prep_t preprocessor = prog->preprocessor;
2172
1ad9cbb8 2173 memset(&result, 0, sizeof(result));
b580563e
WN		 2174 err = preprocessor(prog, i, prog->insns,
2175 prog->insns_cnt, &result);
2176 if (err) {
2177 pr_warning("Preprocessing the %dth instance of program '%s' failed\n",
2178 i, prog->section_name);
2179 goto out;
2180 }
2181
2182 if (!result.new_insn_ptr || !result.new_insn_cnt) {
2183 pr_debug("Skip loading the %dth instance of program '%s'\n",
2184 i, prog->section_name);
2185 prog->instances.fds[i] = -1;
2186 if (result.pfd)
2187 *result.pfd = -1;
2188 continue;
2189 }
2190
2993e051 2191 err = load_program(prog, result.new_insn_ptr,
b580563e 2192 result.new_insn_cnt,
f0187f0b 2193 license, kern_version, &fd);
b580563e
WN		 2194
2195 if (err) {
2196 pr_warning("Loading the %dth instance of program '%s' failed\n",
2197 i, prog->section_name);
2198 goto out;
2199 }
2200
2201 if (result.pfd)
2202 *result.pfd = fd;
2203 prog->instances.fds[i] = fd;
2204 }
2205out:
55cffde2
WN		 2206 if (err)
2207 pr_warning("failed to load program '%s'\n",
2208 prog->section_name);
2209 zfree(&prog->insns);
2210 prog->insns_cnt = 0;
2211 return err;
2212}
2213
9a94f277
JK		 2214static bool bpf_program__is_function_storage(struct bpf_program *prog,
2215 struct bpf_object *obj)
2216{
2217 return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls;
2218}
2219
55cffde2 2220static int
60276f98 2221bpf_object__load_progs(struct bpf_object *obj, int log_level)
55cffde2
WN		 2222{
2223 size_t i;
2224 int err;
2225
2226 for (i = 0; i < obj->nr_programs; i++) {
9a94f277 2227 if (bpf_program__is_function_storage(&obj->programs[i], obj))
48cca7e4 2228 continue;
501b125a 2229 obj->programs[i].log_level |= log_level;
55cffde2
WN		 2230 err = bpf_program__load(&obj->programs[i],
2231 obj->license,
2232 obj->kern_version);
2233 if (err)
2234 return err;
2235 }
2236 return 0;
2237}
2238
17387dd5
JK		 2239static bool bpf_prog_type__needs_kver(enum bpf_prog_type type)
2240{
2241 switch (type) {
2242 case BPF_PROG_TYPE_SOCKET_FILTER:
2243 case BPF_PROG_TYPE_SCHED_CLS:
2244 case BPF_PROG_TYPE_SCHED_ACT:
2245 case BPF_PROG_TYPE_XDP:
2246 case BPF_PROG_TYPE_CGROUP_SKB:
2247 case BPF_PROG_TYPE_CGROUP_SOCK:
2248 case BPF_PROG_TYPE_LWT_IN:
2249 case BPF_PROG_TYPE_LWT_OUT:
2250 case BPF_PROG_TYPE_LWT_XMIT:
004d4b27 2251 case BPF_PROG_TYPE_LWT_SEG6LOCAL:
17387dd5
JK		 2252 case BPF_PROG_TYPE_SOCK_OPS:
2253 case BPF_PROG_TYPE_SK_SKB:
2254 case BPF_PROG_TYPE_CGROUP_DEVICE:
2255 case BPF_PROG_TYPE_SK_MSG:
2256 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
6bdd533c 2257 case BPF_PROG_TYPE_LIRC_MODE2:
6bc8529c 2258 case BPF_PROG_TYPE_SK_REUSEPORT:
c22fbae7 2259 case BPF_PROG_TYPE_FLOW_DISSECTOR:
17387dd5 2260 case BPF_PROG_TYPE_UNSPEC:
17387dd5 2261 case BPF_PROG_TYPE_TRACEPOINT:
17387dd5 2262 case BPF_PROG_TYPE_RAW_TRACEPOINT:
4635b0ae 2263 case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
47ae7e3d 2264 case BPF_PROG_TYPE_PERF_EVENT:
063cc9f0 2265 case BPF_PROG_TYPE_CGROUP_SYSCTL:
47ae7e3d
NS		 2266 return false;
2267 case BPF_PROG_TYPE_KPROBE:
17387dd5
JK		 2268 default:
2269 return true;
2270 }
2271}
2272
2273static int bpf_object__validate(struct bpf_object *obj, bool needs_kver)
cb1e5e96 2274{
17387dd5 2275 if (needs_kver && obj->kern_version == 0) {
cb1e5e96
WN		 2276 pr_warning("%s doesn't provide kernel version\n",
2277 obj->path);
6371ca3b 2278 return -LIBBPF_ERRNO__KVERSION;
cb1e5e96
WN		 2279 }
2280 return 0;
2281}
2282
1a5e3fb1 2283static struct bpf_object *
17387dd5 2284__bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz,
c034a177 2285 bool needs_kver, int flags)
1a5e3fb1
WN		 2286{
2287 struct bpf_object *obj;
6371ca3b 2288 int err;
1a5e3fb1
WN		 2289
2290 if (elf_version(EV_CURRENT) == EV_NONE) {
2291 pr_warning("failed to init libelf for %s\n", path);
6371ca3b 2292 return ERR_PTR(-LIBBPF_ERRNO__LIBELF);
1a5e3fb1
WN		 2293 }
2294
6c956392 2295 obj = bpf_object__new(path, obj_buf, obj_buf_sz);
6371ca3b
WN		 2296 if (IS_ERR(obj))
2297 return obj;
1a5e3fb1 2298
6371ca3b
WN		 2299 CHECK_ERR(bpf_object__elf_init(obj), err, out);
2300 CHECK_ERR(bpf_object__check_endianness(obj), err, out);
8837fe5d 2301 CHECK_ERR(bpf_object__probe_caps(obj), err, out);
c034a177 2302 CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out);
6371ca3b 2303 CHECK_ERR(bpf_object__collect_reloc(obj), err, out);
17387dd5 2304 CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out);
1a5e3fb1
WN		 2305
2306 bpf_object__elf_finish(obj);
2307 return obj;
2308out:
2309 bpf_object__close(obj);
6371ca3b 2310 return ERR_PTR(err);
1a5e3fb1
WN		 2311}
2312
c034a177
JF		 2313struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr,
2314 int flags)
1a5e3fb1
WN		 2315{
2316 /* param validation */
07f2d4ea 2317 if (!attr->file)
1a5e3fb1
WN		 2318 return NULL;
2319
07f2d4ea
JK		 2320 pr_debug("loading %s\n", attr->file);
2321
2322 return __bpf_object__open(attr->file, NULL, 0,
c034a177
JF		 2323 bpf_prog_type__needs_kver(attr->prog_type),
2324 flags);
2325}
2326
2327struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr)
2328{
2329 return __bpf_object__open_xattr(attr, 0);
07f2d4ea
JK		 2330}
2331
2332struct bpf_object *bpf_object__open(const char *path)
2333{
2334 struct bpf_object_open_attr attr = {
2335 .file = path,
2336 .prog_type = BPF_PROG_TYPE_UNSPEC,
2337 };
1a5e3fb1 2338
07f2d4ea 2339 return bpf_object__open_xattr(&attr);
6c956392
WN		 2340}
2341
2342struct bpf_object *bpf_object__open_buffer(void *obj_buf,
acf860ae
WN		 2343 size_t obj_buf_sz,
2344 const char *name)
6c956392 2345{
acf860ae
WN		 2346 char tmp_name[64];
2347
6c956392
WN		 2348 /* param validation */
2349 if (!obj_buf || obj_buf_sz <= 0)
2350 return NULL;
2351
acf860ae
WN		 2352 if (!name) {
2353 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",
2354 (unsigned long)obj_buf,
2355 (unsigned long)obj_buf_sz);
2356 tmp_name[sizeof(tmp_name) - 1] = '\0';
2357 name = tmp_name;
2358 }
2359 pr_debug("loading object '%s' from buffer\n",
2360 name);
6c956392 2361
c034a177 2362 return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true);
1a5e3fb1
WN		 2363}
2364
52d3352e
WN		 2365int bpf_object__unload(struct bpf_object *obj)
2366{
2367 size_t i;
2368
2369 if (!obj)
2370 return -EINVAL;
2371
9d759a9b
WN		 2372 for (i = 0; i < obj->nr_maps; i++)
2373 zclose(obj->maps[i].fd);
52d3352e 2374
55cffde2
WN		 2375 for (i = 0; i < obj->nr_programs; i++)
2376 bpf_program__unload(&obj->programs[i]);
2377
52d3352e
WN		 2378 return 0;
2379}
2380
60276f98 2381int bpf_object__load_xattr(struct bpf_object_load_attr *attr)
52d3352e 2382{
60276f98 2383 struct bpf_object *obj;
6371ca3b
WN		 2384 int err;
2385
60276f98
QM		 2386 if (!attr)
2387 return -EINVAL;
2388 obj = attr->obj;
52d3352e
WN		 2389 if (!obj)
2390 return -EINVAL;
2391
2392 if (obj->loaded) {
2393 pr_warning("object should not be loaded twice\n");
2394 return -EINVAL;
2395 }
2396
2397 obj->loaded = true;
6371ca3b
WN		 2398
2399 CHECK_ERR(bpf_object__create_maps(obj), err, out);
2400 CHECK_ERR(bpf_object__relocate(obj), err, out);
60276f98 2401 CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out);
52d3352e
WN		 2402
2403 return 0;
2404out:
2405 bpf_object__unload(obj);
2406 pr_warning("failed to load object '%s'\n", obj->path);
6371ca3b 2407 return err;
52d3352e
WN		 2408}
2409
60276f98
QM		 2410int bpf_object__load(struct bpf_object *obj)
2411{
2412 struct bpf_object_load_attr attr = {
2413 .obj = obj,
2414 };
2415
2416 return bpf_object__load_xattr(&attr);
2417}
2418
f367540c
JS		 2419static int check_path(const char *path)
2420{
1ce6a9fc 2421 char *cp, errmsg[STRERR_BUFSIZE];
f367540c
JS		 2422 struct statfs st_fs;
2423 char *dname, *dir;
2424 int err = 0;
2425
2426 if (path == NULL)
2427 return -EINVAL;
2428
2429 dname = strdup(path);
2430 if (dname == NULL)
2431 return -ENOMEM;
2432
2433 dir = dirname(dname);
2434 if (statfs(dir, &st_fs)) {
24d6a808 2435 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1ce6a9fc 2436 pr_warning("failed to statfs %s: %s\n", dir, cp);
f367540c
JS		 2437 err = -errno;
2438 }
2439 free(dname);
2440
2441 if (!err && st_fs.f_type != BPF_FS_MAGIC) {
2442 pr_warning("specified path %s is not on BPF FS\n", path);
2443 err = -EINVAL;
2444 }
2445
2446 return err;
2447}
2448
2449int bpf_program__pin_instance(struct bpf_program *prog, const char *path,
2450 int instance)
2451{
1ce6a9fc 2452 char *cp, errmsg[STRERR_BUFSIZE];
f367540c
JS		 2453 int err;
2454
2455 err = check_path(path);
2456 if (err)
2457 return err;
2458
2459 if (prog == NULL) {
2460 pr_warning("invalid program pointer\n");
2461 return -EINVAL;
2462 }
2463
2464 if (instance < 0 || instance >= prog->instances.nr) {
2465 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
2466 instance, prog->section_name, prog->instances.nr);
2467 return -EINVAL;
2468 }
2469
2470 if (bpf_obj_pin(prog->instances.fds[instance], path)) {
24d6a808 2471 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1ce6a9fc 2472 pr_warning("failed to pin program: %s\n", cp);
f367540c
JS		 2473 return -errno;
2474 }
2475 pr_debug("pinned program '%s'\n", path);
2476
2477 return 0;
2478}
2479
0c19a9fb
SF		 2480int bpf_program__unpin_instance(struct bpf_program *prog, const char *path,
2481 int instance)
2482{
2483 int err;
2484
2485 err = check_path(path);
2486 if (err)
2487 return err;
2488
2489 if (prog == NULL) {
2490 pr_warning("invalid program pointer\n");
2491 return -EINVAL;
2492 }
2493
2494 if (instance < 0 || instance >= prog->instances.nr) {
2495 pr_warning("invalid prog instance %d of prog %s (max %d)\n",
2496 instance, prog->section_name, prog->instances.nr);
2497 return -EINVAL;
2498 }
2499
2500 err = unlink(path);
2501 if (err != 0)
2502 return -errno;
2503 pr_debug("unpinned program '%s'\n", path);
2504
2505 return 0;
2506}
2507
f367540c
JS		 2508static int make_dir(const char *path)
2509{
1ce6a9fc 2510 char *cp, errmsg[STRERR_BUFSIZE];
f367540c
JS		 2511 int err = 0;
2512
2513 if (mkdir(path, 0700) && errno != EEXIST)
2514 err = -errno;
2515
1ce6a9fc 2516 if (err) {
24d6a808 2517 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
1ce6a9fc
TR		 2518 pr_warning("failed to mkdir %s: %s\n", path, cp);
2519 }
f367540c
JS		 2520 return err;
2521}
2522
2523int bpf_program__pin(struct bpf_program *prog, const char *path)
2524{
2525 int i, err;
2526
2527 err = check_path(path);
2528 if (err)
2529 return err;
2530
2531 if (prog == NULL) {
2532 pr_warning("invalid program pointer\n");
2533 return -EINVAL;
2534 }
2535
2536 if (prog->instances.nr <= 0) {
2537 pr_warning("no instances of prog %s to pin\n",
2538 prog->section_name);
2539 return -EINVAL;
2540 }
2541
fd734c5c
SF		 2542 if (prog->instances.nr == 1) {
2543 /* don't create subdirs when pinning single instance */
2544 return bpf_program__pin_instance(prog, path, 0);
2545 }
2546
f367540c
JS		 2547 err = make_dir(path);
2548 if (err)
2549 return err;
2550
0c19a9fb
SF		 2551 for (i = 0; i < prog->instances.nr; i++) {
2552 char buf[PATH_MAX];
2553 int len;
2554
2555 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
2556 if (len < 0) {
2557 err = -EINVAL;
2558 goto err_unpin;
2559 } else if (len >= PATH_MAX) {
2560 err = -ENAMETOOLONG;
2561 goto err_unpin;
2562 }
2563
2564 err = bpf_program__pin_instance(prog, buf, i);
2565 if (err)
2566 goto err_unpin;
2567 }
2568
2569 return 0;
2570
2571err_unpin:
2572 for (i = i - 1; i >= 0; i--) {
2573 char buf[PATH_MAX];
2574 int len;
2575
2576 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
2577 if (len < 0)
2578 continue;
2579 else if (len >= PATH_MAX)
2580 continue;
2581
2582 bpf_program__unpin_instance(prog, buf, i);
2583 }
2584
2585 rmdir(path);
2586
2587 return err;
2588}
2589
2590int bpf_program__unpin(struct bpf_program *prog, const char *path)
2591{
2592 int i, err;
2593
2594 err = check_path(path);
2595 if (err)
2596 return err;
2597
2598 if (prog == NULL) {
2599 pr_warning("invalid program pointer\n");
2600 return -EINVAL;
2601 }
2602
2603 if (prog->instances.nr <= 0) {
2604 pr_warning("no instances of prog %s to pin\n",
2605 prog->section_name);
2606 return -EINVAL;
fd734c5c
SF		 2607 }
2608
2609 if (prog->instances.nr == 1) {
2610 /* don't create subdirs when pinning single instance */
2611 return bpf_program__unpin_instance(prog, path, 0);
0c19a9fb
SF		 2612 }
2613
f367540c
JS		 2614 for (i = 0; i < prog->instances.nr; i++) {
2615 char buf[PATH_MAX];
2616 int len;
2617
2618 len = snprintf(buf, PATH_MAX, "%s/%d", path, i);
2619 if (len < 0)
2620 return -EINVAL;
2621 else if (len >= PATH_MAX)
2622 return -ENAMETOOLONG;
2623
0c19a9fb 2624 err = bpf_program__unpin_instance(prog, buf, i);
f367540c
JS		 2625 if (err)
2626 return err;
2627 }
2628
0c19a9fb
SF		 2629 err = rmdir(path);
2630 if (err)
2631 return -errno;
2632
f367540c
JS		 2633 return 0;
2634}
2635
b6989f35
JS		 2636int bpf_map__pin(struct bpf_map *map, const char *path)
2637{
1ce6a9fc 2638 char *cp, errmsg[STRERR_BUFSIZE];
b6989f35
JS		 2639 int err;
2640
2641 err = check_path(path);
2642 if (err)
2643 return err;
2644
2645 if (map == NULL) {
2646 pr_warning("invalid map pointer\n");
2647 return -EINVAL;
2648 }
2649
2650 if (bpf_obj_pin(map->fd, path)) {
24d6a808 2651 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg));
1ce6a9fc 2652 pr_warning("failed to pin map: %s\n", cp);
b6989f35
JS		 2653 return -errno;
2654 }
2655
2656 pr_debug("pinned map '%s'\n", path);
0c19a9fb 2657
b6989f35
JS		 2658 return 0;
2659}
2660
0c19a9fb
SF		 2661int bpf_map__unpin(struct bpf_map *map, const char *path)
2662{
2663 int err;
2664
2665 err = check_path(path);
2666 if (err)
2667 return err;
2668
2669 if (map == NULL) {
2670 pr_warning("invalid map pointer\n");
2671 return -EINVAL;
2672 }
2673
2674 err = unlink(path);
2675 if (err != 0)
2676 return -errno;
2677 pr_debug("unpinned map '%s'\n", path);
2678
2679 return 0;
2680}
2681
2682int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
d5148d85 2683{
d5148d85
JS		 2684 struct bpf_map *map;
2685 int err;
2686
2687 if (!obj)
2688 return -ENOENT;
2689
2690 if (!obj->loaded) {
2691 pr_warning("object not yet loaded; load it first\n");
2692 return -ENOENT;
2693 }
2694
2695 err = make_dir(path);
2696 if (err)
2697 return err;
2698
f74a53d9 2699 bpf_object__for_each_map(map, obj) {
0c19a9fb
SF		 2700 char buf[PATH_MAX];
2701 int len;
2702
2703 len = snprintf(buf, PATH_MAX, "%s/%s", path,
2704 bpf_map__name(map));
2705 if (len < 0) {
2706 err = -EINVAL;
2707 goto err_unpin_maps;
2708 } else if (len >= PATH_MAX) {
2709 err = -ENAMETOOLONG;
2710 goto err_unpin_maps;
2711 }
2712
2713 err = bpf_map__pin(map, buf);
2714 if (err)
2715 goto err_unpin_maps;
2716 }
2717
2718 return 0;
2719
2720err_unpin_maps:
2721 while ((map = bpf_map__prev(map, obj))) {
2722 char buf[PATH_MAX];
2723 int len;
2724
2725 len = snprintf(buf, PATH_MAX, "%s/%s", path,
2726 bpf_map__name(map));
2727 if (len < 0)
2728 continue;
2729 else if (len >= PATH_MAX)
2730 continue;
2731
2732 bpf_map__unpin(map, buf);
2733 }
2734
2735 return err;
2736}
2737
2738int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
2739{
2740 struct bpf_map *map;
2741 int err;
2742
2743 if (!obj)
2744 return -ENOENT;
2745
f74a53d9 2746 bpf_object__for_each_map(map, obj) {
d5148d85
JS		 2747 char buf[PATH_MAX];
2748 int len;
2749
2750 len = snprintf(buf, PATH_MAX, "%s/%s", path,
2751 bpf_map__name(map));
2752 if (len < 0)
2753 return -EINVAL;
2754 else if (len >= PATH_MAX)
2755 return -ENAMETOOLONG;
2756
0c19a9fb 2757 err = bpf_map__unpin(map, buf);
d5148d85
JS		 2758 if (err)
2759 return err;
2760 }
2761
0c19a9fb
SF		 2762 return 0;
2763}
2764
2765int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
2766{
2767 struct bpf_program *prog;
2768 int err;
2769
2770 if (!obj)
2771 return -ENOENT;
2772
2773 if (!obj->loaded) {
2774 pr_warning("object not yet loaded; load it first\n");
2775 return -ENOENT;
2776 }
2777
2778 err = make_dir(path);
2779 if (err)
2780 return err;
2781
2782 bpf_object__for_each_program(prog, obj) {
2783 char buf[PATH_MAX];
2784 int len;
2785
2786 len = snprintf(buf, PATH_MAX, "%s/%s", path,
33a2c75c 2787 prog->pin_name);
0c19a9fb
SF		 2788 if (len < 0) {
2789 err = -EINVAL;
2790 goto err_unpin_programs;
2791 } else if (len >= PATH_MAX) {
2792 err = -ENAMETOOLONG;
2793 goto err_unpin_programs;
2794 }
2795
2796 err = bpf_program__pin(prog, buf);
2797 if (err)
2798 goto err_unpin_programs;
2799 }
2800
2801 return 0;
2802
2803err_unpin_programs:
2804 while ((prog = bpf_program__prev(prog, obj))) {
2805 char buf[PATH_MAX];
2806 int len;
2807
2808 len = snprintf(buf, PATH_MAX, "%s/%s", path,
33a2c75c 2809 prog->pin_name);
0c19a9fb
SF		 2810 if (len < 0)
2811 continue;
2812 else if (len >= PATH_MAX)
2813 continue;
2814
2815 bpf_program__unpin(prog, buf);
2816 }
2817
2818 return err;
2819}
2820
2821int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
2822{
2823 struct bpf_program *prog;
2824 int err;
2825
2826 if (!obj)
2827 return -ENOENT;
2828
d5148d85
JS		 2829 bpf_object__for_each_program(prog, obj) {
2830 char buf[PATH_MAX];
2831 int len;
2832
2833 len = snprintf(buf, PATH_MAX, "%s/%s", path,
33a2c75c 2834 prog->pin_name);
d5148d85
JS		 2835 if (len < 0)
2836 return -EINVAL;
2837 else if (len >= PATH_MAX)
2838 return -ENAMETOOLONG;
2839
0c19a9fb 2840 err = bpf_program__unpin(prog, buf);
d5148d85
JS		 2841 if (err)
2842 return err;
2843 }
2844
2845 return 0;
2846}
2847
0c19a9fb
SF		 2848int bpf_object__pin(struct bpf_object *obj, const char *path)
2849{
2850 int err;
2851
2852 err = bpf_object__pin_maps(obj, path);
2853 if (err)
2854 return err;
2855
2856 err = bpf_object__pin_programs(obj, path);
2857 if (err) {
2858 bpf_object__unpin_maps(obj, path);
2859 return err;
2860 }
2861
2862 return 0;
2863}
2864
1a5e3fb1
WN		 2865void bpf_object__close(struct bpf_object *obj)
2866{
a5b8bd47
WN		 2867 size_t i;
2868
1a5e3fb1
WN		 2869 if (!obj)
2870 return;
2871
10931d24
WN		 2872 if (obj->clear_priv)
2873 obj->clear_priv(obj, obj->priv);
2874
1a5e3fb1 2875 bpf_object__elf_finish(obj);
52d3352e 2876 bpf_object__unload(obj);
8a138aed 2877 btf__free(obj->btf);
2993e051 2878 btf_ext__free(obj->btf_ext);
1a5e3fb1 2879
9d759a9b 2880 for (i = 0; i < obj->nr_maps; i++) {
561bbcca 2881 zfree(&obj->maps[i].name);
9d759a9b
WN		 2882 if (obj->maps[i].clear_priv)
2883 obj->maps[i].clear_priv(&obj->maps[i],
2884 obj->maps[i].priv);
2885 obj->maps[i].priv = NULL;
2886 obj->maps[i].clear_priv = NULL;
2887 }
d859900c
DB		 2888
2889 zfree(&obj->sections.rodata);
2890 zfree(&obj->sections.data);
9d759a9b
WN		 2891 zfree(&obj->maps);
2892 obj->nr_maps = 0;
a5b8bd47
WN		 2893
2894 if (obj->programs && obj->nr_programs) {
2895 for (i = 0; i < obj->nr_programs; i++)
2896 bpf_program__exit(&obj->programs[i]);
2897 }
2898 zfree(&obj->programs);
2899
9a208eff 2900 list_del(&obj->list);
1a5e3fb1
WN		 2901 free(obj);
2902}
aa9b1ac3 2903
9a208eff
WN		 2904struct bpf_object *
2905bpf_object__next(struct bpf_object *prev)
2906{
2907 struct bpf_object *next;
2908
2909 if (!prev)
2910 next = list_first_entry(&bpf_objects_list,
2911 struct bpf_object,
2912 list);
2913 else
2914 next = list_next_entry(prev, list);
2915
2916 /* Empty list is noticed here so don't need checking on entry. */
2917 if (&next->list == &bpf_objects_list)
2918 return NULL;
2919
2920 return next;
2921}
2922
a7fe0450 2923const char *bpf_object__name(struct bpf_object *obj)
acf860ae 2924{
a7fe0450 2925 return obj ? obj->path : ERR_PTR(-EINVAL);
acf860ae
WN		 2926}
2927
a7fe0450 2928unsigned int bpf_object__kversion(struct bpf_object *obj)
45825d8a 2929{
a7fe0450 2930 return obj ? obj->kern_version : 0;
45825d8a
WN		 2931}
2932
789f6bab
AI		 2933struct btf *bpf_object__btf(struct bpf_object *obj)
2934{
2935 return obj ? obj->btf : NULL;
2936}
2937
8a138aed
MKL		 2938int bpf_object__btf_fd(const struct bpf_object *obj)
2939{
2940 return obj->btf ? btf__fd(obj->btf) : -1;
2941}
2942
10931d24
WN		 2943int bpf_object__set_priv(struct bpf_object *obj, void *priv,
2944 bpf_object_clear_priv_t clear_priv)
2945{
2946 if (obj->priv && obj->clear_priv)
2947 obj->clear_priv(obj, obj->priv);
2948
2949 obj->priv = priv;
2950 obj->clear_priv = clear_priv;
2951 return 0;
2952}
2953
2954void *bpf_object__priv(struct bpf_object *obj)
2955{
2956 return obj ? obj->priv : ERR_PTR(-EINVAL);
2957}
2958
eac7d845 2959static struct bpf_program *
a83d6e76 2960__bpf_program__iter(struct bpf_program *p, struct bpf_object *obj, bool forward)
aa9b1ac3 2961{
a83d6e76 2962 size_t nr_programs = obj->nr_programs;
0c19a9fb 2963 ssize_t idx;
aa9b1ac3 2964
a83d6e76 2965 if (!nr_programs)
aa9b1ac3 2966 return NULL;
aa9b1ac3 2967
a83d6e76
MKL		 2968 if (!p)
2969 /* Iter from the beginning */
2970 return forward ? &obj->programs[0] :
2971 &obj->programs[nr_programs - 1];
2972
0c19a9fb 2973 if (p->obj != obj) {
aa9b1ac3
WN		 2974 pr_warning("error: program handler doesn't match object\n");
2975 return NULL;
2976 }
2977
a83d6e76 2978 idx = (p - obj->programs) + (forward ? 1 : -1);
0c19a9fb 2979 if (idx >= obj->nr_programs || idx < 0)
aa9b1ac3
WN		 2980 return NULL;
2981 return &obj->programs[idx];
2982}
2983
eac7d845
JK		 2984struct bpf_program *
2985bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
2986{
2987 struct bpf_program *prog = prev;
2988
2989 do {
a83d6e76 2990 prog = __bpf_program__iter(prog, obj, true);
0c19a9fb
SF		 2991 } while (prog && bpf_program__is_function_storage(prog, obj));
2992
2993 return prog;
2994}
2995
2996struct bpf_program *
2997bpf_program__prev(struct bpf_program *next, struct bpf_object *obj)
2998{
2999 struct bpf_program *prog = next;
3000
0c19a9fb 3001 do {
a83d6e76 3002 prog = __bpf_program__iter(prog, obj, false);
eac7d845
JK		 3003 } while (prog && bpf_program__is_function_storage(prog, obj));
3004
3005 return prog;
3006}
3007
edb13ed4
ACM		 3008int bpf_program__set_priv(struct bpf_program *prog, void *priv,
3009 bpf_program_clear_priv_t clear_priv)
aa9b1ac3
WN		 3010{
3011 if (prog->priv && prog->clear_priv)
3012 prog->clear_priv(prog, prog->priv);
3013
3014 prog->priv = priv;
3015 prog->clear_priv = clear_priv;
3016 return 0;
3017}
3018
be834ffb 3019void *bpf_program__priv(struct bpf_program *prog)
aa9b1ac3 3020{
be834ffb 3021 return prog ? prog->priv : ERR_PTR(-EINVAL);
aa9b1ac3
WN		 3022}
3023
9aba3613
JK		 3024void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
3025{
3026 prog->prog_ifindex = ifindex;
3027}
3028
715f8db9 3029const char *bpf_program__title(struct bpf_program *prog, bool needs_copy)
aa9b1ac3
WN		 3030{
3031 const char *title;
3032
3033 title = prog->section_name;
715f8db9 3034 if (needs_copy) {
aa9b1ac3
WN		 3035 title = strdup(title);
3036 if (!title) {
3037 pr_warning("failed to strdup program title\n");
6371ca3b 3038 return ERR_PTR(-ENOMEM);
aa9b1ac3
WN		 3039 }
3040 }
3041
3042 return title;
3043}
3044
3045int bpf_program__fd(struct bpf_program *prog)
3046{
b580563e
WN		 3047 return bpf_program__nth_fd(prog, 0);
3048}
3049
3050int bpf_program__set_prep(struct bpf_program *prog, int nr_instances,
3051 bpf_program_prep_t prep)
3052{
3053 int *instances_fds;
3054
3055 if (nr_instances <= 0 || !prep)
3056 return -EINVAL;
3057
3058 if (prog->instances.nr > 0 || prog->instances.fds) {
3059 pr_warning("Can't set pre-processor after loading\n");
3060 return -EINVAL;
3061 }
3062
3063 instances_fds = malloc(sizeof(int) * nr_instances);
3064 if (!instances_fds) {
3065 pr_warning("alloc memory failed for fds\n");
3066 return -ENOMEM;
3067 }
3068
3069 /* fill all fd with -1 */
3070 memset(instances_fds, -1, sizeof(int) * nr_instances);
3071
3072 prog->instances.nr = nr_instances;
3073 prog->instances.fds = instances_fds;
3074 prog->preprocessor = prep;
3075 return 0;
3076}
3077
3078int bpf_program__nth_fd(struct bpf_program *prog, int n)
3079{
3080 int fd;
3081
1e960043
JK		 3082 if (!prog)
3083 return -EINVAL;
3084
b580563e
WN		 3085 if (n >= prog->instances.nr || n < 0) {
3086 pr_warning("Can't get the %dth fd from program %s: only %d instances\n",
3087 n, prog->section_name, prog->instances.nr);
3088 return -EINVAL;
3089 }
3090
3091 fd = prog->instances.fds[n];
3092 if (fd < 0) {
3093 pr_warning("%dth instance of program '%s' is invalid\n",
3094 n, prog->section_name);
3095 return -ENOENT;
3096 }
3097
3098 return fd;
aa9b1ac3 3099}
9d759a9b 3100
dd26b7f5 3101void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
5f44e4c8
WN		 3102{
3103 prog->type = type;
3104}
3105
5f44e4c8
WN		 3106static bool bpf_program__is_type(struct bpf_program *prog,
3107 enum bpf_prog_type type)
3108{
3109 return prog ? (prog->type == type) : false;
3110}
3111
ed794073
JS		 3112#define BPF_PROG_TYPE_FNS(NAME, TYPE) \
3113int bpf_program__set_##NAME(struct bpf_program *prog) \
3114{ \
3115 if (!prog) \
3116 return -EINVAL; \
3117 bpf_program__set_type(prog, TYPE); \
3118 return 0; \
3119} \
3120 \
3121bool bpf_program__is_##NAME(struct bpf_program *prog) \
3122{ \
3123 return bpf_program__is_type(prog, TYPE); \
3124} \
3125
7803ba73 3126BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER);
ed794073 3127BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE);
7803ba73
JS		 3128BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS);
3129BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT);
ed794073 3130BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT);
e14c93fd 3131BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT);
7803ba73
JS		 3132BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP);
3133BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT);
5f44e4c8 3134
16962b24
JF		 3135void bpf_program__set_expected_attach_type(struct bpf_program *prog,
3136 enum bpf_attach_type type)
d7be143b
AI		 3137{
3138 prog->expected_attach_type = type;
3139}
3140
36153532
AI		 3141#define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \
3142 { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype }
d7be143b 3143
956b620f 3144/* Programs that can NOT be attached. */
36153532 3145#define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0)
d7be143b 3146
956b620f
AI		 3147/* Programs that can be attached. */
3148#define BPF_APROG_SEC(string, ptype, atype) \
36153532 3149 BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype)
81efee75 3150
956b620f
AI		 3151/* Programs that must specify expected attach type at load time. */
3152#define BPF_EAPROG_SEC(string, ptype, eatype) \
36153532 3153 BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype)
956b620f
AI		 3154
3155/* Programs that can be attached but attach type can't be identified by section
3156 * name. Kept for backward compatibility.
3157 */
3158#define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype)
e50b0a6f 3159
583c9009
RG		 3160static const struct {
3161 const char *sec;
3162 size_t len;
3163 enum bpf_prog_type prog_type;
d7be143b 3164 enum bpf_attach_type expected_attach_type;
36153532 3165 int is_attachable;
956b620f 3166 enum bpf_attach_type attach_type;
583c9009 3167} section_names[] = {
956b620f
AI		 3168 BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER),
3169 BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE),
3170 BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE),
3171 BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS),
3172 BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT),
3173 BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT),
3174 BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT),
3175 BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP),
3176 BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT),
3177 BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN),
3178 BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT),
3179 BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT),
3180 BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL),
bafa7afe
AI		 3181 BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB,
3182 BPF_CGROUP_INET_INGRESS),
3183 BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB,
3184 BPF_CGROUP_INET_EGRESS),
956b620f
AI		 3185 BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB),
3186 BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK,
3187 BPF_CGROUP_INET_SOCK_CREATE),
3188 BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK,
3189 BPF_CGROUP_INET4_POST_BIND),
3190 BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK,
3191 BPF_CGROUP_INET6_POST_BIND),
3192 BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE,
3193 BPF_CGROUP_DEVICE),
3194 BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS,
3195 BPF_CGROUP_SOCK_OPS),
c6f6851b
AI		 3196 BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB,
3197 BPF_SK_SKB_STREAM_PARSER),
3198 BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB,
3199 BPF_SK_SKB_STREAM_VERDICT),
956b620f
AI		 3200 BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB),
3201 BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG,
3202 BPF_SK_MSG_VERDICT),
3203 BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2,
3204 BPF_LIRC_MODE2),
3205 BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR,
3206 BPF_FLOW_DISSECTOR),
3207 BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
3208 BPF_CGROUP_INET4_BIND),
3209 BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
3210 BPF_CGROUP_INET6_BIND),
3211 BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
3212 BPF_CGROUP_INET4_CONNECT),
3213 BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
3214 BPF_CGROUP_INET6_CONNECT),
3215 BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
3216 BPF_CGROUP_UDP4_SENDMSG),
3217 BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR,
3218 BPF_CGROUP_UDP6_SENDMSG),
063cc9f0
AI		 3219 BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL,
3220 BPF_CGROUP_SYSCTL),
583c9009 3221};
d7be143b 3222
956b620f 3223#undef BPF_PROG_SEC_IMPL
583c9009 3224#undef BPF_PROG_SEC
956b620f
AI		 3225#undef BPF_APROG_SEC
3226#undef BPF_EAPROG_SEC
3227#undef BPF_APROG_COMPAT
583c9009 3228
c76e4c22
TS		 3229#define MAX_TYPE_NAME_SIZE 32
3230
3231static char *libbpf_get_type_names(bool attach_type)
3232{
3233 int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE;
3234 char *buf;
3235
3236 buf = malloc(len);
3237 if (!buf)
3238 return NULL;
3239
3240 buf[0] = '\0';
3241 /* Forge string buf with all available names */
3242 for (i = 0; i < ARRAY_SIZE(section_names); i++) {
3243 if (attach_type && !section_names[i].is_attachable)
3244 continue;
3245
3246 if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) {
3247 free(buf);
3248 return NULL;
3249 }
3250 strcat(buf, " ");
3251 strcat(buf, section_names[i].sec);
3252 }
3253
3254 return buf;
3255}
3256
b60df2a0
JK		 3257int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type,
3258 enum bpf_attach_type *expected_attach_type)
583c9009 3259{
c76e4c22 3260 char *type_names;
583c9009
RG		 3261 int i;
3262
b60df2a0
JK		 3263 if (!name)
3264 return -EINVAL;
583c9009 3265
b60df2a0
JK		 3266 for (i = 0; i < ARRAY_SIZE(section_names); i++) {
3267 if (strncmp(name, section_names[i].sec, section_names[i].len))
3268 continue;
3269 *prog_type = section_names[i].prog_type;
3270 *expected_attach_type = section_names[i].expected_attach_type;
3271 return 0;
3272 }
c76e4c22
TS		 3273 pr_warning("failed to guess program type based on ELF section name '%s'\n", name);
3274 type_names = libbpf_get_type_names(false);
3275 if (type_names != NULL) {
3276 pr_info("supported section(type) names are:%s\n", type_names);
3277 free(type_names);
3278 }
3279
b60df2a0
JK		 3280 return -EINVAL;
3281}
583c9009 3282
956b620f
AI		 3283int libbpf_attach_type_by_name(const char *name,
3284 enum bpf_attach_type *attach_type)
3285{
c76e4c22 3286 char *type_names;
956b620f
AI		 3287 int i;
3288
3289 if (!name)
3290 return -EINVAL;
3291
3292 for (i = 0; i < ARRAY_SIZE(section_names); i++) {
3293 if (strncmp(name, section_names[i].sec, section_names[i].len))
3294 continue;
36153532 3295 if (!section_names[i].is_attachable)
956b620f
AI		 3296 return -EINVAL;
3297 *attach_type = section_names[i].attach_type;
3298 return 0;
3299 }
c76e4c22
TS		 3300 pr_warning("failed to guess attach type based on ELF section name '%s'\n", name);
3301 type_names = libbpf_get_type_names(true);
3302 if (type_names != NULL) {
3303 pr_info("attachable section(type) names are:%s\n", type_names);
3304 free(type_names);
3305 }
3306
956b620f
AI		 3307 return -EINVAL;
3308}
3309
b60df2a0
JK		 3310static int
3311bpf_program__identify_section(struct bpf_program *prog,
3312 enum bpf_prog_type *prog_type,
3313 enum bpf_attach_type *expected_attach_type)
3314{
3315 return libbpf_prog_type_by_name(prog->section_name, prog_type,
3316 expected_attach_type);
583c9009
RG		 3317}
3318
6e009e65 3319int bpf_map__fd(struct bpf_map *map)
9d759a9b 3320{
6e009e65 3321 return map ? map->fd : -EINVAL;
9d759a9b
WN		 3322}
3323
53897a78 3324const struct bpf_map_def *bpf_map__def(struct bpf_map *map)
9d759a9b 3325{
53897a78 3326 return map ? &map->def : ERR_PTR(-EINVAL);
9d759a9b
WN		 3327}
3328
009ad5d5 3329const char *bpf_map__name(struct bpf_map *map)
561bbcca 3330{
009ad5d5 3331 return map ? map->name : NULL;
561bbcca
WN		 3332}
3333
5b891af7 3334__u32 bpf_map__btf_key_type_id(const struct bpf_map *map)
8a138aed 3335{
61746dbe 3336 return map ? map->btf_key_type_id : 0;
8a138aed
MKL		 3337}
3338
5b891af7 3339__u32 bpf_map__btf_value_type_id(const struct bpf_map *map)
8a138aed 3340{
61746dbe 3341 return map ? map->btf_value_type_id : 0;
8a138aed
MKL		 3342}
3343
edb13ed4
ACM		 3344int bpf_map__set_priv(struct bpf_map *map, void *priv,
3345 bpf_map_clear_priv_t clear_priv)
9d759a9b
WN		 3346{
3347 if (!map)
3348 return -EINVAL;
3349
3350 if (map->priv) {
3351 if (map->clear_priv)
3352 map->clear_priv(map, map->priv);
3353 }
3354
3355 map->priv = priv;
3356 map->clear_priv = clear_priv;
3357 return 0;
3358}
3359
b4cbfa56 3360void *bpf_map__priv(struct bpf_map *map)
9d759a9b 3361{
b4cbfa56 3362 return map ? map->priv : ERR_PTR(-EINVAL);
9d759a9b
WN		 3363}
3364
f83fb22c
JK		 3365bool bpf_map__is_offload_neutral(struct bpf_map *map)
3366{
3367 return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY;
3368}
3369
d859900c
DB		 3370bool bpf_map__is_internal(struct bpf_map *map)
3371{
3372 return map->libbpf_type != LIBBPF_MAP_UNSPEC;
3373}
3374
9aba3613
JK		 3375void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex)
3376{
3377 map->map_ifindex = ifindex;
3378}
3379
addb9fc9
NS		 3380int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd)
3381{
3382 if (!bpf_map_type__is_map_in_map(map->def.type)) {
3383 pr_warning("error: unsupported map type\n");
3384 return -EINVAL;
3385 }
3386 if (map->inner_map_fd != -1) {
3387 pr_warning("error: inner_map_fd already specified\n");
3388 return -EINVAL;
3389 }
3390 map->inner_map_fd = fd;
3391 return 0;
3392}
3393
0c19a9fb
SF		 3394static struct bpf_map *
3395__bpf_map__iter(struct bpf_map *m, struct bpf_object *obj, int i)
9d759a9b 3396{
0c19a9fb 3397 ssize_t idx;
9d759a9b
WN		 3398 struct bpf_map *s, *e;
3399
3400 if (!obj || !obj->maps)
3401 return NULL;
3402
3403 s = obj->maps;
3404 e = obj->maps + obj->nr_maps;
3405
0c19a9fb 3406 if ((m < s) || (m >= e)) {
9d759a9b
WN		 3407 pr_warning("error in %s: map handler doesn't belong to object\n",
3408 __func__);
3409 return NULL;
3410 }
3411
0c19a9fb
SF		 3412 idx = (m - obj->maps) + i;
3413 if (idx >= obj->nr_maps || idx < 0)
9d759a9b
WN		 3414 return NULL;
3415 return &obj->maps[idx];
3416}
561bbcca 3417
0c19a9fb
SF		 3418struct bpf_map *
3419bpf_map__next(struct bpf_map *prev, struct bpf_object *obj)
3420{
3421 if (prev == NULL)
3422 return obj->maps;
3423
3424 return __bpf_map__iter(prev, obj, 1);
3425}
3426
3427struct bpf_map *
3428bpf_map__prev(struct bpf_map *next, struct bpf_object *obj)
3429{
3430 if (next == NULL) {
3431 if (!obj->nr_maps)
3432 return NULL;
3433 return obj->maps + obj->nr_maps - 1;
3434 }
3435
3436 return __bpf_map__iter(next, obj, -1);
3437}
3438
561bbcca 3439struct bpf_map *
a7fe0450 3440bpf_object__find_map_by_name(struct bpf_object *obj, const char *name)
561bbcca
WN		 3441{
3442 struct bpf_map *pos;
3443
f74a53d9 3444 bpf_object__for_each_map(pos, obj) {
973170e6 3445 if (pos->name && !strcmp(pos->name, name))
561bbcca
WN		 3446 return pos;
3447 }
3448 return NULL;
3449}
5a6acad1 3450
f3cea32d
MF		 3451int
3452bpf_object__find_map_fd_by_name(struct bpf_object *obj, const char *name)
3453{
3454 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
3455}
3456
5a6acad1
WN		 3457struct bpf_map *
3458bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset)
3459{
3460 int i;
3461
3462 for (i = 0; i < obj->nr_maps; i++) {
3463 if (obj->maps[i].offset == offset)
3464 return &obj->maps[i];
3465 }
3466 return ERR_PTR(-ENOENT);
3467}
e28ff1a8
JS		 3468
3469long libbpf_get_error(const void *ptr)
3470{
d98363b5 3471 return PTR_ERR_OR_ZERO(ptr);
e28ff1a8 3472}
6f6d33f3
JF		 3473
3474int bpf_prog_load(const char *file, enum bpf_prog_type type,
3475 struct bpf_object **pobj, int *prog_fd)
d7be143b
AI		 3476{
3477 struct bpf_prog_load_attr attr;
3478
3479 memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
3480 attr.file = file;
3481 attr.prog_type = type;
3482 attr.expected_attach_type = 0;
3483
3484 return bpf_prog_load_xattr(&attr, pobj, prog_fd);
3485}
3486
3487int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr,
3488 struct bpf_object **pobj, int *prog_fd)
6f6d33f3 3489{
07f2d4ea
JK		 3490 struct bpf_object_open_attr open_attr = {
3491 .file = attr->file,
3492 .prog_type = attr->prog_type,
3493 };
48cca7e4 3494 struct bpf_program *prog, *first_prog = NULL;
d7be143b
AI		 3495 enum bpf_attach_type expected_attach_type;
3496 enum bpf_prog_type prog_type;
6f6d33f3 3497 struct bpf_object *obj;
f0307a7e 3498 struct bpf_map *map;
6f6d33f3
JF		 3499 int err;
3500
d7be143b
AI		 3501 if (!attr)
3502 return -EINVAL;
17387dd5
JK		 3503 if (!attr->file)
3504 return -EINVAL;
d7be143b 3505
07f2d4ea 3506 obj = bpf_object__open_xattr(&open_attr);
3597683c 3507 if (IS_ERR_OR_NULL(obj))
6f6d33f3
JF		 3508 return -ENOENT;
3509
48cca7e4
AS		 3510 bpf_object__for_each_program(prog, obj) {
3511 /*
3512 * If type is not specified, try to guess it based on
3513 * section name.
3514 */
d7be143b 3515 prog_type = attr->prog_type;
f0307a7e 3516 prog->prog_ifindex = attr->ifindex;
d7be143b
AI		 3517 expected_attach_type = attr->expected_attach_type;
3518 if (prog_type == BPF_PROG_TYPE_UNSPEC) {
b60df2a0
JK		 3519 err = bpf_program__identify_section(prog, &prog_type,
3520 &expected_attach_type);
3521 if (err < 0) {
48cca7e4
AS		 3522 bpf_object__close(obj);
3523 return -EINVAL;
3524 }
583c9009 3525 }
48cca7e4 3526
d7be143b
AI		 3527 bpf_program__set_type(prog, prog_type);
3528 bpf_program__set_expected_attach_type(prog,
3529 expected_attach_type);
3530
da11b417 3531 prog->log_level = attr->log_level;
04656198 3532 prog->prog_flags = attr->prog_flags;
69495d2a 3533 if (!first_prog)
48cca7e4
AS		 3534 first_prog = prog;
3535 }
3536
f74a53d9 3537 bpf_object__for_each_map(map, obj) {
f83fb22c
JK		 3538 if (!bpf_map__is_offload_neutral(map))
3539 map->map_ifindex = attr->ifindex;
f0307a7e
DB		 3540 }
3541
48cca7e4
AS		 3542 if (!first_prog) {
3543 pr_warning("object file doesn't contain bpf program\n");
3544 bpf_object__close(obj);
3545 return -ENOENT;
583c9009
RG		 3546 }
3547
6f6d33f3
JF		 3548 err = bpf_object__load(obj);
3549 if (err) {
3550 bpf_object__close(obj);
3551 return -EINVAL;
3552 }
3553
3554 *pobj = obj;
48cca7e4 3555 *prog_fd = bpf_program__fd(first_prog);
6f6d33f3
JF		 3556 return 0;
3557}
d0cabbb0
JK		 3558
3559enum bpf_perf_event_ret
3dca2115
DB		 3560bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size,
3561 void **copy_mem, size_t *copy_size,
3562 bpf_perf_event_print_t fn, void *private_data)
d0cabbb0 3563{
3dca2115 3564 struct perf_event_mmap_page *header = mmap_mem;
a64af0ef 3565 __u64 data_head = ring_buffer_read_head(header);
d0cabbb0 3566 __u64 data_tail = header->data_tail;
3dca2115
DB		 3567 void *base = ((__u8 *)header) + page_size;
3568 int ret = LIBBPF_PERF_EVENT_CONT;
3569 struct perf_event_header *ehdr;
3570 size_t ehdr_size;
3571
3572 while (data_head != data_tail) {
3573 ehdr = base + (data_tail & (mmap_size - 1));
3574 ehdr_size = ehdr->size;
3575
3576 if (((void *)ehdr) + ehdr_size > base + mmap_size) {
3577 void *copy_start = ehdr;
3578 size_t len_first = base + mmap_size - copy_start;
3579 size_t len_secnd = ehdr_size - len_first;
3580
3581 if (*copy_size < ehdr_size) {
3582 free(*copy_mem);
3583 *copy_mem = malloc(ehdr_size);
3584 if (!*copy_mem) {
3585 *copy_size = 0;
d0cabbb0
JK		 3586 ret = LIBBPF_PERF_EVENT_ERROR;
3587 break;
3588 }
3dca2115 3589 *copy_size = ehdr_size;
d0cabbb0
JK		 3590 }
3591
3dca2115
DB		 3592 memcpy(*copy_mem, copy_start, len_first);
3593 memcpy(*copy_mem + len_first, base, len_secnd);
3594 ehdr = *copy_mem;
d0cabbb0
JK		 3595 }
3596
3dca2115
DB		 3597 ret = fn(ehdr, private_data);
3598 data_tail += ehdr_size;
d0cabbb0
JK		 3599 if (ret != LIBBPF_PERF_EVENT_CONT)
3600 break;
d0cabbb0
JK		 3601 }
3602
a64af0ef 3603 ring_buffer_write_tail(header, data_tail);
d0cabbb0
JK		 3604 return ret;
3605}
34be1646
SL		 3606
3607struct bpf_prog_info_array_desc {
3608 int array_offset; /* e.g. offset of jited_prog_insns */
3609 int count_offset; /* e.g. offset of jited_prog_len */
3610 int size_offset; /* > 0: offset of rec size,
3611 * < 0: fix size of -size_offset
3612 */
3613};
3614
3615static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = {
3616 [BPF_PROG_INFO_JITED_INSNS] = {
3617 offsetof(struct bpf_prog_info, jited_prog_insns),
3618 offsetof(struct bpf_prog_info, jited_prog_len),
3619 -1,
3620 },
3621 [BPF_PROG_INFO_XLATED_INSNS] = {
3622 offsetof(struct bpf_prog_info, xlated_prog_insns),
3623 offsetof(struct bpf_prog_info, xlated_prog_len),
3624 -1,
3625 },
3626 [BPF_PROG_INFO_MAP_IDS] = {
3627 offsetof(struct bpf_prog_info, map_ids),
3628 offsetof(struct bpf_prog_info, nr_map_ids),
3629 -(int)sizeof(__u32),
3630 },
3631 [BPF_PROG_INFO_JITED_KSYMS] = {
3632 offsetof(struct bpf_prog_info, jited_ksyms),
3633 offsetof(struct bpf_prog_info, nr_jited_ksyms),
3634 -(int)sizeof(__u64),
3635 },
3636 [BPF_PROG_INFO_JITED_FUNC_LENS] = {
3637 offsetof(struct bpf_prog_info, jited_func_lens),
3638 offsetof(struct bpf_prog_info, nr_jited_func_lens),
3639 -(int)sizeof(__u32),
3640 },
3641 [BPF_PROG_INFO_FUNC_INFO] = {
3642 offsetof(struct bpf_prog_info, func_info),
3643 offsetof(struct bpf_prog_info, nr_func_info),
3644 offsetof(struct bpf_prog_info, func_info_rec_size),
3645 },
3646 [BPF_PROG_INFO_LINE_INFO] = {
3647 offsetof(struct bpf_prog_info, line_info),
3648 offsetof(struct bpf_prog_info, nr_line_info),
3649 offsetof(struct bpf_prog_info, line_info_rec_size),
3650 },
3651 [BPF_PROG_INFO_JITED_LINE_INFO] = {
3652 offsetof(struct bpf_prog_info, jited_line_info),
3653 offsetof(struct bpf_prog_info, nr_jited_line_info),
3654 offsetof(struct bpf_prog_info, jited_line_info_rec_size),
3655 },
3656 [BPF_PROG_INFO_PROG_TAGS] = {
3657 offsetof(struct bpf_prog_info, prog_tags),
3658 offsetof(struct bpf_prog_info, nr_prog_tags),
3659 -(int)sizeof(__u8) * BPF_TAG_SIZE,
3660 },
3661
3662};
3663
3664static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset)
3665{
3666 __u32 *array = (__u32 *)info;
3667
3668 if (offset >= 0)
3669 return array[offset / sizeof(__u32)];
3670 return -(int)offset;
3671}
3672
3673static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset)
3674{
3675 __u64 *array = (__u64 *)info;
3676
3677 if (offset >= 0)
3678 return array[offset / sizeof(__u64)];
3679 return -(int)offset;
3680}
3681
3682static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset,
3683 __u32 val)
3684{
3685 __u32 *array = (__u32 *)info;
3686
3687 if (offset >= 0)
3688 array[offset / sizeof(__u32)] = val;
3689}
3690
3691static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset,
3692 __u64 val)
3693{
3694 __u64 *array = (__u64 *)info;
3695
3696 if (offset >= 0)
3697 array[offset / sizeof(__u64)] = val;
3698}
3699
3700struct bpf_prog_info_linear *
3701bpf_program__get_prog_info_linear(int fd, __u64 arrays)
3702{
3703 struct bpf_prog_info_linear *info_linear;
3704 struct bpf_prog_info info = {};
3705 __u32 info_len = sizeof(info);
3706 __u32 data_len = 0;
3707 int i, err;
3708 void *ptr;
3709
3710 if (arrays >> BPF_PROG_INFO_LAST_ARRAY)
3711 return ERR_PTR(-EINVAL);
3712
3713 /* step 1: get array dimensions */
3714 err = bpf_obj_get_info_by_fd(fd, &info, &info_len);
3715 if (err) {
3716 pr_debug("can't get prog info: %s", strerror(errno));
3717 return ERR_PTR(-EFAULT);
3718 }
3719
3720 /* step 2: calculate total size of all arrays */
3721 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
3722 bool include_array = (arrays & (1UL << i)) > 0;
3723 struct bpf_prog_info_array_desc *desc;
3724 __u32 count, size;
3725
3726 desc = bpf_prog_info_array_desc + i;
3727
3728 /* kernel is too old to support this field */
3729 if (info_len < desc->array_offset + sizeof(__u32) ||
3730 info_len < desc->count_offset + sizeof(__u32) ||
3731 (desc->size_offset > 0 && info_len < desc->size_offset))
3732 include_array = false;
3733
3734 if (!include_array) {
3735 arrays &= ~(1UL << i); /* clear the bit */
3736 continue;
3737 }
3738
3739 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
3740 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
3741
3742 data_len += count * size;
3743 }
3744
3745 /* step 3: allocate continuous memory */
3746 data_len = roundup(data_len, sizeof(__u64));
3747 info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len);
3748 if (!info_linear)
3749 return ERR_PTR(-ENOMEM);
3750
3751 /* step 4: fill data to info_linear->info */
3752 info_linear->arrays = arrays;
3753 memset(&info_linear->info, 0, sizeof(info));
3754 ptr = info_linear->data;
3755
3756 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
3757 struct bpf_prog_info_array_desc *desc;
3758 __u32 count, size;
3759
3760 if ((arrays & (1UL << i)) == 0)
3761 continue;
3762
3763 desc = bpf_prog_info_array_desc + i;
3764 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
3765 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
3766 bpf_prog_info_set_offset_u32(&info_linear->info,
3767 desc->count_offset, count);
3768 bpf_prog_info_set_offset_u32(&info_linear->info,
3769 desc->size_offset, size);
3770 bpf_prog_info_set_offset_u64(&info_linear->info,
3771 desc->array_offset,
3772 ptr_to_u64(ptr));
3773 ptr += count * size;
3774 }
3775
3776 /* step 5: call syscall again to get required arrays */
3777 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len);
3778 if (err) {
3779 pr_debug("can't get prog info: %s", strerror(errno));
3780 free(info_linear);
3781 return ERR_PTR(-EFAULT);
3782 }
3783
3784 /* step 6: verify the data */
3785 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
3786 struct bpf_prog_info_array_desc *desc;
3787 __u32 v1, v2;
3788
3789 if ((arrays & (1UL << i)) == 0)
3790 continue;
3791
3792 desc = bpf_prog_info_array_desc + i;
3793 v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset);
3794 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
3795 desc->count_offset);
3796 if (v1 != v2)
3797 pr_warning("%s: mismatch in element count\n", __func__);
3798
3799 v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset);
3800 v2 = bpf_prog_info_read_offset_u32(&info_linear->info,
3801 desc->size_offset);
3802 if (v1 != v2)
3803 pr_warning("%s: mismatch in rec size\n", __func__);
3804 }
3805
3806 /* step 7: update info_len and data_len */
3807 info_linear->info_len = sizeof(struct bpf_prog_info);
3808 info_linear->data_len = data_len;
3809
3810 return info_linear;
3811}
3812
3813void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear)
3814{
3815 int i;
3816
3817 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
3818 struct bpf_prog_info_array_desc *desc;
3819 __u64 addr, offs;
3820
3821 if ((info_linear->arrays & (1UL << i)) == 0)
3822 continue;
3823
3824 desc = bpf_prog_info_array_desc + i;
3825 addr = bpf_prog_info_read_offset_u64(&info_linear->info,
3826 desc->array_offset);
3827 offs = addr - ptr_to_u64(info_linear->data);
3828 bpf_prog_info_set_offset_u64(&info_linear->info,
3829 desc->array_offset, offs);
3830 }
3831}
3832
3833void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear)
3834{
3835 int i;
3836
3837 for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) {
3838 struct bpf_prog_info_array_desc *desc;
3839 __u64 addr, offs;
3840
3841 if ((info_linear->arrays & (1UL << i)) == 0)
3842 continue;
3843
3844 desc = bpf_prog_info_array_desc + i;
3845 offs = bpf_prog_info_read_offset_u64(&info_linear->info,
3846 desc->array_offset);
3847 addr = offs + ptr_to_u64(info_linear->data);
3848 bpf_prog_info_set_offset_u64(&info_linear->info,
3849 desc->array_offset, addr);
3850 }
3851}
Linux 6.x block layer and io_uring tree(s)