Commit | Line | Data |
---|---|---|
1bc38b8f | 1 | // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) |
6061a3d6 | 2 | |
1b76c13e WN |
3 | /* |
4 | * Common eBPF ELF object loading operations. | |
5 | * | |
6 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
7 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
8 | * Copyright (C) 2015 Huawei Inc. | |
f367540c | 9 | * Copyright (C) 2017 Nicira, Inc. |
d859900c | 10 | * Copyright (C) 2019 Isovalent, Inc. |
1b76c13e WN |
11 | */ |
12 | ||
b4269954 | 13 | #ifndef _GNU_SOURCE |
531b014e | 14 | #define _GNU_SOURCE |
b4269954 | 15 | #endif |
1b76c13e | 16 | #include <stdlib.h> |
b3f59d66 WN |
17 | #include <stdio.h> |
18 | #include <stdarg.h> | |
f367540c | 19 | #include <libgen.h> |
34090915 | 20 | #include <inttypes.h> |
b3f59d66 | 21 | #include <string.h> |
1b76c13e | 22 | #include <unistd.h> |
1a5e3fb1 WN |
23 | #include <fcntl.h> |
24 | #include <errno.h> | |
1b76c13e | 25 | #include <asm/unistd.h> |
e28ff1a8 | 26 | #include <linux/err.h> |
cb1e5e96 | 27 | #include <linux/kernel.h> |
1b76c13e | 28 | #include <linux/bpf.h> |
38d5d3b3 | 29 | #include <linux/btf.h> |
47eff617 | 30 | #include <linux/filter.h> |
9a208eff | 31 | #include <linux/list.h> |
f367540c | 32 | #include <linux/limits.h> |
438363c0 | 33 | #include <linux/perf_event.h> |
a64af0ef | 34 | #include <linux/ring_buffer.h> |
fb84b822 | 35 | #include <sys/epoll.h> |
63f2f5ee | 36 | #include <sys/ioctl.h> |
fb84b822 | 37 | #include <sys/mman.h> |
f367540c JS |
38 | #include <sys/stat.h> |
39 | #include <sys/types.h> | |
40 | #include <sys/vfs.h> | |
531b014e | 41 | #include <tools/libc_compat.h> |
1a5e3fb1 WN |
42 | #include <libelf.h> |
43 | #include <gelf.h> | |
1b76c13e WN |
44 | |
45 | #include "libbpf.h" | |
52d3352e | 46 | #include "bpf.h" |
8a138aed | 47 | #include "btf.h" |
6d41907c | 48 | #include "str_error.h" |
d7c4b398 | 49 | #include "libbpf_internal.h" |
b3f59d66 | 50 | |
9b16137a WN |
51 | #ifndef EM_BPF |
52 | #define EM_BPF 247 | |
53 | #endif | |
54 | ||
f367540c JS |
55 | #ifndef BPF_FS_MAGIC |
56 | #define BPF_FS_MAGIC 0xcafe4a11 | |
57 | #endif | |
58 | ||
ff466b58 AI |
59 | /* vsprintf() in __base_pr() uses nonliteral format string. It may break |
60 | * compilation if user enables corresponding warning. Disable it explicitly. | |
61 | */ | |
62 | #pragma GCC diagnostic ignored "-Wformat-nonliteral" | |
63 | ||
b3f59d66 WN |
64 | #define __printf(a, b) __attribute__((format(printf, a, b))) |
65 | ||
a8a1f7d0 SF |
66 | static int __base_pr(enum libbpf_print_level level, const char *format, |
67 | va_list args) | |
b3f59d66 | 68 | { |
6f1ae8b6 YS |
69 | if (level == LIBBPF_DEBUG) |
70 | return 0; | |
71 | ||
a8a1f7d0 | 72 | return vfprintf(stderr, format, args); |
b3f59d66 WN |
73 | } |
74 | ||
a8a1f7d0 | 75 | static libbpf_print_fn_t __libbpf_pr = __base_pr; |
b3f59d66 | 76 | |
e87fd8ba | 77 | libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) |
b3f59d66 | 78 | { |
e87fd8ba AN |
79 | libbpf_print_fn_t old_print_fn = __libbpf_pr; |
80 | ||
6f1ae8b6 | 81 | __libbpf_pr = fn; |
e87fd8ba | 82 | return old_print_fn; |
b3f59d66 | 83 | } |
1a5e3fb1 | 84 | |
8461ef8b YS |
85 | __printf(2, 3) |
86 | void libbpf_print(enum libbpf_print_level level, const char *format, ...) | |
87 | { | |
88 | va_list args; | |
89 | ||
6f1ae8b6 YS |
90 | if (!__libbpf_pr) |
91 | return; | |
92 | ||
8461ef8b | 93 | va_start(args, format); |
6f1ae8b6 | 94 | __libbpf_pr(level, format, args); |
8461ef8b YS |
95 | va_end(args); |
96 | } | |
97 | ||
6371ca3b WN |
98 | #define STRERR_BUFSIZE 128 |
99 | ||
6371ca3b WN |
100 | #define CHECK_ERR(action, err, out) do { \ |
101 | err = action; \ | |
102 | if (err) \ | |
103 | goto out; \ | |
104 | } while(0) | |
105 | ||
106 | ||
1a5e3fb1 WN |
107 | /* Copied from tools/perf/util/util.h */ |
108 | #ifndef zfree | |
109 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
110 | #endif | |
111 | ||
112 | #ifndef zclose | |
113 | # define zclose(fd) ({ \ | |
114 | int ___err = 0; \ | |
115 | if ((fd) >= 0) \ | |
116 | ___err = close((fd)); \ | |
117 | fd = -1; \ | |
118 | ___err; }) | |
119 | #endif | |
120 | ||
121 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
122 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
123 | #else | |
124 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
125 | #endif | |
126 | ||
34be1646 SL |
127 | static inline __u64 ptr_to_u64(const void *ptr) |
128 | { | |
129 | return (__u64) (unsigned long) ptr; | |
130 | } | |
131 | ||
47eff617 SF |
132 | struct bpf_capabilities { |
133 | /* v4.14: kernel support for program & map names. */ | |
134 | __u32 name:1; | |
8837fe5d DB |
135 | /* v5.2: kernel support for global data sections. */ |
136 | __u32 global_data:1; | |
d7c4b398 AN |
137 | /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ |
138 | __u32 btf_func:1; | |
139 | /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ | |
140 | __u32 btf_datasec:1; | |
47eff617 SF |
141 | }; |
142 | ||
a5b8bd47 WN |
143 | /* |
144 | * bpf_prog should be a better name but it has been used in | |
145 | * linux/filter.h. | |
146 | */ | |
147 | struct bpf_program { | |
148 | /* Index in elf obj file, for relocation use. */ | |
149 | int idx; | |
88cda1c9 | 150 | char *name; |
f0307a7e | 151 | int prog_ifindex; |
a5b8bd47 | 152 | char *section_name; |
33a2c75c SF |
153 | /* section_name with / replaced by _; makes recursive pinning |
154 | * in bpf_object__pin_programs easier | |
155 | */ | |
156 | char *pin_name; | |
a5b8bd47 | 157 | struct bpf_insn *insns; |
48cca7e4 | 158 | size_t insns_cnt, main_prog_cnt; |
5f44e4c8 | 159 | enum bpf_prog_type type; |
34090915 | 160 | |
48cca7e4 AS |
161 | struct reloc_desc { |
162 | enum { | |
163 | RELO_LD64, | |
164 | RELO_CALL, | |
d859900c | 165 | RELO_DATA, |
48cca7e4 | 166 | } type; |
34090915 | 167 | int insn_idx; |
48cca7e4 AS |
168 | union { |
169 | int map_idx; | |
170 | int text_off; | |
171 | }; | |
34090915 WN |
172 | } *reloc_desc; |
173 | int nr_reloc; | |
da11b417 | 174 | int log_level; |
55cffde2 | 175 | |
b580563e WN |
176 | struct { |
177 | int nr; | |
178 | int *fds; | |
179 | } instances; | |
180 | bpf_program_prep_t preprocessor; | |
aa9b1ac3 WN |
181 | |
182 | struct bpf_object *obj; | |
183 | void *priv; | |
184 | bpf_program_clear_priv_t clear_priv; | |
d7be143b AI |
185 | |
186 | enum bpf_attach_type expected_attach_type; | |
2993e051 YS |
187 | int btf_fd; |
188 | void *func_info; | |
189 | __u32 func_info_rec_size; | |
f0187f0b | 190 | __u32 func_info_cnt; |
47eff617 SF |
191 | |
192 | struct bpf_capabilities *caps; | |
3d650141 MKL |
193 | |
194 | void *line_info; | |
195 | __u32 line_info_rec_size; | |
196 | __u32 line_info_cnt; | |
04656198 | 197 | __u32 prog_flags; |
a5b8bd47 WN |
198 | }; |
199 | ||
d859900c DB |
200 | enum libbpf_map_type { |
201 | LIBBPF_MAP_UNSPEC, | |
202 | LIBBPF_MAP_DATA, | |
203 | LIBBPF_MAP_BSS, | |
204 | LIBBPF_MAP_RODATA, | |
205 | }; | |
206 | ||
207 | static const char * const libbpf_type_to_btf_name[] = { | |
208 | [LIBBPF_MAP_DATA] = ".data", | |
209 | [LIBBPF_MAP_BSS] = ".bss", | |
210 | [LIBBPF_MAP_RODATA] = ".rodata", | |
211 | }; | |
212 | ||
9d759a9b WN |
213 | struct bpf_map { |
214 | int fd; | |
561bbcca | 215 | char *name; |
db48814b AN |
216 | int sec_idx; |
217 | size_t sec_offset; | |
f0307a7e | 218 | int map_ifindex; |
addb9fc9 | 219 | int inner_map_fd; |
9d759a9b | 220 | struct bpf_map_def def; |
5b891af7 MKL |
221 | __u32 btf_key_type_id; |
222 | __u32 btf_value_type_id; | |
9d759a9b WN |
223 | void *priv; |
224 | bpf_map_clear_priv_t clear_priv; | |
d859900c DB |
225 | enum libbpf_map_type libbpf_type; |
226 | }; | |
227 | ||
228 | struct bpf_secdata { | |
229 | void *rodata; | |
230 | void *data; | |
9d759a9b WN |
231 | }; |
232 | ||
9a208eff WN |
233 | static LIST_HEAD(bpf_objects_list); |
234 | ||
1a5e3fb1 | 235 | struct bpf_object { |
d859900c | 236 | char name[BPF_OBJ_NAME_LEN]; |
cb1e5e96 | 237 | char license[64]; |
438363c0 | 238 | __u32 kern_version; |
0b3d1efa | 239 | |
a5b8bd47 WN |
240 | struct bpf_program *programs; |
241 | size_t nr_programs; | |
9d759a9b WN |
242 | struct bpf_map *maps; |
243 | size_t nr_maps; | |
bf829271 | 244 | size_t maps_cap; |
d859900c | 245 | struct bpf_secdata sections; |
9d759a9b | 246 | |
52d3352e | 247 | bool loaded; |
9a94f277 | 248 | bool has_pseudo_calls; |
a5b8bd47 | 249 | |
1a5e3fb1 WN |
250 | /* |
251 | * Information when doing elf related work. Only valid if fd | |
252 | * is valid. | |
253 | */ | |
254 | struct { | |
255 | int fd; | |
6c956392 WN |
256 | void *obj_buf; |
257 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
258 | Elf *elf; |
259 | GElf_Ehdr ehdr; | |
bec7d68c | 260 | Elf_Data *symbols; |
d859900c DB |
261 | Elf_Data *data; |
262 | Elf_Data *rodata; | |
263 | Elf_Data *bss; | |
77ba9a5b | 264 | size_t strtabidx; |
b62f06e8 WN |
265 | struct { |
266 | GElf_Shdr shdr; | |
267 | Elf_Data *data; | |
268 | } *reloc; | |
269 | int nr_reloc; | |
666810e8 | 270 | int maps_shndx; |
abd29c93 | 271 | int btf_maps_shndx; |
48cca7e4 | 272 | int text_shndx; |
d859900c DB |
273 | int data_shndx; |
274 | int rodata_shndx; | |
275 | int bss_shndx; | |
1a5e3fb1 | 276 | } efile; |
9a208eff WN |
277 | /* |
278 | * All loaded bpf_object is linked in a list, which is | |
279 | * hidden to caller. bpf_objects__<func> handlers deal with | |
280 | * all objects. | |
281 | */ | |
282 | struct list_head list; | |
10931d24 | 283 | |
8a138aed | 284 | struct btf *btf; |
2993e051 | 285 | struct btf_ext *btf_ext; |
8a138aed | 286 | |
10931d24 WN |
287 | void *priv; |
288 | bpf_object_clear_priv_t clear_priv; | |
289 | ||
47eff617 SF |
290 | struct bpf_capabilities caps; |
291 | ||
1a5e3fb1 WN |
292 | char path[]; |
293 | }; | |
294 | #define obj_elf_valid(o) ((o)->efile.elf) | |
295 | ||
29cd77f4 | 296 | void bpf_program__unload(struct bpf_program *prog) |
55cffde2 | 297 | { |
b580563e WN |
298 | int i; |
299 | ||
55cffde2 WN |
300 | if (!prog) |
301 | return; | |
302 | ||
b580563e WN |
303 | /* |
304 | * If the object is opened but the program was never loaded, | |
305 | * it is possible that prog->instances.nr == -1. | |
306 | */ | |
307 | if (prog->instances.nr > 0) { | |
308 | for (i = 0; i < prog->instances.nr; i++) | |
309 | zclose(prog->instances.fds[i]); | |
310 | } else if (prog->instances.nr != -1) { | |
311 | pr_warning("Internal error: instances.nr is %d\n", | |
312 | prog->instances.nr); | |
313 | } | |
314 | ||
315 | prog->instances.nr = -1; | |
316 | zfree(&prog->instances.fds); | |
2993e051 YS |
317 | |
318 | zclose(prog->btf_fd); | |
319 | zfree(&prog->func_info); | |
07a09d1b | 320 | zfree(&prog->line_info); |
55cffde2 WN |
321 | } |
322 | ||
a5b8bd47 WN |
323 | static void bpf_program__exit(struct bpf_program *prog) |
324 | { | |
325 | if (!prog) | |
326 | return; | |
327 | ||
aa9b1ac3 WN |
328 | if (prog->clear_priv) |
329 | prog->clear_priv(prog, prog->priv); | |
330 | ||
331 | prog->priv = NULL; | |
332 | prog->clear_priv = NULL; | |
333 | ||
55cffde2 | 334 | bpf_program__unload(prog); |
88cda1c9 | 335 | zfree(&prog->name); |
a5b8bd47 | 336 | zfree(&prog->section_name); |
33a2c75c | 337 | zfree(&prog->pin_name); |
a5b8bd47 | 338 | zfree(&prog->insns); |
34090915 WN |
339 | zfree(&prog->reloc_desc); |
340 | ||
341 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
342 | prog->insns_cnt = 0; |
343 | prog->idx = -1; | |
344 | } | |
345 | ||
33a2c75c SF |
346 | static char *__bpf_program__pin_name(struct bpf_program *prog) |
347 | { | |
348 | char *name, *p; | |
349 | ||
350 | name = p = strdup(prog->section_name); | |
351 | while ((p = strchr(p, '/'))) | |
352 | *p = '_'; | |
353 | ||
354 | return name; | |
355 | } | |
356 | ||
a5b8bd47 | 357 | static int |
88cda1c9 MKL |
358 | bpf_program__init(void *data, size_t size, char *section_name, int idx, |
359 | struct bpf_program *prog) | |
a5b8bd47 | 360 | { |
8ca990ce AN |
361 | const size_t bpf_insn_sz = sizeof(struct bpf_insn); |
362 | ||
363 | if (size == 0 || size % bpf_insn_sz) { | |
364 | pr_warning("corrupted section '%s', size: %zu\n", | |
365 | section_name, size); | |
a5b8bd47 WN |
366 | return -EINVAL; |
367 | } | |
368 | ||
1ad9cbb8 | 369 | memset(prog, 0, sizeof(*prog)); |
a5b8bd47 | 370 | |
88cda1c9 | 371 | prog->section_name = strdup(section_name); |
a5b8bd47 | 372 | if (!prog->section_name) { |
077c066a JDB |
373 | pr_warning("failed to alloc name for prog under section(%d) %s\n", |
374 | idx, section_name); | |
a5b8bd47 WN |
375 | goto errout; |
376 | } | |
377 | ||
33a2c75c SF |
378 | prog->pin_name = __bpf_program__pin_name(prog); |
379 | if (!prog->pin_name) { | |
380 | pr_warning("failed to alloc pin name for prog under section(%d) %s\n", | |
381 | idx, section_name); | |
382 | goto errout; | |
383 | } | |
384 | ||
a5b8bd47 WN |
385 | prog->insns = malloc(size); |
386 | if (!prog->insns) { | |
88cda1c9 MKL |
387 | pr_warning("failed to alloc insns for prog under section %s\n", |
388 | section_name); | |
a5b8bd47 WN |
389 | goto errout; |
390 | } | |
8ca990ce AN |
391 | prog->insns_cnt = size / bpf_insn_sz; |
392 | memcpy(prog->insns, data, size); | |
a5b8bd47 | 393 | prog->idx = idx; |
b580563e WN |
394 | prog->instances.fds = NULL; |
395 | prog->instances.nr = -1; | |
47ae7e3d | 396 | prog->type = BPF_PROG_TYPE_UNSPEC; |
2993e051 | 397 | prog->btf_fd = -1; |
a5b8bd47 WN |
398 | |
399 | return 0; | |
400 | errout: | |
401 | bpf_program__exit(prog); | |
402 | return -ENOMEM; | |
403 | } | |
404 | ||
405 | static int | |
406 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
88cda1c9 | 407 | char *section_name, int idx) |
a5b8bd47 WN |
408 | { |
409 | struct bpf_program prog, *progs; | |
410 | int nr_progs, err; | |
411 | ||
88cda1c9 | 412 | err = bpf_program__init(data, size, section_name, idx, &prog); |
a5b8bd47 WN |
413 | if (err) |
414 | return err; | |
415 | ||
47eff617 | 416 | prog.caps = &obj->caps; |
a5b8bd47 WN |
417 | progs = obj->programs; |
418 | nr_progs = obj->nr_programs; | |
419 | ||
531b014e | 420 | progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); |
a5b8bd47 WN |
421 | if (!progs) { |
422 | /* | |
423 | * In this case the original obj->programs | |
424 | * is still valid, so don't need special treat for | |
425 | * bpf_close_object(). | |
426 | */ | |
88cda1c9 MKL |
427 | pr_warning("failed to alloc a new program under section '%s'\n", |
428 | section_name); | |
a5b8bd47 WN |
429 | bpf_program__exit(&prog); |
430 | return -ENOMEM; | |
431 | } | |
432 | ||
433 | pr_debug("found program %s\n", prog.section_name); | |
434 | obj->programs = progs; | |
435 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 436 | prog.obj = obj; |
a5b8bd47 WN |
437 | progs[nr_progs] = prog; |
438 | return 0; | |
439 | } | |
440 | ||
88cda1c9 MKL |
441 | static int |
442 | bpf_object__init_prog_names(struct bpf_object *obj) | |
443 | { | |
444 | Elf_Data *symbols = obj->efile.symbols; | |
445 | struct bpf_program *prog; | |
446 | size_t pi, si; | |
447 | ||
448 | for (pi = 0; pi < obj->nr_programs; pi++) { | |
48cca7e4 | 449 | const char *name = NULL; |
88cda1c9 MKL |
450 | |
451 | prog = &obj->programs[pi]; | |
452 | ||
453 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; | |
454 | si++) { | |
455 | GElf_Sym sym; | |
456 | ||
457 | if (!gelf_getsym(symbols, si, &sym)) | |
458 | continue; | |
459 | if (sym.st_shndx != prog->idx) | |
460 | continue; | |
fe4d44b2 RG |
461 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) |
462 | continue; | |
88cda1c9 MKL |
463 | |
464 | name = elf_strptr(obj->efile.elf, | |
465 | obj->efile.strtabidx, | |
466 | sym.st_name); | |
467 | if (!name) { | |
468 | pr_warning("failed to get sym name string for prog %s\n", | |
469 | prog->section_name); | |
470 | return -LIBBPF_ERRNO__LIBELF; | |
471 | } | |
472 | } | |
473 | ||
9a94f277 JK |
474 | if (!name && prog->idx == obj->efile.text_shndx) |
475 | name = ".text"; | |
476 | ||
88cda1c9 MKL |
477 | if (!name) { |
478 | pr_warning("failed to find sym for prog %s\n", | |
479 | prog->section_name); | |
480 | return -EINVAL; | |
481 | } | |
9a94f277 | 482 | |
88cda1c9 MKL |
483 | prog->name = strdup(name); |
484 | if (!prog->name) { | |
485 | pr_warning("failed to allocate memory for prog sym %s\n", | |
486 | name); | |
487 | return -ENOMEM; | |
488 | } | |
489 | } | |
490 | ||
491 | return 0; | |
492 | } | |
493 | ||
6c956392 WN |
494 | static struct bpf_object *bpf_object__new(const char *path, |
495 | void *obj_buf, | |
496 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
497 | { |
498 | struct bpf_object *obj; | |
d859900c | 499 | char *end; |
1a5e3fb1 WN |
500 | |
501 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
502 | if (!obj) { | |
503 | pr_warning("alloc memory failed for %s\n", path); | |
6371ca3b | 504 | return ERR_PTR(-ENOMEM); |
1a5e3fb1 WN |
505 | } |
506 | ||
507 | strcpy(obj->path, path); | |
d859900c | 508 | /* Using basename() GNU version which doesn't modify arg. */ |
399dc65e | 509 | strncpy(obj->name, basename((void *)path), sizeof(obj->name) - 1); |
d859900c DB |
510 | end = strchr(obj->name, '.'); |
511 | if (end) | |
512 | *end = 0; | |
6c956392 | 513 | |
d859900c | 514 | obj->efile.fd = -1; |
6c956392 | 515 | /* |
76e1022b | 516 | * Caller of this function should also call |
6c956392 WN |
517 | * bpf_object__elf_finish() after data collection to return |
518 | * obj_buf to user. If not, we should duplicate the buffer to | |
519 | * avoid user freeing them before elf finish. | |
520 | */ | |
521 | obj->efile.obj_buf = obj_buf; | |
522 | obj->efile.obj_buf_sz = obj_buf_sz; | |
666810e8 | 523 | obj->efile.maps_shndx = -1; |
abd29c93 | 524 | obj->efile.btf_maps_shndx = -1; |
d859900c DB |
525 | obj->efile.data_shndx = -1; |
526 | obj->efile.rodata_shndx = -1; | |
527 | obj->efile.bss_shndx = -1; | |
6c956392 | 528 | |
52d3352e | 529 | obj->loaded = false; |
9a208eff WN |
530 | |
531 | INIT_LIST_HEAD(&obj->list); | |
532 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
533 | return obj; |
534 | } | |
535 | ||
536 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
537 | { | |
538 | if (!obj_elf_valid(obj)) | |
539 | return; | |
540 | ||
541 | if (obj->efile.elf) { | |
542 | elf_end(obj->efile.elf); | |
543 | obj->efile.elf = NULL; | |
544 | } | |
bec7d68c | 545 | obj->efile.symbols = NULL; |
d859900c DB |
546 | obj->efile.data = NULL; |
547 | obj->efile.rodata = NULL; | |
548 | obj->efile.bss = NULL; | |
b62f06e8 WN |
549 | |
550 | zfree(&obj->efile.reloc); | |
551 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 552 | zclose(obj->efile.fd); |
6c956392 WN |
553 | obj->efile.obj_buf = NULL; |
554 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
555 | } |
556 | ||
557 | static int bpf_object__elf_init(struct bpf_object *obj) | |
558 | { | |
559 | int err = 0; | |
560 | GElf_Ehdr *ep; | |
561 | ||
562 | if (obj_elf_valid(obj)) { | |
563 | pr_warning("elf init: internal error\n"); | |
6371ca3b | 564 | return -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
565 | } |
566 | ||
6c956392 WN |
567 | if (obj->efile.obj_buf_sz > 0) { |
568 | /* | |
569 | * obj_buf should have been validated by | |
570 | * bpf_object__open_buffer(). | |
571 | */ | |
572 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
573 | obj->efile.obj_buf_sz); | |
574 | } else { | |
575 | obj->efile.fd = open(obj->path, O_RDONLY); | |
576 | if (obj->efile.fd < 0) { | |
be5c5d4e | 577 | char errmsg[STRERR_BUFSIZE], *cp; |
1ce6a9fc | 578 | |
be5c5d4e AN |
579 | err = -errno; |
580 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
1ce6a9fc | 581 | pr_warning("failed to open %s: %s\n", obj->path, cp); |
be5c5d4e | 582 | return err; |
6c956392 WN |
583 | } |
584 | ||
585 | obj->efile.elf = elf_begin(obj->efile.fd, | |
76e1022b | 586 | LIBBPF_ELF_C_READ_MMAP, NULL); |
1a5e3fb1 WN |
587 | } |
588 | ||
1a5e3fb1 | 589 | if (!obj->efile.elf) { |
399dc65e | 590 | pr_warning("failed to open %s as ELF file\n", obj->path); |
6371ca3b | 591 | err = -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
592 | goto errout; |
593 | } | |
594 | ||
595 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
399dc65e | 596 | pr_warning("failed to get EHDR from %s\n", obj->path); |
6371ca3b | 597 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
598 | goto errout; |
599 | } | |
600 | ep = &obj->efile.ehdr; | |
601 | ||
9b16137a | 602 | /* Old LLVM set e_machine to EM_NONE */ |
76e1022b AN |
603 | if (ep->e_type != ET_REL || |
604 | (ep->e_machine && ep->e_machine != EM_BPF)) { | |
605 | pr_warning("%s is not an eBPF object file\n", obj->path); | |
6371ca3b | 606 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
607 | goto errout; |
608 | } | |
609 | ||
610 | return 0; | |
611 | errout: | |
612 | bpf_object__elf_finish(obj); | |
613 | return err; | |
614 | } | |
615 | ||
12ef5634 | 616 | static int bpf_object__check_endianness(struct bpf_object *obj) |
cc4228d5 | 617 | { |
12ef5634 AN |
618 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ |
619 | if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) | |
620 | return 0; | |
621 | #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ | |
622 | if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) | |
623 | return 0; | |
624 | #else | |
625 | # error "Unrecognized __BYTE_ORDER__" | |
626 | #endif | |
627 | pr_warning("endianness mismatch.\n"); | |
6371ca3b | 628 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
629 | } |
630 | ||
cb1e5e96 | 631 | static int |
399dc65e | 632 | bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) |
cb1e5e96 | 633 | { |
399dc65e | 634 | memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); |
cb1e5e96 WN |
635 | pr_debug("license of %s is %s\n", obj->path, obj->license); |
636 | return 0; | |
637 | } | |
638 | ||
639 | static int | |
399dc65e | 640 | bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) |
cb1e5e96 | 641 | { |
438363c0 | 642 | __u32 kver; |
cb1e5e96 WN |
643 | |
644 | if (size != sizeof(kver)) { | |
645 | pr_warning("invalid kver section in %s\n", obj->path); | |
6371ca3b | 646 | return -LIBBPF_ERRNO__FORMAT; |
cb1e5e96 WN |
647 | } |
648 | memcpy(&kver, data, sizeof(kver)); | |
649 | obj->kern_version = kver; | |
399dc65e | 650 | pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); |
cb1e5e96 WN |
651 | return 0; |
652 | } | |
653 | ||
4708bbda EL |
654 | static int compare_bpf_map(const void *_a, const void *_b) |
655 | { | |
656 | const struct bpf_map *a = _a; | |
657 | const struct bpf_map *b = _b; | |
9d759a9b | 658 | |
db48814b AN |
659 | if (a->sec_idx != b->sec_idx) |
660 | return a->sec_idx - b->sec_idx; | |
661 | return a->sec_offset - b->sec_offset; | |
0b3d1efa WN |
662 | } |
663 | ||
addb9fc9 NS |
664 | static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) |
665 | { | |
666 | if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || | |
667 | type == BPF_MAP_TYPE_HASH_OF_MAPS) | |
668 | return true; | |
669 | return false; | |
670 | } | |
671 | ||
1713d68b DB |
672 | static int bpf_object_search_section_size(const struct bpf_object *obj, |
673 | const char *name, size_t *d_size) | |
674 | { | |
675 | const GElf_Ehdr *ep = &obj->efile.ehdr; | |
676 | Elf *elf = obj->efile.elf; | |
677 | Elf_Scn *scn = NULL; | |
678 | int idx = 0; | |
679 | ||
680 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
681 | const char *sec_name; | |
682 | Elf_Data *data; | |
683 | GElf_Shdr sh; | |
684 | ||
685 | idx++; | |
686 | if (gelf_getshdr(scn, &sh) != &sh) { | |
687 | pr_warning("failed to get section(%d) header from %s\n", | |
688 | idx, obj->path); | |
689 | return -EIO; | |
690 | } | |
691 | ||
692 | sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
693 | if (!sec_name) { | |
694 | pr_warning("failed to get section(%d) name from %s\n", | |
695 | idx, obj->path); | |
696 | return -EIO; | |
697 | } | |
698 | ||
699 | if (strcmp(name, sec_name)) | |
700 | continue; | |
701 | ||
702 | data = elf_getdata(scn, 0); | |
703 | if (!data) { | |
704 | pr_warning("failed to get section(%d) data from %s(%s)\n", | |
705 | idx, name, obj->path); | |
706 | return -EIO; | |
707 | } | |
708 | ||
709 | *d_size = data->d_size; | |
710 | return 0; | |
711 | } | |
712 | ||
713 | return -ENOENT; | |
714 | } | |
715 | ||
716 | int bpf_object__section_size(const struct bpf_object *obj, const char *name, | |
717 | __u32 *size) | |
718 | { | |
719 | int ret = -ENOENT; | |
720 | size_t d_size; | |
721 | ||
722 | *size = 0; | |
723 | if (!name) { | |
724 | return -EINVAL; | |
725 | } else if (!strcmp(name, ".data")) { | |
726 | if (obj->efile.data) | |
727 | *size = obj->efile.data->d_size; | |
728 | } else if (!strcmp(name, ".bss")) { | |
729 | if (obj->efile.bss) | |
730 | *size = obj->efile.bss->d_size; | |
731 | } else if (!strcmp(name, ".rodata")) { | |
732 | if (obj->efile.rodata) | |
733 | *size = obj->efile.rodata->d_size; | |
734 | } else { | |
735 | ret = bpf_object_search_section_size(obj, name, &d_size); | |
736 | if (!ret) | |
737 | *size = d_size; | |
738 | } | |
739 | ||
740 | return *size ? 0 : ret; | |
741 | } | |
742 | ||
743 | int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, | |
744 | __u32 *off) | |
745 | { | |
746 | Elf_Data *symbols = obj->efile.symbols; | |
747 | const char *sname; | |
748 | size_t si; | |
749 | ||
750 | if (!name || !off) | |
751 | return -EINVAL; | |
752 | ||
753 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { | |
754 | GElf_Sym sym; | |
755 | ||
756 | if (!gelf_getsym(symbols, si, &sym)) | |
757 | continue; | |
758 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || | |
759 | GELF_ST_TYPE(sym.st_info) != STT_OBJECT) | |
760 | continue; | |
761 | ||
762 | sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
763 | sym.st_name); | |
764 | if (!sname) { | |
765 | pr_warning("failed to get sym name string for var %s\n", | |
766 | name); | |
767 | return -EIO; | |
768 | } | |
769 | if (strcmp(name, sname) == 0) { | |
770 | *off = sym.st_value; | |
771 | return 0; | |
772 | } | |
773 | } | |
774 | ||
775 | return -ENOENT; | |
776 | } | |
777 | ||
bf829271 | 778 | static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) |
d859900c | 779 | { |
bf829271 AN |
780 | struct bpf_map *new_maps; |
781 | size_t new_cap; | |
782 | int i; | |
783 | ||
784 | if (obj->nr_maps < obj->maps_cap) | |
785 | return &obj->maps[obj->nr_maps++]; | |
786 | ||
95064979 | 787 | new_cap = max((size_t)4, obj->maps_cap * 3 / 2); |
bf829271 AN |
788 | new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); |
789 | if (!new_maps) { | |
790 | pr_warning("alloc maps for object failed\n"); | |
791 | return ERR_PTR(-ENOMEM); | |
792 | } | |
793 | ||
794 | obj->maps_cap = new_cap; | |
795 | obj->maps = new_maps; | |
796 | ||
797 | /* zero out new maps */ | |
798 | memset(obj->maps + obj->nr_maps, 0, | |
799 | (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); | |
800 | /* | |
801 | * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) | |
802 | * when failure (zclose won't close negative fd)). | |
803 | */ | |
804 | for (i = obj->nr_maps; i < obj->maps_cap; i++) { | |
805 | obj->maps[i].fd = -1; | |
806 | obj->maps[i].inner_map_fd = -1; | |
807 | } | |
808 | ||
809 | return &obj->maps[obj->nr_maps++]; | |
d859900c DB |
810 | } |
811 | ||
812 | static int | |
bf829271 | 813 | bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, |
db48814b | 814 | int sec_idx, Elf_Data *data, void **data_buff) |
d859900c | 815 | { |
d859900c | 816 | char map_name[BPF_OBJ_NAME_LEN]; |
bf829271 AN |
817 | struct bpf_map_def *def; |
818 | struct bpf_map *map; | |
819 | ||
820 | map = bpf_object__add_map(obj); | |
821 | if (IS_ERR(map)) | |
822 | return PTR_ERR(map); | |
d859900c DB |
823 | |
824 | map->libbpf_type = type; | |
db48814b AN |
825 | map->sec_idx = sec_idx; |
826 | map->sec_offset = 0; | |
d859900c DB |
827 | snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, |
828 | libbpf_type_to_btf_name[type]); | |
829 | map->name = strdup(map_name); | |
830 | if (!map->name) { | |
831 | pr_warning("failed to alloc map name\n"); | |
832 | return -ENOMEM; | |
833 | } | |
db48814b AN |
834 | pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n", |
835 | map_name, map->sec_idx, map->sec_offset); | |
d859900c | 836 | |
bf829271 | 837 | def = &map->def; |
d859900c DB |
838 | def->type = BPF_MAP_TYPE_ARRAY; |
839 | def->key_size = sizeof(int); | |
840 | def->value_size = data->d_size; | |
841 | def->max_entries = 1; | |
399dc65e | 842 | def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0; |
d859900c DB |
843 | if (data_buff) { |
844 | *data_buff = malloc(data->d_size); | |
845 | if (!*data_buff) { | |
846 | zfree(&map->name); | |
847 | pr_warning("failed to alloc map content buffer\n"); | |
848 | return -ENOMEM; | |
849 | } | |
850 | memcpy(*data_buff, data->d_buf, data->d_size); | |
851 | } | |
852 | ||
e1d1dc46 | 853 | pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); |
d859900c DB |
854 | return 0; |
855 | } | |
856 | ||
bf829271 AN |
857 | static int bpf_object__init_global_data_maps(struct bpf_object *obj) |
858 | { | |
859 | int err; | |
860 | ||
861 | if (!obj->caps.global_data) | |
862 | return 0; | |
863 | /* | |
864 | * Populate obj->maps with libbpf internal maps. | |
865 | */ | |
866 | if (obj->efile.data_shndx >= 0) { | |
867 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, | |
db48814b | 868 | obj->efile.data_shndx, |
bf829271 AN |
869 | obj->efile.data, |
870 | &obj->sections.data); | |
871 | if (err) | |
872 | return err; | |
873 | } | |
874 | if (obj->efile.rodata_shndx >= 0) { | |
875 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, | |
db48814b | 876 | obj->efile.rodata_shndx, |
bf829271 AN |
877 | obj->efile.rodata, |
878 | &obj->sections.rodata); | |
879 | if (err) | |
880 | return err; | |
881 | } | |
882 | if (obj->efile.bss_shndx >= 0) { | |
883 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, | |
db48814b | 884 | obj->efile.bss_shndx, |
bf829271 AN |
885 | obj->efile.bss, NULL); |
886 | if (err) | |
887 | return err; | |
888 | } | |
889 | return 0; | |
890 | } | |
891 | ||
892 | static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) | |
561bbcca | 893 | { |
561bbcca | 894 | Elf_Data *symbols = obj->efile.symbols; |
bf829271 | 895 | int i, map_def_sz = 0, nr_maps = 0, nr_syms; |
d859900c | 896 | Elf_Data *data = NULL; |
bf829271 AN |
897 | Elf_Scn *scn; |
898 | ||
899 | if (obj->efile.maps_shndx < 0) | |
900 | return 0; | |
561bbcca | 901 | |
4708bbda EL |
902 | if (!symbols) |
903 | return -EINVAL; | |
904 | ||
bf829271 AN |
905 | scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); |
906 | if (scn) | |
907 | data = elf_getdata(scn, NULL); | |
908 | if (!scn || !data) { | |
909 | pr_warning("failed to get Elf_Data from map section %d\n", | |
910 | obj->efile.maps_shndx); | |
911 | return -EINVAL; | |
4708bbda | 912 | } |
561bbcca | 913 | |
4708bbda EL |
914 | /* |
915 | * Count number of maps. Each map has a name. | |
916 | * Array of maps is not supported: only the first element is | |
917 | * considered. | |
918 | * | |
919 | * TODO: Detect array of map and report error. | |
920 | */ | |
bf829271 AN |
921 | nr_syms = symbols->d_size / sizeof(GElf_Sym); |
922 | for (i = 0; i < nr_syms; i++) { | |
561bbcca | 923 | GElf_Sym sym; |
4708bbda EL |
924 | |
925 | if (!gelf_getsym(symbols, i, &sym)) | |
926 | continue; | |
927 | if (sym.st_shndx != obj->efile.maps_shndx) | |
928 | continue; | |
929 | nr_maps++; | |
930 | } | |
b13c5c14 | 931 | /* Assume equally sized map definitions */ |
bf829271 AN |
932 | pr_debug("maps in %s: %d maps in %zd bytes\n", |
933 | obj->path, nr_maps, data->d_size); | |
934 | ||
935 | map_def_sz = data->d_size / nr_maps; | |
936 | if (!data->d_size || (data->d_size % nr_maps) != 0) { | |
937 | pr_warning("unable to determine map definition size " | |
938 | "section %s, %d maps in %zd bytes\n", | |
939 | obj->path, nr_maps, data->d_size); | |
940 | return -EINVAL; | |
addb9fc9 | 941 | } |
4708bbda | 942 | |
bf829271 AN |
943 | /* Fill obj->maps using data in "maps" section. */ |
944 | for (i = 0; i < nr_syms; i++) { | |
4708bbda | 945 | GElf_Sym sym; |
561bbcca | 946 | const char *map_name; |
4708bbda | 947 | struct bpf_map_def *def; |
bf829271 | 948 | struct bpf_map *map; |
561bbcca WN |
949 | |
950 | if (!gelf_getsym(symbols, i, &sym)) | |
951 | continue; | |
666810e8 | 952 | if (sym.st_shndx != obj->efile.maps_shndx) |
561bbcca WN |
953 | continue; |
954 | ||
bf829271 AN |
955 | map = bpf_object__add_map(obj); |
956 | if (IS_ERR(map)) | |
957 | return PTR_ERR(map); | |
958 | ||
959 | map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
561bbcca | 960 | sym.st_name); |
c51829bb AN |
961 | if (!map_name) { |
962 | pr_warning("failed to get map #%d name sym string for obj %s\n", | |
bf829271 | 963 | i, obj->path); |
c51829bb AN |
964 | return -LIBBPF_ERRNO__FORMAT; |
965 | } | |
d859900c | 966 | |
bf829271 | 967 | map->libbpf_type = LIBBPF_MAP_UNSPEC; |
db48814b AN |
968 | map->sec_idx = sym.st_shndx; |
969 | map->sec_offset = sym.st_value; | |
970 | pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", | |
971 | map_name, map->sec_idx, map->sec_offset); | |
b13c5c14 | 972 | if (sym.st_value + map_def_sz > data->d_size) { |
4708bbda EL |
973 | pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", |
974 | obj->path, map_name); | |
975 | return -EINVAL; | |
561bbcca | 976 | } |
4708bbda | 977 | |
bf829271 AN |
978 | map->name = strdup(map_name); |
979 | if (!map->name) { | |
973170e6 WN |
980 | pr_warning("failed to alloc map name\n"); |
981 | return -ENOMEM; | |
982 | } | |
bf829271 | 983 | pr_debug("map %d is \"%s\"\n", i, map->name); |
4708bbda | 984 | def = (struct bpf_map_def *)(data->d_buf + sym.st_value); |
b13c5c14 CG |
985 | /* |
986 | * If the definition of the map in the object file fits in | |
987 | * bpf_map_def, copy it. Any extra fields in our version | |
988 | * of bpf_map_def will default to zero as a result of the | |
989 | * calloc above. | |
990 | */ | |
991 | if (map_def_sz <= sizeof(struct bpf_map_def)) { | |
bf829271 | 992 | memcpy(&map->def, def, map_def_sz); |
b13c5c14 CG |
993 | } else { |
994 | /* | |
995 | * Here the map structure being read is bigger than what | |
996 | * we expect, truncate if the excess bits are all zero. | |
997 | * If they are not zero, reject this map as | |
998 | * incompatible. | |
999 | */ | |
1000 | char *b; | |
1001 | for (b = ((char *)def) + sizeof(struct bpf_map_def); | |
1002 | b < ((char *)def) + map_def_sz; b++) { | |
1003 | if (*b != 0) { | |
1004 | pr_warning("maps section in %s: \"%s\" " | |
1005 | "has unrecognized, non-zero " | |
1006 | "options\n", | |
1007 | obj->path, map_name); | |
c034a177 JF |
1008 | if (strict) |
1009 | return -EINVAL; | |
b13c5c14 CG |
1010 | } |
1011 | } | |
bf829271 | 1012 | memcpy(&map->def, def, sizeof(struct bpf_map_def)); |
b13c5c14 | 1013 | } |
561bbcca | 1014 | } |
bf829271 AN |
1015 | return 0; |
1016 | } | |
4708bbda | 1017 | |
abd29c93 AN |
1018 | static const struct btf_type *skip_mods_and_typedefs(const struct btf *btf, |
1019 | __u32 id) | |
1020 | { | |
1021 | const struct btf_type *t = btf__type_by_id(btf, id); | |
8837fe5d | 1022 | |
abd29c93 AN |
1023 | while (true) { |
1024 | switch (BTF_INFO_KIND(t->info)) { | |
1025 | case BTF_KIND_VOLATILE: | |
1026 | case BTF_KIND_CONST: | |
1027 | case BTF_KIND_RESTRICT: | |
1028 | case BTF_KIND_TYPEDEF: | |
1029 | t = btf__type_by_id(btf, t->type); | |
1030 | break; | |
1031 | default: | |
1032 | return t; | |
1033 | } | |
1034 | } | |
1035 | } | |
1036 | ||
ef99b02b AN |
1037 | /* |
1038 | * Fetch integer attribute of BTF map definition. Such attributes are | |
1039 | * represented using a pointer to an array, in which dimensionality of array | |
1040 | * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; | |
1041 | * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF | |
1042 | * type definition, while using only sizeof(void *) space in ELF data section. | |
1043 | */ | |
1044 | static bool get_map_field_int(const char *map_name, const struct btf *btf, | |
abd29c93 | 1045 | const struct btf_type *def, |
ef99b02b | 1046 | const struct btf_member *m, __u32 *res) { |
abd29c93 AN |
1047 | const struct btf_type *t = skip_mods_and_typedefs(btf, m->type); |
1048 | const char *name = btf__name_by_offset(btf, m->name_off); | |
ef99b02b AN |
1049 | const struct btf_array *arr_info; |
1050 | const struct btf_type *arr_t; | |
abd29c93 | 1051 | |
b03bc685 | 1052 | if (!btf_is_ptr(t)) { |
ef99b02b | 1053 | pr_warning("map '%s': attr '%s': expected PTR, got %u.\n", |
b03bc685 | 1054 | map_name, name, btf_kind(t)); |
abd29c93 AN |
1055 | return false; |
1056 | } | |
ef99b02b AN |
1057 | |
1058 | arr_t = btf__type_by_id(btf, t->type); | |
1059 | if (!arr_t) { | |
1060 | pr_warning("map '%s': attr '%s': type [%u] not found.\n", | |
1061 | map_name, name, t->type); | |
abd29c93 AN |
1062 | return false; |
1063 | } | |
b03bc685 | 1064 | if (!btf_is_array(arr_t)) { |
ef99b02b | 1065 | pr_warning("map '%s': attr '%s': expected ARRAY, got %u.\n", |
b03bc685 | 1066 | map_name, name, btf_kind(arr_t)); |
abd29c93 AN |
1067 | return false; |
1068 | } | |
b03bc685 | 1069 | arr_info = btf_array(arr_t); |
ef99b02b | 1070 | *res = arr_info->nelems; |
abd29c93 AN |
1071 | return true; |
1072 | } | |
1073 | ||
1074 | static int bpf_object__init_user_btf_map(struct bpf_object *obj, | |
1075 | const struct btf_type *sec, | |
1076 | int var_idx, int sec_idx, | |
1077 | const Elf_Data *data, bool strict) | |
1078 | { | |
1079 | const struct btf_type *var, *def, *t; | |
1080 | const struct btf_var_secinfo *vi; | |
1081 | const struct btf_var *var_extra; | |
1082 | const struct btf_member *m; | |
abd29c93 AN |
1083 | const char *map_name; |
1084 | struct bpf_map *map; | |
1085 | int vlen, i; | |
1086 | ||
b03bc685 | 1087 | vi = btf_var_secinfos(sec) + var_idx; |
abd29c93 | 1088 | var = btf__type_by_id(obj->btf, vi->type); |
b03bc685 | 1089 | var_extra = btf_var(var); |
abd29c93 | 1090 | map_name = btf__name_by_offset(obj->btf, var->name_off); |
b03bc685 | 1091 | vlen = btf_vlen(var); |
abd29c93 AN |
1092 | |
1093 | if (map_name == NULL || map_name[0] == '\0') { | |
1094 | pr_warning("map #%d: empty name.\n", var_idx); | |
1095 | return -EINVAL; | |
1096 | } | |
1097 | if ((__u64)vi->offset + vi->size > data->d_size) { | |
1098 | pr_warning("map '%s' BTF data is corrupted.\n", map_name); | |
1099 | return -EINVAL; | |
1100 | } | |
b03bc685 | 1101 | if (!btf_is_var(var)) { |
abd29c93 | 1102 | pr_warning("map '%s': unexpected var kind %u.\n", |
b03bc685 | 1103 | map_name, btf_kind(var)); |
abd29c93 AN |
1104 | return -EINVAL; |
1105 | } | |
1106 | if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && | |
1107 | var_extra->linkage != BTF_VAR_STATIC) { | |
1108 | pr_warning("map '%s': unsupported var linkage %u.\n", | |
1109 | map_name, var_extra->linkage); | |
1110 | return -EOPNOTSUPP; | |
1111 | } | |
1112 | ||
1113 | def = skip_mods_and_typedefs(obj->btf, var->type); | |
b03bc685 | 1114 | if (!btf_is_struct(def)) { |
abd29c93 | 1115 | pr_warning("map '%s': unexpected def kind %u.\n", |
b03bc685 | 1116 | map_name, btf_kind(var)); |
abd29c93 AN |
1117 | return -EINVAL; |
1118 | } | |
1119 | if (def->size > vi->size) { | |
1120 | pr_warning("map '%s': invalid def size.\n", map_name); | |
1121 | return -EINVAL; | |
1122 | } | |
1123 | ||
1124 | map = bpf_object__add_map(obj); | |
1125 | if (IS_ERR(map)) | |
1126 | return PTR_ERR(map); | |
1127 | map->name = strdup(map_name); | |
1128 | if (!map->name) { | |
1129 | pr_warning("map '%s': failed to alloc map name.\n", map_name); | |
1130 | return -ENOMEM; | |
1131 | } | |
1132 | map->libbpf_type = LIBBPF_MAP_UNSPEC; | |
1133 | map->def.type = BPF_MAP_TYPE_UNSPEC; | |
1134 | map->sec_idx = sec_idx; | |
1135 | map->sec_offset = vi->offset; | |
1136 | pr_debug("map '%s': at sec_idx %d, offset %zu.\n", | |
1137 | map_name, map->sec_idx, map->sec_offset); | |
1138 | ||
b03bc685 AN |
1139 | vlen = btf_vlen(def); |
1140 | m = btf_members(def); | |
abd29c93 AN |
1141 | for (i = 0; i < vlen; i++, m++) { |
1142 | const char *name = btf__name_by_offset(obj->btf, m->name_off); | |
1143 | ||
1144 | if (!name) { | |
1145 | pr_warning("map '%s': invalid field #%d.\n", | |
1146 | map_name, i); | |
1147 | return -EINVAL; | |
1148 | } | |
1149 | if (strcmp(name, "type") == 0) { | |
1150 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1151 | &map->def.type)) |
abd29c93 AN |
1152 | return -EINVAL; |
1153 | pr_debug("map '%s': found type = %u.\n", | |
1154 | map_name, map->def.type); | |
1155 | } else if (strcmp(name, "max_entries") == 0) { | |
1156 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1157 | &map->def.max_entries)) |
abd29c93 AN |
1158 | return -EINVAL; |
1159 | pr_debug("map '%s': found max_entries = %u.\n", | |
1160 | map_name, map->def.max_entries); | |
1161 | } else if (strcmp(name, "map_flags") == 0) { | |
1162 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1163 | &map->def.map_flags)) |
abd29c93 AN |
1164 | return -EINVAL; |
1165 | pr_debug("map '%s': found map_flags = %u.\n", | |
1166 | map_name, map->def.map_flags); | |
1167 | } else if (strcmp(name, "key_size") == 0) { | |
1168 | __u32 sz; | |
1169 | ||
1170 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1171 | &sz)) |
abd29c93 AN |
1172 | return -EINVAL; |
1173 | pr_debug("map '%s': found key_size = %u.\n", | |
1174 | map_name, sz); | |
1175 | if (map->def.key_size && map->def.key_size != sz) { | |
900de4ac | 1176 | pr_warning("map '%s': conflicting key size %u != %u.\n", |
abd29c93 AN |
1177 | map_name, map->def.key_size, sz); |
1178 | return -EINVAL; | |
1179 | } | |
1180 | map->def.key_size = sz; | |
1181 | } else if (strcmp(name, "key") == 0) { | |
1182 | __s64 sz; | |
1183 | ||
1184 | t = btf__type_by_id(obj->btf, m->type); | |
1185 | if (!t) { | |
1186 | pr_warning("map '%s': key type [%d] not found.\n", | |
1187 | map_name, m->type); | |
1188 | return -EINVAL; | |
1189 | } | |
b03bc685 | 1190 | if (!btf_is_ptr(t)) { |
abd29c93 | 1191 | pr_warning("map '%s': key spec is not PTR: %u.\n", |
b03bc685 | 1192 | map_name, btf_kind(t)); |
abd29c93 AN |
1193 | return -EINVAL; |
1194 | } | |
1195 | sz = btf__resolve_size(obj->btf, t->type); | |
1196 | if (sz < 0) { | |
1197 | pr_warning("map '%s': can't determine key size for type [%u]: %lld.\n", | |
1198 | map_name, t->type, sz); | |
1199 | return sz; | |
1200 | } | |
1201 | pr_debug("map '%s': found key [%u], sz = %lld.\n", | |
1202 | map_name, t->type, sz); | |
1203 | if (map->def.key_size && map->def.key_size != sz) { | |
900de4ac | 1204 | pr_warning("map '%s': conflicting key size %u != %lld.\n", |
abd29c93 AN |
1205 | map_name, map->def.key_size, sz); |
1206 | return -EINVAL; | |
1207 | } | |
1208 | map->def.key_size = sz; | |
1209 | map->btf_key_type_id = t->type; | |
1210 | } else if (strcmp(name, "value_size") == 0) { | |
1211 | __u32 sz; | |
1212 | ||
1213 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1214 | &sz)) |
abd29c93 AN |
1215 | return -EINVAL; |
1216 | pr_debug("map '%s': found value_size = %u.\n", | |
1217 | map_name, sz); | |
1218 | if (map->def.value_size && map->def.value_size != sz) { | |
900de4ac | 1219 | pr_warning("map '%s': conflicting value size %u != %u.\n", |
abd29c93 AN |
1220 | map_name, map->def.value_size, sz); |
1221 | return -EINVAL; | |
1222 | } | |
1223 | map->def.value_size = sz; | |
1224 | } else if (strcmp(name, "value") == 0) { | |
1225 | __s64 sz; | |
1226 | ||
1227 | t = btf__type_by_id(obj->btf, m->type); | |
1228 | if (!t) { | |
1229 | pr_warning("map '%s': value type [%d] not found.\n", | |
1230 | map_name, m->type); | |
1231 | return -EINVAL; | |
1232 | } | |
b03bc685 | 1233 | if (!btf_is_ptr(t)) { |
abd29c93 | 1234 | pr_warning("map '%s': value spec is not PTR: %u.\n", |
b03bc685 | 1235 | map_name, btf_kind(t)); |
abd29c93 AN |
1236 | return -EINVAL; |
1237 | } | |
1238 | sz = btf__resolve_size(obj->btf, t->type); | |
1239 | if (sz < 0) { | |
1240 | pr_warning("map '%s': can't determine value size for type [%u]: %lld.\n", | |
1241 | map_name, t->type, sz); | |
1242 | return sz; | |
1243 | } | |
1244 | pr_debug("map '%s': found value [%u], sz = %lld.\n", | |
1245 | map_name, t->type, sz); | |
1246 | if (map->def.value_size && map->def.value_size != sz) { | |
900de4ac | 1247 | pr_warning("map '%s': conflicting value size %u != %lld.\n", |
abd29c93 AN |
1248 | map_name, map->def.value_size, sz); |
1249 | return -EINVAL; | |
1250 | } | |
1251 | map->def.value_size = sz; | |
1252 | map->btf_value_type_id = t->type; | |
1253 | } else { | |
1254 | if (strict) { | |
1255 | pr_warning("map '%s': unknown field '%s'.\n", | |
1256 | map_name, name); | |
1257 | return -ENOTSUP; | |
1258 | } | |
1259 | pr_debug("map '%s': ignoring unknown field '%s'.\n", | |
1260 | map_name, name); | |
1261 | } | |
1262 | } | |
1263 | ||
1264 | if (map->def.type == BPF_MAP_TYPE_UNSPEC) { | |
1265 | pr_warning("map '%s': map type isn't specified.\n", map_name); | |
1266 | return -EINVAL; | |
1267 | } | |
1268 | ||
1269 | return 0; | |
1270 | } | |
1271 | ||
1272 | static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) | |
1273 | { | |
1274 | const struct btf_type *sec = NULL; | |
1275 | int nr_types, i, vlen, err; | |
1276 | const struct btf_type *t; | |
1277 | const char *name; | |
1278 | Elf_Data *data; | |
1279 | Elf_Scn *scn; | |
1280 | ||
1281 | if (obj->efile.btf_maps_shndx < 0) | |
1282 | return 0; | |
1283 | ||
1284 | scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); | |
1285 | if (scn) | |
1286 | data = elf_getdata(scn, NULL); | |
1287 | if (!scn || !data) { | |
1288 | pr_warning("failed to get Elf_Data from map section %d (%s)\n", | |
1289 | obj->efile.maps_shndx, MAPS_ELF_SEC); | |
1290 | return -EINVAL; | |
1291 | } | |
1292 | ||
1293 | nr_types = btf__get_nr_types(obj->btf); | |
1294 | for (i = 1; i <= nr_types; i++) { | |
1295 | t = btf__type_by_id(obj->btf, i); | |
b03bc685 | 1296 | if (!btf_is_datasec(t)) |
abd29c93 AN |
1297 | continue; |
1298 | name = btf__name_by_offset(obj->btf, t->name_off); | |
1299 | if (strcmp(name, MAPS_ELF_SEC) == 0) { | |
1300 | sec = t; | |
1301 | break; | |
1302 | } | |
1303 | } | |
1304 | ||
1305 | if (!sec) { | |
1306 | pr_warning("DATASEC '%s' not found.\n", MAPS_ELF_SEC); | |
1307 | return -ENOENT; | |
1308 | } | |
1309 | ||
b03bc685 | 1310 | vlen = btf_vlen(sec); |
abd29c93 AN |
1311 | for (i = 0; i < vlen; i++) { |
1312 | err = bpf_object__init_user_btf_map(obj, sec, i, | |
1313 | obj->efile.btf_maps_shndx, | |
1314 | data, strict); | |
1315 | if (err) | |
1316 | return err; | |
1317 | } | |
1318 | ||
1319 | return 0; | |
1320 | } | |
1321 | ||
bf829271 AN |
1322 | static int bpf_object__init_maps(struct bpf_object *obj, int flags) |
1323 | { | |
1324 | bool strict = !(flags & MAPS_RELAX_COMPAT); | |
1325 | int err; | |
8837fe5d | 1326 | |
bf829271 AN |
1327 | err = bpf_object__init_user_maps(obj, strict); |
1328 | if (err) | |
1329 | return err; | |
1330 | ||
abd29c93 AN |
1331 | err = bpf_object__init_user_btf_maps(obj, strict); |
1332 | if (err) | |
1333 | return err; | |
1334 | ||
bf829271 AN |
1335 | err = bpf_object__init_global_data_maps(obj); |
1336 | if (err) | |
1337 | return err; | |
1338 | ||
1339 | if (obj->nr_maps) { | |
d859900c DB |
1340 | qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), |
1341 | compare_bpf_map); | |
bf829271 AN |
1342 | } |
1343 | return 0; | |
561bbcca WN |
1344 | } |
1345 | ||
e3d91b0c JDB |
1346 | static bool section_have_execinstr(struct bpf_object *obj, int idx) |
1347 | { | |
1348 | Elf_Scn *scn; | |
1349 | GElf_Shdr sh; | |
1350 | ||
1351 | scn = elf_getscn(obj->efile.elf, idx); | |
1352 | if (!scn) | |
1353 | return false; | |
1354 | ||
1355 | if (gelf_getshdr(scn, &sh) != &sh) | |
1356 | return false; | |
1357 | ||
1358 | if (sh.sh_flags & SHF_EXECINSTR) | |
1359 | return true; | |
1360 | ||
1361 | return false; | |
1362 | } | |
1363 | ||
d7c4b398 AN |
1364 | static void bpf_object__sanitize_btf(struct bpf_object *obj) |
1365 | { | |
1366 | bool has_datasec = obj->caps.btf_datasec; | |
1367 | bool has_func = obj->caps.btf_func; | |
1368 | struct btf *btf = obj->btf; | |
1369 | struct btf_type *t; | |
1370 | int i, j, vlen; | |
d7c4b398 AN |
1371 | |
1372 | if (!obj->btf || (has_func && has_datasec)) | |
1373 | return; | |
1374 | ||
1375 | for (i = 1; i <= btf__get_nr_types(btf); i++) { | |
1376 | t = (struct btf_type *)btf__type_by_id(btf, i); | |
d7c4b398 | 1377 | |
b03bc685 | 1378 | if (!has_datasec && btf_is_var(t)) { |
d7c4b398 AN |
1379 | /* replace VAR with INT */ |
1380 | t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); | |
1381 | t->size = sizeof(int); | |
b03bc685 AN |
1382 | *(int *)(t + 1) = BTF_INT_ENC(0, 0, 32); |
1383 | } else if (!has_datasec && btf_is_datasec(t)) { | |
d7c4b398 | 1384 | /* replace DATASEC with STRUCT */ |
b03bc685 AN |
1385 | const struct btf_var_secinfo *v = btf_var_secinfos(t); |
1386 | struct btf_member *m = btf_members(t); | |
d7c4b398 AN |
1387 | struct btf_type *vt; |
1388 | char *name; | |
1389 | ||
1390 | name = (char *)btf__name_by_offset(btf, t->name_off); | |
1391 | while (*name) { | |
1392 | if (*name == '.') | |
1393 | *name = '_'; | |
1394 | name++; | |
1395 | } | |
1396 | ||
b03bc685 | 1397 | vlen = btf_vlen(t); |
d7c4b398 AN |
1398 | t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); |
1399 | for (j = 0; j < vlen; j++, v++, m++) { | |
1400 | /* order of field assignments is important */ | |
1401 | m->offset = v->offset * 8; | |
1402 | m->type = v->type; | |
1403 | /* preserve variable name as member name */ | |
1404 | vt = (void *)btf__type_by_id(btf, v->type); | |
1405 | m->name_off = vt->name_off; | |
1406 | } | |
b03bc685 | 1407 | } else if (!has_func && btf_is_func_proto(t)) { |
d7c4b398 | 1408 | /* replace FUNC_PROTO with ENUM */ |
b03bc685 | 1409 | vlen = btf_vlen(t); |
d7c4b398 AN |
1410 | t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); |
1411 | t->size = sizeof(__u32); /* kernel enforced */ | |
b03bc685 | 1412 | } else if (!has_func && btf_is_func(t)) { |
d7c4b398 AN |
1413 | /* replace FUNC with TYPEDEF */ |
1414 | t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); | |
1415 | } | |
1416 | } | |
1417 | } | |
1418 | ||
1419 | static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) | |
1420 | { | |
1421 | if (!obj->btf_ext) | |
1422 | return; | |
1423 | ||
1424 | if (!obj->caps.btf_func) { | |
1425 | btf_ext__free(obj->btf_ext); | |
1426 | obj->btf_ext = NULL; | |
1427 | } | |
1428 | } | |
1429 | ||
abd29c93 AN |
1430 | static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj) |
1431 | { | |
1432 | return obj->efile.btf_maps_shndx >= 0; | |
1433 | } | |
1434 | ||
063183bf | 1435 | static int bpf_object__init_btf(struct bpf_object *obj, |
9c6660d0 AN |
1436 | Elf_Data *btf_data, |
1437 | Elf_Data *btf_ext_data) | |
1438 | { | |
abd29c93 | 1439 | bool btf_required = bpf_object__is_btf_mandatory(obj); |
9c6660d0 AN |
1440 | int err = 0; |
1441 | ||
1442 | if (btf_data) { | |
1443 | obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); | |
1444 | if (IS_ERR(obj->btf)) { | |
1445 | pr_warning("Error loading ELF section %s: %d.\n", | |
1446 | BTF_ELF_SEC, err); | |
1447 | goto out; | |
1448 | } | |
1449 | err = btf__finalize_data(obj, obj->btf); | |
1450 | if (err) { | |
1451 | pr_warning("Error finalizing %s: %d.\n", | |
1452 | BTF_ELF_SEC, err); | |
1453 | goto out; | |
1454 | } | |
9c6660d0 AN |
1455 | } |
1456 | if (btf_ext_data) { | |
1457 | if (!obj->btf) { | |
1458 | pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", | |
1459 | BTF_EXT_ELF_SEC, BTF_ELF_SEC); | |
1460 | goto out; | |
1461 | } | |
1462 | obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, | |
1463 | btf_ext_data->d_size); | |
1464 | if (IS_ERR(obj->btf_ext)) { | |
1465 | pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", | |
1466 | BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); | |
1467 | obj->btf_ext = NULL; | |
1468 | goto out; | |
1469 | } | |
9c6660d0 AN |
1470 | } |
1471 | out: | |
1472 | if (err || IS_ERR(obj->btf)) { | |
abd29c93 AN |
1473 | if (btf_required) |
1474 | err = err ? : PTR_ERR(obj->btf); | |
1475 | else | |
1476 | err = 0; | |
9c6660d0 AN |
1477 | if (!IS_ERR_OR_NULL(obj->btf)) |
1478 | btf__free(obj->btf); | |
1479 | obj->btf = NULL; | |
1480 | } | |
abd29c93 AN |
1481 | if (btf_required && !obj->btf) { |
1482 | pr_warning("BTF is required, but is missing or corrupted.\n"); | |
1483 | return err == 0 ? -ENOENT : err; | |
1484 | } | |
9c6660d0 AN |
1485 | return 0; |
1486 | } | |
1487 | ||
063183bf AN |
1488 | static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) |
1489 | { | |
1490 | int err = 0; | |
1491 | ||
1492 | if (!obj->btf) | |
1493 | return 0; | |
1494 | ||
1495 | bpf_object__sanitize_btf(obj); | |
1496 | bpf_object__sanitize_btf_ext(obj); | |
1497 | ||
1498 | err = btf__load(obj->btf); | |
1499 | if (err) { | |
1500 | pr_warning("Error loading %s into kernel: %d.\n", | |
1501 | BTF_ELF_SEC, err); | |
1502 | btf__free(obj->btf); | |
1503 | obj->btf = NULL; | |
abd29c93 AN |
1504 | if (bpf_object__is_btf_mandatory(obj)) |
1505 | return err; | |
063183bf AN |
1506 | } |
1507 | return 0; | |
1508 | } | |
1509 | ||
c034a177 | 1510 | static int bpf_object__elf_collect(struct bpf_object *obj, int flags) |
29603665 WN |
1511 | { |
1512 | Elf *elf = obj->efile.elf; | |
1513 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
f0187f0b | 1514 | Elf_Data *btf_ext_data = NULL; |
1713d68b | 1515 | Elf_Data *btf_data = NULL; |
29603665 | 1516 | Elf_Scn *scn = NULL; |
666810e8 | 1517 | int idx = 0, err = 0; |
29603665 WN |
1518 | |
1519 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
1520 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
399dc65e | 1521 | pr_warning("failed to get e_shstrndx from %s\n", obj->path); |
6371ca3b | 1522 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
1523 | } |
1524 | ||
1525 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
1526 | char *name; | |
1527 | GElf_Shdr sh; | |
1528 | Elf_Data *data; | |
1529 | ||
1530 | idx++; | |
1531 | if (gelf_getshdr(scn, &sh) != &sh) { | |
077c066a JDB |
1532 | pr_warning("failed to get section(%d) header from %s\n", |
1533 | idx, obj->path); | |
01b29d1d | 1534 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
1535 | } |
1536 | ||
1537 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
1538 | if (!name) { | |
077c066a JDB |
1539 | pr_warning("failed to get section(%d) name from %s\n", |
1540 | idx, obj->path); | |
01b29d1d | 1541 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
1542 | } |
1543 | ||
1544 | data = elf_getdata(scn, 0); | |
1545 | if (!data) { | |
077c066a JDB |
1546 | pr_warning("failed to get section(%d) data from %s(%s)\n", |
1547 | idx, name, obj->path); | |
01b29d1d | 1548 | return -LIBBPF_ERRNO__FORMAT; |
29603665 | 1549 | } |
077c066a JDB |
1550 | pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", |
1551 | idx, name, (unsigned long)data->d_size, | |
29603665 WN |
1552 | (int)sh.sh_link, (unsigned long)sh.sh_flags, |
1553 | (int)sh.sh_type); | |
cb1e5e96 | 1554 | |
1713d68b | 1555 | if (strcmp(name, "license") == 0) { |
cb1e5e96 WN |
1556 | err = bpf_object__init_license(obj, |
1557 | data->d_buf, | |
1558 | data->d_size); | |
01b29d1d AN |
1559 | if (err) |
1560 | return err; | |
1713d68b | 1561 | } else if (strcmp(name, "version") == 0) { |
cb1e5e96 WN |
1562 | err = bpf_object__init_kversion(obj, |
1563 | data->d_buf, | |
1564 | data->d_size); | |
01b29d1d AN |
1565 | if (err) |
1566 | return err; | |
1713d68b | 1567 | } else if (strcmp(name, "maps") == 0) { |
666810e8 | 1568 | obj->efile.maps_shndx = idx; |
abd29c93 AN |
1569 | } else if (strcmp(name, MAPS_ELF_SEC) == 0) { |
1570 | obj->efile.btf_maps_shndx = idx; | |
1713d68b DB |
1571 | } else if (strcmp(name, BTF_ELF_SEC) == 0) { |
1572 | btf_data = data; | |
2993e051 | 1573 | } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { |
f0187f0b | 1574 | btf_ext_data = data; |
8a138aed | 1575 | } else if (sh.sh_type == SHT_SYMTAB) { |
bec7d68c WN |
1576 | if (obj->efile.symbols) { |
1577 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
1578 | obj->path); | |
01b29d1d | 1579 | return -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 1580 | } |
01b29d1d AN |
1581 | obj->efile.symbols = data; |
1582 | obj->efile.strtabidx = sh.sh_link; | |
f8c7a4d4 JS |
1583 | } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { |
1584 | if (sh.sh_flags & SHF_EXECINSTR) { | |
1585 | if (strcmp(name, ".text") == 0) | |
1586 | obj->efile.text_shndx = idx; | |
1587 | err = bpf_object__add_program(obj, data->d_buf, | |
1588 | data->d_size, name, idx); | |
1589 | if (err) { | |
1590 | char errmsg[STRERR_BUFSIZE]; | |
1591 | char *cp = libbpf_strerror_r(-err, errmsg, | |
1592 | sizeof(errmsg)); | |
1593 | ||
1594 | pr_warning("failed to alloc program %s (%s): %s", | |
1595 | name, obj->path, cp); | |
01b29d1d | 1596 | return err; |
f8c7a4d4 | 1597 | } |
d859900c DB |
1598 | } else if (strcmp(name, ".data") == 0) { |
1599 | obj->efile.data = data; | |
1600 | obj->efile.data_shndx = idx; | |
1601 | } else if (strcmp(name, ".rodata") == 0) { | |
1602 | obj->efile.rodata = data; | |
1603 | obj->efile.rodata_shndx = idx; | |
1604 | } else { | |
1605 | pr_debug("skip section(%d) %s\n", idx, name); | |
a5b8bd47 | 1606 | } |
b62f06e8 | 1607 | } else if (sh.sh_type == SHT_REL) { |
01b29d1d | 1608 | int nr_reloc = obj->efile.nr_reloc; |
b62f06e8 | 1609 | void *reloc = obj->efile.reloc; |
e3d91b0c JDB |
1610 | int sec = sh.sh_info; /* points to other section */ |
1611 | ||
1612 | /* Only do relo for section with exec instructions */ | |
1613 | if (!section_have_execinstr(obj, sec)) { | |
1614 | pr_debug("skip relo %s(%d) for section(%d)\n", | |
1615 | name, idx, sec); | |
1616 | continue; | |
1617 | } | |
b62f06e8 | 1618 | |
01b29d1d | 1619 | reloc = reallocarray(reloc, nr_reloc + 1, |
531b014e | 1620 | sizeof(*obj->efile.reloc)); |
b62f06e8 WN |
1621 | if (!reloc) { |
1622 | pr_warning("realloc failed\n"); | |
01b29d1d AN |
1623 | return -ENOMEM; |
1624 | } | |
b62f06e8 | 1625 | |
01b29d1d AN |
1626 | obj->efile.reloc = reloc; |
1627 | obj->efile.nr_reloc++; | |
b62f06e8 | 1628 | |
01b29d1d AN |
1629 | obj->efile.reloc[nr_reloc].shdr = sh; |
1630 | obj->efile.reloc[nr_reloc].data = data; | |
d859900c DB |
1631 | } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { |
1632 | obj->efile.bss = data; | |
1633 | obj->efile.bss_shndx = idx; | |
077c066a JDB |
1634 | } else { |
1635 | pr_debug("skip section(%d) %s\n", idx, name); | |
bec7d68c | 1636 | } |
29603665 | 1637 | } |
561bbcca | 1638 | |
77ba9a5b WN |
1639 | if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { |
1640 | pr_warning("Corrupted ELF file: index of strtab invalid\n"); | |
f102154d | 1641 | return -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 1642 | } |
063183bf | 1643 | err = bpf_object__init_btf(obj, btf_data, btf_ext_data); |
bf829271 | 1644 | if (!err) |
c034a177 | 1645 | err = bpf_object__init_maps(obj, flags); |
063183bf AN |
1646 | if (!err) |
1647 | err = bpf_object__sanitize_and_load_btf(obj); | |
bf829271 AN |
1648 | if (!err) |
1649 | err = bpf_object__init_prog_names(obj); | |
29603665 WN |
1650 | return err; |
1651 | } | |
1652 | ||
34090915 WN |
1653 | static struct bpf_program * |
1654 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
1655 | { | |
1656 | struct bpf_program *prog; | |
1657 | size_t i; | |
1658 | ||
1659 | for (i = 0; i < obj->nr_programs; i++) { | |
1660 | prog = &obj->programs[i]; | |
1661 | if (prog->idx == idx) | |
1662 | return prog; | |
1663 | } | |
1664 | return NULL; | |
1665 | } | |
1666 | ||
6d4b198b | 1667 | struct bpf_program * |
a324aae3 AN |
1668 | bpf_object__find_program_by_title(const struct bpf_object *obj, |
1669 | const char *title) | |
6d4b198b JK |
1670 | { |
1671 | struct bpf_program *pos; | |
1672 | ||
1673 | bpf_object__for_each_program(pos, obj) { | |
1674 | if (pos->section_name && !strcmp(pos->section_name, title)) | |
1675 | return pos; | |
1676 | } | |
1677 | return NULL; | |
1678 | } | |
1679 | ||
d859900c DB |
1680 | static bool bpf_object__shndx_is_data(const struct bpf_object *obj, |
1681 | int shndx) | |
1682 | { | |
1683 | return shndx == obj->efile.data_shndx || | |
1684 | shndx == obj->efile.bss_shndx || | |
1685 | shndx == obj->efile.rodata_shndx; | |
1686 | } | |
1687 | ||
1688 | static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, | |
1689 | int shndx) | |
1690 | { | |
abd29c93 AN |
1691 | return shndx == obj->efile.maps_shndx || |
1692 | shndx == obj->efile.btf_maps_shndx; | |
d859900c DB |
1693 | } |
1694 | ||
1695 | static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, | |
1696 | int shndx) | |
1697 | { | |
1698 | return shndx == obj->efile.text_shndx || | |
1699 | bpf_object__shndx_is_maps(obj, shndx) || | |
1700 | bpf_object__shndx_is_data(obj, shndx); | |
1701 | } | |
1702 | ||
1703 | static enum libbpf_map_type | |
1704 | bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) | |
1705 | { | |
1706 | if (shndx == obj->efile.data_shndx) | |
1707 | return LIBBPF_MAP_DATA; | |
1708 | else if (shndx == obj->efile.bss_shndx) | |
1709 | return LIBBPF_MAP_BSS; | |
1710 | else if (shndx == obj->efile.rodata_shndx) | |
1711 | return LIBBPF_MAP_RODATA; | |
1712 | else | |
1713 | return LIBBPF_MAP_UNSPEC; | |
1714 | } | |
1715 | ||
34090915 | 1716 | static int |
48cca7e4 AS |
1717 | bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, |
1718 | Elf_Data *data, struct bpf_object *obj) | |
34090915 | 1719 | { |
48cca7e4 | 1720 | Elf_Data *symbols = obj->efile.symbols; |
48cca7e4 AS |
1721 | struct bpf_map *maps = obj->maps; |
1722 | size_t nr_maps = obj->nr_maps; | |
34090915 WN |
1723 | int i, nrels; |
1724 | ||
399dc65e | 1725 | pr_debug("collecting relocating info for: '%s'\n", prog->section_name); |
34090915 WN |
1726 | nrels = shdr->sh_size / shdr->sh_entsize; |
1727 | ||
1728 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
1729 | if (!prog->reloc_desc) { | |
1730 | pr_warning("failed to alloc memory in relocation\n"); | |
1731 | return -ENOMEM; | |
1732 | } | |
1733 | prog->nr_reloc = nrels; | |
1734 | ||
1735 | for (i = 0; i < nrels; i++) { | |
34090915 | 1736 | struct bpf_insn *insns = prog->insns; |
d859900c | 1737 | enum libbpf_map_type type; |
abd29c93 AN |
1738 | unsigned int insn_idx; |
1739 | unsigned int shdr_idx; | |
d859900c | 1740 | const char *name; |
34090915 | 1741 | size_t map_idx; |
abd29c93 AN |
1742 | GElf_Sym sym; |
1743 | GElf_Rel rel; | |
34090915 WN |
1744 | |
1745 | if (!gelf_getrel(data, i, &rel)) { | |
1746 | pr_warning("relocation: failed to get %d reloc\n", i); | |
6371ca3b | 1747 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
1748 | } |
1749 | ||
399dc65e | 1750 | if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { |
34090915 WN |
1751 | pr_warning("relocation: symbol %"PRIx64" not found\n", |
1752 | GELF_R_SYM(rel.r_info)); | |
6371ca3b | 1753 | return -LIBBPF_ERRNO__FORMAT; |
34090915 | 1754 | } |
d859900c DB |
1755 | |
1756 | name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
1757 | sym.st_name) ? : "<?>"; | |
1758 | ||
1759 | pr_debug("relo for %lld value %lld name %d (\'%s\')\n", | |
7d9890ef | 1760 | (long long) (rel.r_info >> 32), |
d859900c | 1761 | (long long) sym.st_value, sym.st_name, name); |
34090915 | 1762 | |
d859900c | 1763 | shdr_idx = sym.st_shndx; |
f2a3e4e9 AN |
1764 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); |
1765 | pr_debug("relocation: insn_idx=%u, shdr_idx=%u\n", | |
1766 | insn_idx, shdr_idx); | |
1767 | ||
1768 | if (shdr_idx >= SHN_LORESERVE) { | |
1769 | pr_warning("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n", | |
1770 | name, shdr_idx, insn_idx, | |
1771 | insns[insn_idx].code); | |
1772 | return -LIBBPF_ERRNO__RELOC; | |
1773 | } | |
d859900c DB |
1774 | if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { |
1775 | pr_warning("Program '%s' contains unrecognized relo data pointing to section %u\n", | |
1776 | prog->section_name, shdr_idx); | |
666810e8 WN |
1777 | return -LIBBPF_ERRNO__RELOC; |
1778 | } | |
1779 | ||
48cca7e4 AS |
1780 | if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { |
1781 | if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { | |
1782 | pr_warning("incorrect bpf_call opcode\n"); | |
1783 | return -LIBBPF_ERRNO__RELOC; | |
1784 | } | |
1785 | prog->reloc_desc[i].type = RELO_CALL; | |
1786 | prog->reloc_desc[i].insn_idx = insn_idx; | |
1787 | prog->reloc_desc[i].text_off = sym.st_value; | |
9a94f277 | 1788 | obj->has_pseudo_calls = true; |
48cca7e4 AS |
1789 | continue; |
1790 | } | |
1791 | ||
34090915 WN |
1792 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { |
1793 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
1794 | insn_idx, insns[insn_idx].code); | |
6371ca3b | 1795 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
1796 | } |
1797 | ||
d859900c DB |
1798 | if (bpf_object__shndx_is_maps(obj, shdr_idx) || |
1799 | bpf_object__shndx_is_data(obj, shdr_idx)) { | |
1800 | type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); | |
8837fe5d DB |
1801 | if (type != LIBBPF_MAP_UNSPEC) { |
1802 | if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { | |
1803 | pr_warning("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", | |
1804 | name, insn_idx, insns[insn_idx].code); | |
1805 | return -LIBBPF_ERRNO__RELOC; | |
1806 | } | |
1807 | if (!obj->caps.global_data) { | |
1808 | pr_warning("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", | |
1809 | name, insn_idx); | |
1810 | return -LIBBPF_ERRNO__RELOC; | |
1811 | } | |
d859900c DB |
1812 | } |
1813 | ||
f8c7a4d4 | 1814 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { |
d859900c DB |
1815 | if (maps[map_idx].libbpf_type != type) |
1816 | continue; | |
1817 | if (type != LIBBPF_MAP_UNSPEC || | |
db48814b AN |
1818 | (maps[map_idx].sec_idx == sym.st_shndx && |
1819 | maps[map_idx].sec_offset == sym.st_value)) { | |
1820 | pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n", | |
1821 | map_idx, maps[map_idx].name, | |
1822 | maps[map_idx].sec_idx, | |
1823 | maps[map_idx].sec_offset, | |
1824 | insn_idx); | |
f8c7a4d4 JS |
1825 | break; |
1826 | } | |
94e5adec | 1827 | } |
94e5adec | 1828 | |
f8c7a4d4 | 1829 | if (map_idx >= nr_maps) { |
76e1022b | 1830 | pr_warning("bpf relocation: map_idx %d larger than %d\n", |
f8c7a4d4 JS |
1831 | (int)map_idx, (int)nr_maps - 1); |
1832 | return -LIBBPF_ERRNO__RELOC; | |
1833 | } | |
34090915 | 1834 | |
d859900c DB |
1835 | prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? |
1836 | RELO_DATA : RELO_LD64; | |
f8c7a4d4 JS |
1837 | prog->reloc_desc[i].insn_idx = insn_idx; |
1838 | prog->reloc_desc[i].map_idx = map_idx; | |
1839 | } | |
34090915 WN |
1840 | } |
1841 | return 0; | |
1842 | } | |
1843 | ||
abd29c93 | 1844 | static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) |
8a138aed MKL |
1845 | { |
1846 | struct bpf_map_def *def = &map->def; | |
d859900c | 1847 | __u32 key_type_id = 0, value_type_id = 0; |
96408c43 | 1848 | int ret; |
8a138aed | 1849 | |
abd29c93 AN |
1850 | /* if it's BTF-defined map, we don't need to search for type IDs */ |
1851 | if (map->sec_idx == obj->efile.btf_maps_shndx) | |
1852 | return 0; | |
1853 | ||
d859900c | 1854 | if (!bpf_map__is_internal(map)) { |
abd29c93 | 1855 | ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, |
d859900c DB |
1856 | def->value_size, &key_type_id, |
1857 | &value_type_id); | |
1858 | } else { | |
1859 | /* | |
1860 | * LLVM annotates global data differently in BTF, that is, | |
1861 | * only as '.data', '.bss' or '.rodata'. | |
1862 | */ | |
abd29c93 | 1863 | ret = btf__find_by_name(obj->btf, |
d859900c DB |
1864 | libbpf_type_to_btf_name[map->libbpf_type]); |
1865 | } | |
1866 | if (ret < 0) | |
96408c43 | 1867 | return ret; |
8a138aed | 1868 | |
96408c43 | 1869 | map->btf_key_type_id = key_type_id; |
d859900c DB |
1870 | map->btf_value_type_id = bpf_map__is_internal(map) ? |
1871 | ret : value_type_id; | |
8a138aed MKL |
1872 | return 0; |
1873 | } | |
1874 | ||
26736eb9 JK |
1875 | int bpf_map__reuse_fd(struct bpf_map *map, int fd) |
1876 | { | |
1877 | struct bpf_map_info info = {}; | |
1878 | __u32 len = sizeof(info); | |
1879 | int new_fd, err; | |
1880 | char *new_name; | |
1881 | ||
1882 | err = bpf_obj_get_info_by_fd(fd, &info, &len); | |
1883 | if (err) | |
1884 | return err; | |
1885 | ||
1886 | new_name = strdup(info.name); | |
1887 | if (!new_name) | |
1888 | return -errno; | |
1889 | ||
1890 | new_fd = open("/", O_RDONLY | O_CLOEXEC); | |
1891 | if (new_fd < 0) | |
1892 | goto err_free_new_name; | |
1893 | ||
1894 | new_fd = dup3(fd, new_fd, O_CLOEXEC); | |
1895 | if (new_fd < 0) | |
1896 | goto err_close_new_fd; | |
1897 | ||
1898 | err = zclose(map->fd); | |
1899 | if (err) | |
1900 | goto err_close_new_fd; | |
1901 | free(map->name); | |
1902 | ||
1903 | map->fd = new_fd; | |
1904 | map->name = new_name; | |
1905 | map->def.type = info.type; | |
1906 | map->def.key_size = info.key_size; | |
1907 | map->def.value_size = info.value_size; | |
1908 | map->def.max_entries = info.max_entries; | |
1909 | map->def.map_flags = info.map_flags; | |
1910 | map->btf_key_type_id = info.btf_key_type_id; | |
1911 | map->btf_value_type_id = info.btf_value_type_id; | |
1912 | ||
1913 | return 0; | |
1914 | ||
1915 | err_close_new_fd: | |
1916 | close(new_fd); | |
1917 | err_free_new_name: | |
1918 | free(new_name); | |
1919 | return -errno; | |
1920 | } | |
1921 | ||
1a11a4c7 AI |
1922 | int bpf_map__resize(struct bpf_map *map, __u32 max_entries) |
1923 | { | |
1924 | if (!map || !max_entries) | |
1925 | return -EINVAL; | |
1926 | ||
1927 | /* If map already created, its attributes can't be changed. */ | |
1928 | if (map->fd >= 0) | |
1929 | return -EBUSY; | |
1930 | ||
1931 | map->def.max_entries = max_entries; | |
1932 | ||
1933 | return 0; | |
1934 | } | |
1935 | ||
47eff617 SF |
1936 | static int |
1937 | bpf_object__probe_name(struct bpf_object *obj) | |
1938 | { | |
1939 | struct bpf_load_program_attr attr; | |
1940 | char *cp, errmsg[STRERR_BUFSIZE]; | |
1941 | struct bpf_insn insns[] = { | |
1942 | BPF_MOV64_IMM(BPF_REG_0, 0), | |
1943 | BPF_EXIT_INSN(), | |
1944 | }; | |
1945 | int ret; | |
1946 | ||
1947 | /* make sure basic loading works */ | |
1948 | ||
1949 | memset(&attr, 0, sizeof(attr)); | |
1950 | attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; | |
1951 | attr.insns = insns; | |
1952 | attr.insns_cnt = ARRAY_SIZE(insns); | |
1953 | attr.license = "GPL"; | |
1954 | ||
1955 | ret = bpf_load_program_xattr(&attr, NULL, 0); | |
1956 | if (ret < 0) { | |
1957 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
1958 | pr_warning("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", | |
1959 | __func__, cp, errno); | |
1960 | return -errno; | |
1961 | } | |
1962 | close(ret); | |
1963 | ||
1964 | /* now try the same program, but with the name */ | |
1965 | ||
1966 | attr.name = "test"; | |
1967 | ret = bpf_load_program_xattr(&attr, NULL, 0); | |
1968 | if (ret >= 0) { | |
1969 | obj->caps.name = 1; | |
1970 | close(ret); | |
1971 | } | |
1972 | ||
1973 | return 0; | |
1974 | } | |
1975 | ||
8837fe5d DB |
1976 | static int |
1977 | bpf_object__probe_global_data(struct bpf_object *obj) | |
1978 | { | |
1979 | struct bpf_load_program_attr prg_attr; | |
1980 | struct bpf_create_map_attr map_attr; | |
1981 | char *cp, errmsg[STRERR_BUFSIZE]; | |
1982 | struct bpf_insn insns[] = { | |
1983 | BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), | |
1984 | BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), | |
1985 | BPF_MOV64_IMM(BPF_REG_0, 0), | |
1986 | BPF_EXIT_INSN(), | |
1987 | }; | |
1988 | int ret, map; | |
1989 | ||
1990 | memset(&map_attr, 0, sizeof(map_attr)); | |
1991 | map_attr.map_type = BPF_MAP_TYPE_ARRAY; | |
1992 | map_attr.key_size = sizeof(int); | |
1993 | map_attr.value_size = 32; | |
1994 | map_attr.max_entries = 1; | |
1995 | ||
1996 | map = bpf_create_map_xattr(&map_attr); | |
1997 | if (map < 0) { | |
1998 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
1999 | pr_warning("Error in %s():%s(%d). Couldn't create simple array map.\n", | |
2000 | __func__, cp, errno); | |
2001 | return -errno; | |
2002 | } | |
2003 | ||
2004 | insns[0].imm = map; | |
2005 | ||
2006 | memset(&prg_attr, 0, sizeof(prg_attr)); | |
2007 | prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; | |
2008 | prg_attr.insns = insns; | |
2009 | prg_attr.insns_cnt = ARRAY_SIZE(insns); | |
2010 | prg_attr.license = "GPL"; | |
2011 | ||
2012 | ret = bpf_load_program_xattr(&prg_attr, NULL, 0); | |
2013 | if (ret >= 0) { | |
2014 | obj->caps.global_data = 1; | |
2015 | close(ret); | |
2016 | } | |
2017 | ||
2018 | close(map); | |
2019 | return 0; | |
2020 | } | |
2021 | ||
d7c4b398 AN |
2022 | static int bpf_object__probe_btf_func(struct bpf_object *obj) |
2023 | { | |
2024 | const char strs[] = "\0int\0x\0a"; | |
2025 | /* void x(int a) {} */ | |
2026 | __u32 types[] = { | |
2027 | /* int */ | |
2028 | BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ | |
2029 | /* FUNC_PROTO */ /* [2] */ | |
2030 | BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), | |
2031 | BTF_PARAM_ENC(7, 1), | |
2032 | /* FUNC x */ /* [3] */ | |
2033 | BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), | |
2034 | }; | |
cfd49210 | 2035 | int btf_fd; |
d7c4b398 | 2036 | |
cfd49210 MR |
2037 | btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), |
2038 | strs, sizeof(strs)); | |
2039 | if (btf_fd >= 0) { | |
d7c4b398 | 2040 | obj->caps.btf_func = 1; |
cfd49210 MR |
2041 | close(btf_fd); |
2042 | return 1; | |
2043 | } | |
2044 | ||
d7c4b398 AN |
2045 | return 0; |
2046 | } | |
2047 | ||
2048 | static int bpf_object__probe_btf_datasec(struct bpf_object *obj) | |
2049 | { | |
2050 | const char strs[] = "\0x\0.data"; | |
2051 | /* static int a; */ | |
2052 | __u32 types[] = { | |
2053 | /* int */ | |
2054 | BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ | |
2055 | /* VAR x */ /* [2] */ | |
2056 | BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), | |
2057 | BTF_VAR_STATIC, | |
2058 | /* DATASEC val */ /* [3] */ | |
2059 | BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), | |
2060 | BTF_VAR_SECINFO_ENC(2, 0, 4), | |
2061 | }; | |
cfd49210 | 2062 | int btf_fd; |
d7c4b398 | 2063 | |
cfd49210 MR |
2064 | btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), |
2065 | strs, sizeof(strs)); | |
2066 | if (btf_fd >= 0) { | |
d7c4b398 | 2067 | obj->caps.btf_datasec = 1; |
cfd49210 MR |
2068 | close(btf_fd); |
2069 | return 1; | |
2070 | } | |
2071 | ||
d7c4b398 AN |
2072 | return 0; |
2073 | } | |
2074 | ||
47eff617 SF |
2075 | static int |
2076 | bpf_object__probe_caps(struct bpf_object *obj) | |
2077 | { | |
8837fe5d DB |
2078 | int (*probe_fn[])(struct bpf_object *obj) = { |
2079 | bpf_object__probe_name, | |
2080 | bpf_object__probe_global_data, | |
d7c4b398 AN |
2081 | bpf_object__probe_btf_func, |
2082 | bpf_object__probe_btf_datasec, | |
8837fe5d DB |
2083 | }; |
2084 | int i, ret; | |
2085 | ||
2086 | for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { | |
2087 | ret = probe_fn[i](obj); | |
2088 | if (ret < 0) | |
15ea164e | 2089 | pr_debug("Probe #%d failed with %d.\n", i, ret); |
8837fe5d DB |
2090 | } |
2091 | ||
2092 | return 0; | |
47eff617 SF |
2093 | } |
2094 | ||
d859900c DB |
2095 | static int |
2096 | bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) | |
2097 | { | |
2098 | char *cp, errmsg[STRERR_BUFSIZE]; | |
2099 | int err, zero = 0; | |
2100 | __u8 *data; | |
2101 | ||
2102 | /* Nothing to do here since kernel already zero-initializes .bss map. */ | |
2103 | if (map->libbpf_type == LIBBPF_MAP_BSS) | |
2104 | return 0; | |
2105 | ||
2106 | data = map->libbpf_type == LIBBPF_MAP_DATA ? | |
2107 | obj->sections.data : obj->sections.rodata; | |
2108 | ||
2109 | err = bpf_map_update_elem(map->fd, &zero, data, 0); | |
2110 | /* Freeze .rodata map as read-only from syscall side. */ | |
2111 | if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { | |
2112 | err = bpf_map_freeze(map->fd); | |
2113 | if (err) { | |
2114 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
2115 | pr_warning("Error freezing map(%s) as read-only: %s\n", | |
2116 | map->name, cp); | |
2117 | err = 0; | |
2118 | } | |
2119 | } | |
2120 | return err; | |
2121 | } | |
2122 | ||
52d3352e WN |
2123 | static int |
2124 | bpf_object__create_maps(struct bpf_object *obj) | |
2125 | { | |
8a138aed | 2126 | struct bpf_create_map_attr create_attr = {}; |
d7ff34d5 | 2127 | int nr_cpus = 0; |
52d3352e | 2128 | unsigned int i; |
8a138aed | 2129 | int err; |
52d3352e | 2130 | |
9d759a9b | 2131 | for (i = 0; i < obj->nr_maps; i++) { |
8a138aed MKL |
2132 | struct bpf_map *map = &obj->maps[i]; |
2133 | struct bpf_map_def *def = &map->def; | |
1ce6a9fc | 2134 | char *cp, errmsg[STRERR_BUFSIZE]; |
8a138aed MKL |
2135 | int *pfd = &map->fd; |
2136 | ||
26736eb9 JK |
2137 | if (map->fd >= 0) { |
2138 | pr_debug("skip map create (preset) %s: fd=%d\n", | |
2139 | map->name, map->fd); | |
2140 | continue; | |
2141 | } | |
2142 | ||
94cb310c SF |
2143 | if (obj->caps.name) |
2144 | create_attr.name = map->name; | |
f0307a7e | 2145 | create_attr.map_ifindex = map->map_ifindex; |
8a138aed MKL |
2146 | create_attr.map_type = def->type; |
2147 | create_attr.map_flags = def->map_flags; | |
2148 | create_attr.key_size = def->key_size; | |
2149 | create_attr.value_size = def->value_size; | |
d7ff34d5 AN |
2150 | if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && |
2151 | !def->max_entries) { | |
2152 | if (!nr_cpus) | |
2153 | nr_cpus = libbpf_num_possible_cpus(); | |
2154 | if (nr_cpus < 0) { | |
2155 | pr_warning("failed to determine number of system CPUs: %d\n", | |
2156 | nr_cpus); | |
2157 | err = nr_cpus; | |
2158 | goto err_out; | |
2159 | } | |
2160 | pr_debug("map '%s': setting size to %d\n", | |
2161 | map->name, nr_cpus); | |
2162 | create_attr.max_entries = nr_cpus; | |
2163 | } else { | |
2164 | create_attr.max_entries = def->max_entries; | |
2165 | } | |
e55d54f4 | 2166 | create_attr.btf_fd = 0; |
61746dbe MKL |
2167 | create_attr.btf_key_type_id = 0; |
2168 | create_attr.btf_value_type_id = 0; | |
addb9fc9 NS |
2169 | if (bpf_map_type__is_map_in_map(def->type) && |
2170 | map->inner_map_fd >= 0) | |
2171 | create_attr.inner_map_fd = map->inner_map_fd; | |
8a138aed | 2172 | |
abd29c93 | 2173 | if (obj->btf && !bpf_map_find_btf_info(obj, map)) { |
8a138aed | 2174 | create_attr.btf_fd = btf__fd(obj->btf); |
61746dbe MKL |
2175 | create_attr.btf_key_type_id = map->btf_key_type_id; |
2176 | create_attr.btf_value_type_id = map->btf_value_type_id; | |
8a138aed MKL |
2177 | } |
2178 | ||
2179 | *pfd = bpf_create_map_xattr(&create_attr); | |
e55d54f4 AN |
2180 | if (*pfd < 0 && (create_attr.btf_key_type_id || |
2181 | create_attr.btf_value_type_id)) { | |
d7ff34d5 AN |
2182 | err = -errno; |
2183 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
8a138aed | 2184 | pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", |
d7ff34d5 | 2185 | map->name, cp, err); |
e55d54f4 | 2186 | create_attr.btf_fd = 0; |
61746dbe MKL |
2187 | create_attr.btf_key_type_id = 0; |
2188 | create_attr.btf_value_type_id = 0; | |
2189 | map->btf_key_type_id = 0; | |
2190 | map->btf_value_type_id = 0; | |
8a138aed MKL |
2191 | *pfd = bpf_create_map_xattr(&create_attr); |
2192 | } | |
2193 | ||
52d3352e WN |
2194 | if (*pfd < 0) { |
2195 | size_t j; | |
52d3352e | 2196 | |
d7ff34d5 | 2197 | err = -errno; |
d859900c | 2198 | err_out: |
d7ff34d5 AN |
2199 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); |
2200 | pr_warning("failed to create map (name: '%s'): %s(%d)\n", | |
2201 | map->name, cp, err); | |
52d3352e | 2202 | for (j = 0; j < i; j++) |
9d759a9b | 2203 | zclose(obj->maps[j].fd); |
52d3352e WN |
2204 | return err; |
2205 | } | |
d859900c DB |
2206 | |
2207 | if (bpf_map__is_internal(map)) { | |
2208 | err = bpf_object__populate_internal_map(obj, map); | |
2209 | if (err < 0) { | |
2210 | zclose(*pfd); | |
2211 | goto err_out; | |
2212 | } | |
2213 | } | |
2214 | ||
76e1022b | 2215 | pr_debug("created map %s: fd=%d\n", map->name, *pfd); |
52d3352e WN |
2216 | } |
2217 | ||
52d3352e WN |
2218 | return 0; |
2219 | } | |
2220 | ||
f0187f0b MKL |
2221 | static int |
2222 | check_btf_ext_reloc_err(struct bpf_program *prog, int err, | |
2223 | void *btf_prog_info, const char *info_name) | |
2224 | { | |
2225 | if (err != -ENOENT) { | |
2226 | pr_warning("Error in loading %s for sec %s.\n", | |
2227 | info_name, prog->section_name); | |
2228 | return err; | |
2229 | } | |
2230 | ||
2231 | /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ | |
2232 | ||
2233 | if (btf_prog_info) { | |
2234 | /* | |
2235 | * Some info has already been found but has problem | |
399dc65e | 2236 | * in the last btf_ext reloc. Must have to error out. |
f0187f0b MKL |
2237 | */ |
2238 | pr_warning("Error in relocating %s for sec %s.\n", | |
2239 | info_name, prog->section_name); | |
2240 | return err; | |
2241 | } | |
2242 | ||
399dc65e | 2243 | /* Have problem loading the very first info. Ignore the rest. */ |
f0187f0b MKL |
2244 | pr_warning("Cannot find %s for main program sec %s. Ignore all %s.\n", |
2245 | info_name, prog->section_name, info_name); | |
2246 | return 0; | |
2247 | } | |
2248 | ||
2249 | static int | |
2250 | bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, | |
2251 | const char *section_name, __u32 insn_offset) | |
2252 | { | |
2253 | int err; | |
2254 | ||
2255 | if (!insn_offset || prog->func_info) { | |
2256 | /* | |
2257 | * !insn_offset => main program | |
2258 | * | |
2259 | * For sub prog, the main program's func_info has to | |
2260 | * be loaded first (i.e. prog->func_info != NULL) | |
2261 | */ | |
2262 | err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, | |
2263 | section_name, insn_offset, | |
2264 | &prog->func_info, | |
2265 | &prog->func_info_cnt); | |
2266 | if (err) | |
2267 | return check_btf_ext_reloc_err(prog, err, | |
2268 | prog->func_info, | |
2269 | "bpf_func_info"); | |
2270 | ||
2271 | prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); | |
2272 | } | |
2273 | ||
3d650141 MKL |
2274 | if (!insn_offset || prog->line_info) { |
2275 | err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, | |
2276 | section_name, insn_offset, | |
2277 | &prog->line_info, | |
2278 | &prog->line_info_cnt); | |
2279 | if (err) | |
2280 | return check_btf_ext_reloc_err(prog, err, | |
2281 | prog->line_info, | |
2282 | "bpf_line_info"); | |
2283 | ||
2284 | prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); | |
2285 | } | |
2286 | ||
f0187f0b MKL |
2287 | if (!insn_offset) |
2288 | prog->btf_fd = btf__fd(obj->btf); | |
2289 | ||
2290 | return 0; | |
2291 | } | |
2292 | ||
48cca7e4 AS |
2293 | static int |
2294 | bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, | |
2295 | struct reloc_desc *relo) | |
2296 | { | |
2297 | struct bpf_insn *insn, *new_insn; | |
2298 | struct bpf_program *text; | |
2299 | size_t new_cnt; | |
2993e051 | 2300 | int err; |
48cca7e4 AS |
2301 | |
2302 | if (relo->type != RELO_CALL) | |
2303 | return -LIBBPF_ERRNO__RELOC; | |
2304 | ||
2305 | if (prog->idx == obj->efile.text_shndx) { | |
2306 | pr_warning("relo in .text insn %d into off %d\n", | |
2307 | relo->insn_idx, relo->text_off); | |
2308 | return -LIBBPF_ERRNO__RELOC; | |
2309 | } | |
2310 | ||
2311 | if (prog->main_prog_cnt == 0) { | |
2312 | text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); | |
2313 | if (!text) { | |
2314 | pr_warning("no .text section found yet relo into text exist\n"); | |
2315 | return -LIBBPF_ERRNO__RELOC; | |
2316 | } | |
2317 | new_cnt = prog->insns_cnt + text->insns_cnt; | |
531b014e | 2318 | new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); |
48cca7e4 AS |
2319 | if (!new_insn) { |
2320 | pr_warning("oom in prog realloc\n"); | |
2321 | return -ENOMEM; | |
2322 | } | |
2993e051 | 2323 | |
f0187f0b MKL |
2324 | if (obj->btf_ext) { |
2325 | err = bpf_program_reloc_btf_ext(prog, obj, | |
2326 | text->section_name, | |
2327 | prog->insns_cnt); | |
2328 | if (err) | |
2993e051 | 2329 | return err; |
2993e051 YS |
2330 | } |
2331 | ||
48cca7e4 AS |
2332 | memcpy(new_insn + prog->insns_cnt, text->insns, |
2333 | text->insns_cnt * sizeof(*insn)); | |
2334 | prog->insns = new_insn; | |
2335 | prog->main_prog_cnt = prog->insns_cnt; | |
2336 | prog->insns_cnt = new_cnt; | |
b1a2ce82 JC |
2337 | pr_debug("added %zd insn from %s to prog %s\n", |
2338 | text->insns_cnt, text->section_name, | |
2339 | prog->section_name); | |
48cca7e4 AS |
2340 | } |
2341 | insn = &prog->insns[relo->insn_idx]; | |
2342 | insn->imm += prog->main_prog_cnt - relo->insn_idx; | |
48cca7e4 AS |
2343 | return 0; |
2344 | } | |
2345 | ||
8a47a6c5 | 2346 | static int |
9d759a9b | 2347 | bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) |
8a47a6c5 | 2348 | { |
48cca7e4 | 2349 | int i, err; |
8a47a6c5 | 2350 | |
2993e051 YS |
2351 | if (!prog) |
2352 | return 0; | |
2353 | ||
f0187f0b MKL |
2354 | if (obj->btf_ext) { |
2355 | err = bpf_program_reloc_btf_ext(prog, obj, | |
2356 | prog->section_name, 0); | |
2357 | if (err) | |
2993e051 | 2358 | return err; |
2993e051 YS |
2359 | } |
2360 | ||
2361 | if (!prog->reloc_desc) | |
8a47a6c5 WN |
2362 | return 0; |
2363 | ||
2364 | for (i = 0; i < prog->nr_reloc; i++) { | |
d859900c DB |
2365 | if (prog->reloc_desc[i].type == RELO_LD64 || |
2366 | prog->reloc_desc[i].type == RELO_DATA) { | |
2367 | bool relo_data = prog->reloc_desc[i].type == RELO_DATA; | |
48cca7e4 AS |
2368 | struct bpf_insn *insns = prog->insns; |
2369 | int insn_idx, map_idx; | |
8a47a6c5 | 2370 | |
48cca7e4 AS |
2371 | insn_idx = prog->reloc_desc[i].insn_idx; |
2372 | map_idx = prog->reloc_desc[i].map_idx; | |
8a47a6c5 | 2373 | |
d859900c | 2374 | if (insn_idx + 1 >= (int)prog->insns_cnt) { |
48cca7e4 AS |
2375 | pr_warning("relocation out of range: '%s'\n", |
2376 | prog->section_name); | |
2377 | return -LIBBPF_ERRNO__RELOC; | |
2378 | } | |
d859900c DB |
2379 | |
2380 | if (!relo_data) { | |
2381 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
2382 | } else { | |
2383 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; | |
2384 | insns[insn_idx + 1].imm = insns[insn_idx].imm; | |
2385 | } | |
48cca7e4 | 2386 | insns[insn_idx].imm = obj->maps[map_idx].fd; |
f8c7a4d4 | 2387 | } else if (prog->reloc_desc[i].type == RELO_CALL) { |
48cca7e4 AS |
2388 | err = bpf_program__reloc_text(prog, obj, |
2389 | &prog->reloc_desc[i]); | |
2390 | if (err) | |
2391 | return err; | |
8a47a6c5 | 2392 | } |
8a47a6c5 WN |
2393 | } |
2394 | ||
2395 | zfree(&prog->reloc_desc); | |
2396 | prog->nr_reloc = 0; | |
2397 | return 0; | |
2398 | } | |
2399 | ||
2400 | ||
2401 | static int | |
2402 | bpf_object__relocate(struct bpf_object *obj) | |
2403 | { | |
2404 | struct bpf_program *prog; | |
2405 | size_t i; | |
2406 | int err; | |
2407 | ||
2408 | for (i = 0; i < obj->nr_programs; i++) { | |
2409 | prog = &obj->programs[i]; | |
2410 | ||
9d759a9b | 2411 | err = bpf_program__relocate(prog, obj); |
8a47a6c5 WN |
2412 | if (err) { |
2413 | pr_warning("failed to relocate '%s'\n", | |
2414 | prog->section_name); | |
2415 | return err; | |
2416 | } | |
2417 | } | |
2418 | return 0; | |
2419 | } | |
2420 | ||
34090915 WN |
2421 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
2422 | { | |
2423 | int i, err; | |
2424 | ||
2425 | if (!obj_elf_valid(obj)) { | |
2426 | pr_warning("Internal error: elf object is closed\n"); | |
6371ca3b | 2427 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
2428 | } |
2429 | ||
2430 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
2431 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
2432 | Elf_Data *data = obj->efile.reloc[i].data; | |
2433 | int idx = shdr->sh_info; | |
2434 | struct bpf_program *prog; | |
34090915 WN |
2435 | |
2436 | if (shdr->sh_type != SHT_REL) { | |
2437 | pr_warning("internal error at %d\n", __LINE__); | |
6371ca3b | 2438 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
2439 | } |
2440 | ||
2441 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
2442 | if (!prog) { | |
077c066a | 2443 | pr_warning("relocation failed: no section(%d)\n", idx); |
6371ca3b | 2444 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
2445 | } |
2446 | ||
399dc65e | 2447 | err = bpf_program__collect_reloc(prog, shdr, data, obj); |
34090915 | 2448 | if (err) |
6371ca3b | 2449 | return err; |
34090915 WN |
2450 | } |
2451 | return 0; | |
2452 | } | |
2453 | ||
55cffde2 | 2454 | static int |
2993e051 | 2455 | load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, |
f0187f0b | 2456 | char *license, __u32 kern_version, int *pfd) |
55cffde2 | 2457 | { |
d7be143b | 2458 | struct bpf_load_program_attr load_attr; |
1ce6a9fc | 2459 | char *cp, errmsg[STRERR_BUFSIZE]; |
da11b417 | 2460 | int log_buf_size = BPF_LOG_BUF_SIZE; |
55cffde2 | 2461 | char *log_buf; |
d7be143b | 2462 | int ret; |
55cffde2 | 2463 | |
fba01a06 AN |
2464 | if (!insns || !insns_cnt) |
2465 | return -EINVAL; | |
2466 | ||
d7be143b | 2467 | memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); |
2993e051 YS |
2468 | load_attr.prog_type = prog->type; |
2469 | load_attr.expected_attach_type = prog->expected_attach_type; | |
5b32a23e SF |
2470 | if (prog->caps->name) |
2471 | load_attr.name = prog->name; | |
d7be143b AI |
2472 | load_attr.insns = insns; |
2473 | load_attr.insns_cnt = insns_cnt; | |
2474 | load_attr.license = license; | |
2475 | load_attr.kern_version = kern_version; | |
2993e051 | 2476 | load_attr.prog_ifindex = prog->prog_ifindex; |
e55d54f4 | 2477 | load_attr.prog_btf_fd = prog->btf_fd >= 0 ? prog->btf_fd : 0; |
2993e051 YS |
2478 | load_attr.func_info = prog->func_info; |
2479 | load_attr.func_info_rec_size = prog->func_info_rec_size; | |
f0187f0b | 2480 | load_attr.func_info_cnt = prog->func_info_cnt; |
3d650141 MKL |
2481 | load_attr.line_info = prog->line_info; |
2482 | load_attr.line_info_rec_size = prog->line_info_rec_size; | |
2483 | load_attr.line_info_cnt = prog->line_info_cnt; | |
da11b417 | 2484 | load_attr.log_level = prog->log_level; |
04656198 | 2485 | load_attr.prog_flags = prog->prog_flags; |
55cffde2 | 2486 | |
da11b417 AS |
2487 | retry_load: |
2488 | log_buf = malloc(log_buf_size); | |
55cffde2 WN |
2489 | if (!log_buf) |
2490 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
2491 | ||
da11b417 | 2492 | ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); |
55cffde2 WN |
2493 | |
2494 | if (ret >= 0) { | |
da11b417 AS |
2495 | if (load_attr.log_level) |
2496 | pr_debug("verifier log:\n%s", log_buf); | |
55cffde2 WN |
2497 | *pfd = ret; |
2498 | ret = 0; | |
2499 | goto out; | |
2500 | } | |
2501 | ||
da11b417 AS |
2502 | if (errno == ENOSPC) { |
2503 | log_buf_size <<= 1; | |
2504 | free(log_buf); | |
2505 | goto retry_load; | |
2506 | } | |
6371ca3b | 2507 | ret = -LIBBPF_ERRNO__LOAD; |
24d6a808 | 2508 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 2509 | pr_warning("load bpf program failed: %s\n", cp); |
55cffde2 | 2510 | |
6371ca3b WN |
2511 | if (log_buf && log_buf[0] != '\0') { |
2512 | ret = -LIBBPF_ERRNO__VERIFY; | |
55cffde2 WN |
2513 | pr_warning("-- BEGIN DUMP LOG ---\n"); |
2514 | pr_warning("\n%s\n", log_buf); | |
2515 | pr_warning("-- END LOG --\n"); | |
d7be143b AI |
2516 | } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { |
2517 | pr_warning("Program too large (%zu insns), at most %d insns\n", | |
2518 | load_attr.insns_cnt, BPF_MAXINSNS); | |
705fa219 | 2519 | ret = -LIBBPF_ERRNO__PROG2BIG; |
6371ca3b | 2520 | } else { |
705fa219 | 2521 | /* Wrong program type? */ |
d7be143b | 2522 | if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { |
705fa219 WN |
2523 | int fd; |
2524 | ||
d7be143b AI |
2525 | load_attr.prog_type = BPF_PROG_TYPE_KPROBE; |
2526 | load_attr.expected_attach_type = 0; | |
2527 | fd = bpf_load_program_xattr(&load_attr, NULL, 0); | |
705fa219 WN |
2528 | if (fd >= 0) { |
2529 | close(fd); | |
2530 | ret = -LIBBPF_ERRNO__PROGTYPE; | |
2531 | goto out; | |
2532 | } | |
6371ca3b | 2533 | } |
705fa219 WN |
2534 | |
2535 | if (log_buf) | |
2536 | ret = -LIBBPF_ERRNO__KVER; | |
55cffde2 WN |
2537 | } |
2538 | ||
2539 | out: | |
2540 | free(log_buf); | |
2541 | return ret; | |
2542 | } | |
2543 | ||
29cd77f4 | 2544 | int |
55cffde2 | 2545 | bpf_program__load(struct bpf_program *prog, |
e5b0863c | 2546 | char *license, __u32 kern_version) |
55cffde2 | 2547 | { |
b580563e | 2548 | int err = 0, fd, i; |
55cffde2 | 2549 | |
b580563e WN |
2550 | if (prog->instances.nr < 0 || !prog->instances.fds) { |
2551 | if (prog->preprocessor) { | |
2552 | pr_warning("Internal error: can't load program '%s'\n", | |
2553 | prog->section_name); | |
2554 | return -LIBBPF_ERRNO__INTERNAL; | |
2555 | } | |
55cffde2 | 2556 | |
b580563e WN |
2557 | prog->instances.fds = malloc(sizeof(int)); |
2558 | if (!prog->instances.fds) { | |
2559 | pr_warning("Not enough memory for BPF fds\n"); | |
2560 | return -ENOMEM; | |
2561 | } | |
2562 | prog->instances.nr = 1; | |
2563 | prog->instances.fds[0] = -1; | |
2564 | } | |
2565 | ||
2566 | if (!prog->preprocessor) { | |
2567 | if (prog->instances.nr != 1) { | |
2568 | pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", | |
2569 | prog->section_name, prog->instances.nr); | |
2570 | } | |
2993e051 | 2571 | err = load_program(prog, prog->insns, prog->insns_cnt, |
f0187f0b | 2572 | license, kern_version, &fd); |
b580563e WN |
2573 | if (!err) |
2574 | prog->instances.fds[0] = fd; | |
2575 | goto out; | |
2576 | } | |
2577 | ||
2578 | for (i = 0; i < prog->instances.nr; i++) { | |
2579 | struct bpf_prog_prep_result result; | |
2580 | bpf_program_prep_t preprocessor = prog->preprocessor; | |
2581 | ||
1ad9cbb8 | 2582 | memset(&result, 0, sizeof(result)); |
b580563e WN |
2583 | err = preprocessor(prog, i, prog->insns, |
2584 | prog->insns_cnt, &result); | |
2585 | if (err) { | |
2586 | pr_warning("Preprocessing the %dth instance of program '%s' failed\n", | |
2587 | i, prog->section_name); | |
2588 | goto out; | |
2589 | } | |
2590 | ||
2591 | if (!result.new_insn_ptr || !result.new_insn_cnt) { | |
2592 | pr_debug("Skip loading the %dth instance of program '%s'\n", | |
2593 | i, prog->section_name); | |
2594 | prog->instances.fds[i] = -1; | |
2595 | if (result.pfd) | |
2596 | *result.pfd = -1; | |
2597 | continue; | |
2598 | } | |
2599 | ||
2993e051 | 2600 | err = load_program(prog, result.new_insn_ptr, |
b580563e | 2601 | result.new_insn_cnt, |
f0187f0b | 2602 | license, kern_version, &fd); |
b580563e WN |
2603 | |
2604 | if (err) { | |
2605 | pr_warning("Loading the %dth instance of program '%s' failed\n", | |
2606 | i, prog->section_name); | |
2607 | goto out; | |
2608 | } | |
2609 | ||
2610 | if (result.pfd) | |
2611 | *result.pfd = fd; | |
2612 | prog->instances.fds[i] = fd; | |
2613 | } | |
2614 | out: | |
55cffde2 WN |
2615 | if (err) |
2616 | pr_warning("failed to load program '%s'\n", | |
2617 | prog->section_name); | |
2618 | zfree(&prog->insns); | |
2619 | prog->insns_cnt = 0; | |
2620 | return err; | |
2621 | } | |
2622 | ||
a324aae3 AN |
2623 | static bool bpf_program__is_function_storage(const struct bpf_program *prog, |
2624 | const struct bpf_object *obj) | |
9a94f277 JK |
2625 | { |
2626 | return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; | |
2627 | } | |
2628 | ||
55cffde2 | 2629 | static int |
60276f98 | 2630 | bpf_object__load_progs(struct bpf_object *obj, int log_level) |
55cffde2 WN |
2631 | { |
2632 | size_t i; | |
2633 | int err; | |
2634 | ||
2635 | for (i = 0; i < obj->nr_programs; i++) { | |
9a94f277 | 2636 | if (bpf_program__is_function_storage(&obj->programs[i], obj)) |
48cca7e4 | 2637 | continue; |
501b125a | 2638 | obj->programs[i].log_level |= log_level; |
55cffde2 WN |
2639 | err = bpf_program__load(&obj->programs[i], |
2640 | obj->license, | |
2641 | obj->kern_version); | |
2642 | if (err) | |
2643 | return err; | |
2644 | } | |
2645 | return 0; | |
2646 | } | |
2647 | ||
17387dd5 JK |
2648 | static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) |
2649 | { | |
2650 | switch (type) { | |
2651 | case BPF_PROG_TYPE_SOCKET_FILTER: | |
2652 | case BPF_PROG_TYPE_SCHED_CLS: | |
2653 | case BPF_PROG_TYPE_SCHED_ACT: | |
2654 | case BPF_PROG_TYPE_XDP: | |
2655 | case BPF_PROG_TYPE_CGROUP_SKB: | |
2656 | case BPF_PROG_TYPE_CGROUP_SOCK: | |
2657 | case BPF_PROG_TYPE_LWT_IN: | |
2658 | case BPF_PROG_TYPE_LWT_OUT: | |
2659 | case BPF_PROG_TYPE_LWT_XMIT: | |
004d4b27 | 2660 | case BPF_PROG_TYPE_LWT_SEG6LOCAL: |
17387dd5 JK |
2661 | case BPF_PROG_TYPE_SOCK_OPS: |
2662 | case BPF_PROG_TYPE_SK_SKB: | |
2663 | case BPF_PROG_TYPE_CGROUP_DEVICE: | |
2664 | case BPF_PROG_TYPE_SK_MSG: | |
2665 | case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: | |
6bdd533c | 2666 | case BPF_PROG_TYPE_LIRC_MODE2: |
6bc8529c | 2667 | case BPF_PROG_TYPE_SK_REUSEPORT: |
c22fbae7 | 2668 | case BPF_PROG_TYPE_FLOW_DISSECTOR: |
17387dd5 | 2669 | case BPF_PROG_TYPE_UNSPEC: |
17387dd5 | 2670 | case BPF_PROG_TYPE_TRACEPOINT: |
17387dd5 | 2671 | case BPF_PROG_TYPE_RAW_TRACEPOINT: |
4635b0ae | 2672 | case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE: |
47ae7e3d | 2673 | case BPF_PROG_TYPE_PERF_EVENT: |
063cc9f0 | 2674 | case BPF_PROG_TYPE_CGROUP_SYSCTL: |
4cdbfb59 | 2675 | case BPF_PROG_TYPE_CGROUP_SOCKOPT: |
47ae7e3d NS |
2676 | return false; |
2677 | case BPF_PROG_TYPE_KPROBE: | |
17387dd5 JK |
2678 | default: |
2679 | return true; | |
2680 | } | |
2681 | } | |
2682 | ||
2683 | static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) | |
cb1e5e96 | 2684 | { |
17387dd5 | 2685 | if (needs_kver && obj->kern_version == 0) { |
cb1e5e96 WN |
2686 | pr_warning("%s doesn't provide kernel version\n", |
2687 | obj->path); | |
6371ca3b | 2688 | return -LIBBPF_ERRNO__KVERSION; |
cb1e5e96 WN |
2689 | } |
2690 | return 0; | |
2691 | } | |
2692 | ||
1a5e3fb1 | 2693 | static struct bpf_object * |
17387dd5 | 2694 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, |
c034a177 | 2695 | bool needs_kver, int flags) |
1a5e3fb1 WN |
2696 | { |
2697 | struct bpf_object *obj; | |
6371ca3b | 2698 | int err; |
1a5e3fb1 WN |
2699 | |
2700 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
2701 | pr_warning("failed to init libelf for %s\n", path); | |
6371ca3b | 2702 | return ERR_PTR(-LIBBPF_ERRNO__LIBELF); |
1a5e3fb1 WN |
2703 | } |
2704 | ||
6c956392 | 2705 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
6371ca3b WN |
2706 | if (IS_ERR(obj)) |
2707 | return obj; | |
1a5e3fb1 | 2708 | |
6371ca3b WN |
2709 | CHECK_ERR(bpf_object__elf_init(obj), err, out); |
2710 | CHECK_ERR(bpf_object__check_endianness(obj), err, out); | |
8837fe5d | 2711 | CHECK_ERR(bpf_object__probe_caps(obj), err, out); |
c034a177 | 2712 | CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); |
6371ca3b | 2713 | CHECK_ERR(bpf_object__collect_reloc(obj), err, out); |
17387dd5 | 2714 | CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); |
1a5e3fb1 WN |
2715 | |
2716 | bpf_object__elf_finish(obj); | |
2717 | return obj; | |
2718 | out: | |
2719 | bpf_object__close(obj); | |
6371ca3b | 2720 | return ERR_PTR(err); |
1a5e3fb1 WN |
2721 | } |
2722 | ||
c034a177 JF |
2723 | struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, |
2724 | int flags) | |
1a5e3fb1 WN |
2725 | { |
2726 | /* param validation */ | |
07f2d4ea | 2727 | if (!attr->file) |
1a5e3fb1 WN |
2728 | return NULL; |
2729 | ||
07f2d4ea JK |
2730 | pr_debug("loading %s\n", attr->file); |
2731 | ||
2732 | return __bpf_object__open(attr->file, NULL, 0, | |
c034a177 JF |
2733 | bpf_prog_type__needs_kver(attr->prog_type), |
2734 | flags); | |
2735 | } | |
2736 | ||
2737 | struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) | |
2738 | { | |
2739 | return __bpf_object__open_xattr(attr, 0); | |
07f2d4ea JK |
2740 | } |
2741 | ||
2742 | struct bpf_object *bpf_object__open(const char *path) | |
2743 | { | |
2744 | struct bpf_object_open_attr attr = { | |
2745 | .file = path, | |
2746 | .prog_type = BPF_PROG_TYPE_UNSPEC, | |
2747 | }; | |
1a5e3fb1 | 2748 | |
07f2d4ea | 2749 | return bpf_object__open_xattr(&attr); |
6c956392 WN |
2750 | } |
2751 | ||
2752 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
acf860ae WN |
2753 | size_t obj_buf_sz, |
2754 | const char *name) | |
6c956392 | 2755 | { |
acf860ae WN |
2756 | char tmp_name[64]; |
2757 | ||
6c956392 WN |
2758 | /* param validation */ |
2759 | if (!obj_buf || obj_buf_sz <= 0) | |
2760 | return NULL; | |
2761 | ||
acf860ae WN |
2762 | if (!name) { |
2763 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
2764 | (unsigned long)obj_buf, | |
2765 | (unsigned long)obj_buf_sz); | |
acf860ae WN |
2766 | name = tmp_name; |
2767 | } | |
399dc65e | 2768 | pr_debug("loading object '%s' from buffer\n", name); |
6c956392 | 2769 | |
c034a177 | 2770 | return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); |
1a5e3fb1 WN |
2771 | } |
2772 | ||
52d3352e WN |
2773 | int bpf_object__unload(struct bpf_object *obj) |
2774 | { | |
2775 | size_t i; | |
2776 | ||
2777 | if (!obj) | |
2778 | return -EINVAL; | |
2779 | ||
9d759a9b WN |
2780 | for (i = 0; i < obj->nr_maps; i++) |
2781 | zclose(obj->maps[i].fd); | |
52d3352e | 2782 | |
55cffde2 WN |
2783 | for (i = 0; i < obj->nr_programs; i++) |
2784 | bpf_program__unload(&obj->programs[i]); | |
2785 | ||
52d3352e WN |
2786 | return 0; |
2787 | } | |
2788 | ||
60276f98 | 2789 | int bpf_object__load_xattr(struct bpf_object_load_attr *attr) |
52d3352e | 2790 | { |
60276f98 | 2791 | struct bpf_object *obj; |
6371ca3b WN |
2792 | int err; |
2793 | ||
60276f98 QM |
2794 | if (!attr) |
2795 | return -EINVAL; | |
2796 | obj = attr->obj; | |
52d3352e WN |
2797 | if (!obj) |
2798 | return -EINVAL; | |
2799 | ||
2800 | if (obj->loaded) { | |
2801 | pr_warning("object should not be loaded twice\n"); | |
2802 | return -EINVAL; | |
2803 | } | |
2804 | ||
2805 | obj->loaded = true; | |
6371ca3b WN |
2806 | |
2807 | CHECK_ERR(bpf_object__create_maps(obj), err, out); | |
2808 | CHECK_ERR(bpf_object__relocate(obj), err, out); | |
60276f98 | 2809 | CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out); |
52d3352e WN |
2810 | |
2811 | return 0; | |
2812 | out: | |
2813 | bpf_object__unload(obj); | |
2814 | pr_warning("failed to load object '%s'\n", obj->path); | |
6371ca3b | 2815 | return err; |
52d3352e WN |
2816 | } |
2817 | ||
60276f98 QM |
2818 | int bpf_object__load(struct bpf_object *obj) |
2819 | { | |
2820 | struct bpf_object_load_attr attr = { | |
2821 | .obj = obj, | |
2822 | }; | |
2823 | ||
2824 | return bpf_object__load_xattr(&attr); | |
2825 | } | |
2826 | ||
f367540c JS |
2827 | static int check_path(const char *path) |
2828 | { | |
1ce6a9fc | 2829 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
2830 | struct statfs st_fs; |
2831 | char *dname, *dir; | |
2832 | int err = 0; | |
2833 | ||
2834 | if (path == NULL) | |
2835 | return -EINVAL; | |
2836 | ||
2837 | dname = strdup(path); | |
2838 | if (dname == NULL) | |
2839 | return -ENOMEM; | |
2840 | ||
2841 | dir = dirname(dname); | |
2842 | if (statfs(dir, &st_fs)) { | |
24d6a808 | 2843 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 2844 | pr_warning("failed to statfs %s: %s\n", dir, cp); |
f367540c JS |
2845 | err = -errno; |
2846 | } | |
2847 | free(dname); | |
2848 | ||
2849 | if (!err && st_fs.f_type != BPF_FS_MAGIC) { | |
2850 | pr_warning("specified path %s is not on BPF FS\n", path); | |
2851 | err = -EINVAL; | |
2852 | } | |
2853 | ||
2854 | return err; | |
2855 | } | |
2856 | ||
2857 | int bpf_program__pin_instance(struct bpf_program *prog, const char *path, | |
2858 | int instance) | |
2859 | { | |
1ce6a9fc | 2860 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
2861 | int err; |
2862 | ||
2863 | err = check_path(path); | |
2864 | if (err) | |
2865 | return err; | |
2866 | ||
2867 | if (prog == NULL) { | |
2868 | pr_warning("invalid program pointer\n"); | |
2869 | return -EINVAL; | |
2870 | } | |
2871 | ||
2872 | if (instance < 0 || instance >= prog->instances.nr) { | |
2873 | pr_warning("invalid prog instance %d of prog %s (max %d)\n", | |
2874 | instance, prog->section_name, prog->instances.nr); | |
2875 | return -EINVAL; | |
2876 | } | |
2877 | ||
2878 | if (bpf_obj_pin(prog->instances.fds[instance], path)) { | |
24d6a808 | 2879 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 2880 | pr_warning("failed to pin program: %s\n", cp); |
f367540c JS |
2881 | return -errno; |
2882 | } | |
2883 | pr_debug("pinned program '%s'\n", path); | |
2884 | ||
2885 | return 0; | |
2886 | } | |
2887 | ||
0c19a9fb SF |
2888 | int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, |
2889 | int instance) | |
2890 | { | |
2891 | int err; | |
2892 | ||
2893 | err = check_path(path); | |
2894 | if (err) | |
2895 | return err; | |
2896 | ||
2897 | if (prog == NULL) { | |
2898 | pr_warning("invalid program pointer\n"); | |
2899 | return -EINVAL; | |
2900 | } | |
2901 | ||
2902 | if (instance < 0 || instance >= prog->instances.nr) { | |
2903 | pr_warning("invalid prog instance %d of prog %s (max %d)\n", | |
2904 | instance, prog->section_name, prog->instances.nr); | |
2905 | return -EINVAL; | |
2906 | } | |
2907 | ||
2908 | err = unlink(path); | |
2909 | if (err != 0) | |
2910 | return -errno; | |
2911 | pr_debug("unpinned program '%s'\n", path); | |
2912 | ||
2913 | return 0; | |
2914 | } | |
2915 | ||
f367540c JS |
2916 | static int make_dir(const char *path) |
2917 | { | |
1ce6a9fc | 2918 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
2919 | int err = 0; |
2920 | ||
2921 | if (mkdir(path, 0700) && errno != EEXIST) | |
2922 | err = -errno; | |
2923 | ||
1ce6a9fc | 2924 | if (err) { |
24d6a808 | 2925 | cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); |
1ce6a9fc TR |
2926 | pr_warning("failed to mkdir %s: %s\n", path, cp); |
2927 | } | |
f367540c JS |
2928 | return err; |
2929 | } | |
2930 | ||
2931 | int bpf_program__pin(struct bpf_program *prog, const char *path) | |
2932 | { | |
2933 | int i, err; | |
2934 | ||
2935 | err = check_path(path); | |
2936 | if (err) | |
2937 | return err; | |
2938 | ||
2939 | if (prog == NULL) { | |
2940 | pr_warning("invalid program pointer\n"); | |
2941 | return -EINVAL; | |
2942 | } | |
2943 | ||
2944 | if (prog->instances.nr <= 0) { | |
2945 | pr_warning("no instances of prog %s to pin\n", | |
2946 | prog->section_name); | |
2947 | return -EINVAL; | |
2948 | } | |
2949 | ||
fd734c5c SF |
2950 | if (prog->instances.nr == 1) { |
2951 | /* don't create subdirs when pinning single instance */ | |
2952 | return bpf_program__pin_instance(prog, path, 0); | |
2953 | } | |
2954 | ||
f367540c JS |
2955 | err = make_dir(path); |
2956 | if (err) | |
2957 | return err; | |
2958 | ||
0c19a9fb SF |
2959 | for (i = 0; i < prog->instances.nr; i++) { |
2960 | char buf[PATH_MAX]; | |
2961 | int len; | |
2962 | ||
2963 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
2964 | if (len < 0) { | |
2965 | err = -EINVAL; | |
2966 | goto err_unpin; | |
2967 | } else if (len >= PATH_MAX) { | |
2968 | err = -ENAMETOOLONG; | |
2969 | goto err_unpin; | |
2970 | } | |
2971 | ||
2972 | err = bpf_program__pin_instance(prog, buf, i); | |
2973 | if (err) | |
2974 | goto err_unpin; | |
2975 | } | |
2976 | ||
2977 | return 0; | |
2978 | ||
2979 | err_unpin: | |
2980 | for (i = i - 1; i >= 0; i--) { | |
2981 | char buf[PATH_MAX]; | |
2982 | int len; | |
2983 | ||
2984 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
2985 | if (len < 0) | |
2986 | continue; | |
2987 | else if (len >= PATH_MAX) | |
2988 | continue; | |
2989 | ||
2990 | bpf_program__unpin_instance(prog, buf, i); | |
2991 | } | |
2992 | ||
2993 | rmdir(path); | |
2994 | ||
2995 | return err; | |
2996 | } | |
2997 | ||
2998 | int bpf_program__unpin(struct bpf_program *prog, const char *path) | |
2999 | { | |
3000 | int i, err; | |
3001 | ||
3002 | err = check_path(path); | |
3003 | if (err) | |
3004 | return err; | |
3005 | ||
3006 | if (prog == NULL) { | |
3007 | pr_warning("invalid program pointer\n"); | |
3008 | return -EINVAL; | |
3009 | } | |
3010 | ||
3011 | if (prog->instances.nr <= 0) { | |
3012 | pr_warning("no instances of prog %s to pin\n", | |
3013 | prog->section_name); | |
3014 | return -EINVAL; | |
fd734c5c SF |
3015 | } |
3016 | ||
3017 | if (prog->instances.nr == 1) { | |
3018 | /* don't create subdirs when pinning single instance */ | |
3019 | return bpf_program__unpin_instance(prog, path, 0); | |
0c19a9fb SF |
3020 | } |
3021 | ||
f367540c JS |
3022 | for (i = 0; i < prog->instances.nr; i++) { |
3023 | char buf[PATH_MAX]; | |
3024 | int len; | |
3025 | ||
3026 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
3027 | if (len < 0) | |
3028 | return -EINVAL; | |
3029 | else if (len >= PATH_MAX) | |
3030 | return -ENAMETOOLONG; | |
3031 | ||
0c19a9fb | 3032 | err = bpf_program__unpin_instance(prog, buf, i); |
f367540c JS |
3033 | if (err) |
3034 | return err; | |
3035 | } | |
3036 | ||
0c19a9fb SF |
3037 | err = rmdir(path); |
3038 | if (err) | |
3039 | return -errno; | |
3040 | ||
f367540c JS |
3041 | return 0; |
3042 | } | |
3043 | ||
b6989f35 JS |
3044 | int bpf_map__pin(struct bpf_map *map, const char *path) |
3045 | { | |
1ce6a9fc | 3046 | char *cp, errmsg[STRERR_BUFSIZE]; |
b6989f35 JS |
3047 | int err; |
3048 | ||
3049 | err = check_path(path); | |
3050 | if (err) | |
3051 | return err; | |
3052 | ||
3053 | if (map == NULL) { | |
3054 | pr_warning("invalid map pointer\n"); | |
3055 | return -EINVAL; | |
3056 | } | |
3057 | ||
3058 | if (bpf_obj_pin(map->fd, path)) { | |
24d6a808 | 3059 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 3060 | pr_warning("failed to pin map: %s\n", cp); |
b6989f35 JS |
3061 | return -errno; |
3062 | } | |
3063 | ||
3064 | pr_debug("pinned map '%s'\n", path); | |
0c19a9fb | 3065 | |
b6989f35 JS |
3066 | return 0; |
3067 | } | |
3068 | ||
0c19a9fb SF |
3069 | int bpf_map__unpin(struct bpf_map *map, const char *path) |
3070 | { | |
3071 | int err; | |
3072 | ||
3073 | err = check_path(path); | |
3074 | if (err) | |
3075 | return err; | |
3076 | ||
3077 | if (map == NULL) { | |
3078 | pr_warning("invalid map pointer\n"); | |
3079 | return -EINVAL; | |
3080 | } | |
3081 | ||
3082 | err = unlink(path); | |
3083 | if (err != 0) | |
3084 | return -errno; | |
3085 | pr_debug("unpinned map '%s'\n", path); | |
3086 | ||
3087 | return 0; | |
3088 | } | |
3089 | ||
3090 | int bpf_object__pin_maps(struct bpf_object *obj, const char *path) | |
d5148d85 | 3091 | { |
d5148d85 JS |
3092 | struct bpf_map *map; |
3093 | int err; | |
3094 | ||
3095 | if (!obj) | |
3096 | return -ENOENT; | |
3097 | ||
3098 | if (!obj->loaded) { | |
3099 | pr_warning("object not yet loaded; load it first\n"); | |
3100 | return -ENOENT; | |
3101 | } | |
3102 | ||
3103 | err = make_dir(path); | |
3104 | if (err) | |
3105 | return err; | |
3106 | ||
f74a53d9 | 3107 | bpf_object__for_each_map(map, obj) { |
0c19a9fb SF |
3108 | char buf[PATH_MAX]; |
3109 | int len; | |
3110 | ||
3111 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
3112 | bpf_map__name(map)); | |
3113 | if (len < 0) { | |
3114 | err = -EINVAL; | |
3115 | goto err_unpin_maps; | |
3116 | } else if (len >= PATH_MAX) { | |
3117 | err = -ENAMETOOLONG; | |
3118 | goto err_unpin_maps; | |
3119 | } | |
3120 | ||
3121 | err = bpf_map__pin(map, buf); | |
3122 | if (err) | |
3123 | goto err_unpin_maps; | |
3124 | } | |
3125 | ||
3126 | return 0; | |
3127 | ||
3128 | err_unpin_maps: | |
3129 | while ((map = bpf_map__prev(map, obj))) { | |
3130 | char buf[PATH_MAX]; | |
3131 | int len; | |
3132 | ||
3133 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
3134 | bpf_map__name(map)); | |
3135 | if (len < 0) | |
3136 | continue; | |
3137 | else if (len >= PATH_MAX) | |
3138 | continue; | |
3139 | ||
3140 | bpf_map__unpin(map, buf); | |
3141 | } | |
3142 | ||
3143 | return err; | |
3144 | } | |
3145 | ||
3146 | int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) | |
3147 | { | |
3148 | struct bpf_map *map; | |
3149 | int err; | |
3150 | ||
3151 | if (!obj) | |
3152 | return -ENOENT; | |
3153 | ||
f74a53d9 | 3154 | bpf_object__for_each_map(map, obj) { |
d5148d85 JS |
3155 | char buf[PATH_MAX]; |
3156 | int len; | |
3157 | ||
3158 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
3159 | bpf_map__name(map)); | |
3160 | if (len < 0) | |
3161 | return -EINVAL; | |
3162 | else if (len >= PATH_MAX) | |
3163 | return -ENAMETOOLONG; | |
3164 | ||
0c19a9fb | 3165 | err = bpf_map__unpin(map, buf); |
d5148d85 JS |
3166 | if (err) |
3167 | return err; | |
3168 | } | |
3169 | ||
0c19a9fb SF |
3170 | return 0; |
3171 | } | |
3172 | ||
3173 | int bpf_object__pin_programs(struct bpf_object *obj, const char *path) | |
3174 | { | |
3175 | struct bpf_program *prog; | |
3176 | int err; | |
3177 | ||
3178 | if (!obj) | |
3179 | return -ENOENT; | |
3180 | ||
3181 | if (!obj->loaded) { | |
3182 | pr_warning("object not yet loaded; load it first\n"); | |
3183 | return -ENOENT; | |
3184 | } | |
3185 | ||
3186 | err = make_dir(path); | |
3187 | if (err) | |
3188 | return err; | |
3189 | ||
3190 | bpf_object__for_each_program(prog, obj) { | |
3191 | char buf[PATH_MAX]; | |
3192 | int len; | |
3193 | ||
3194 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 3195 | prog->pin_name); |
0c19a9fb SF |
3196 | if (len < 0) { |
3197 | err = -EINVAL; | |
3198 | goto err_unpin_programs; | |
3199 | } else if (len >= PATH_MAX) { | |
3200 | err = -ENAMETOOLONG; | |
3201 | goto err_unpin_programs; | |
3202 | } | |
3203 | ||
3204 | err = bpf_program__pin(prog, buf); | |
3205 | if (err) | |
3206 | goto err_unpin_programs; | |
3207 | } | |
3208 | ||
3209 | return 0; | |
3210 | ||
3211 | err_unpin_programs: | |
3212 | while ((prog = bpf_program__prev(prog, obj))) { | |
3213 | char buf[PATH_MAX]; | |
3214 | int len; | |
3215 | ||
3216 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 3217 | prog->pin_name); |
0c19a9fb SF |
3218 | if (len < 0) |
3219 | continue; | |
3220 | else if (len >= PATH_MAX) | |
3221 | continue; | |
3222 | ||
3223 | bpf_program__unpin(prog, buf); | |
3224 | } | |
3225 | ||
3226 | return err; | |
3227 | } | |
3228 | ||
3229 | int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) | |
3230 | { | |
3231 | struct bpf_program *prog; | |
3232 | int err; | |
3233 | ||
3234 | if (!obj) | |
3235 | return -ENOENT; | |
3236 | ||
d5148d85 JS |
3237 | bpf_object__for_each_program(prog, obj) { |
3238 | char buf[PATH_MAX]; | |
3239 | int len; | |
3240 | ||
3241 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 3242 | prog->pin_name); |
d5148d85 JS |
3243 | if (len < 0) |
3244 | return -EINVAL; | |
3245 | else if (len >= PATH_MAX) | |
3246 | return -ENAMETOOLONG; | |
3247 | ||
0c19a9fb | 3248 | err = bpf_program__unpin(prog, buf); |
d5148d85 JS |
3249 | if (err) |
3250 | return err; | |
3251 | } | |
3252 | ||
3253 | return 0; | |
3254 | } | |
3255 | ||
0c19a9fb SF |
3256 | int bpf_object__pin(struct bpf_object *obj, const char *path) |
3257 | { | |
3258 | int err; | |
3259 | ||
3260 | err = bpf_object__pin_maps(obj, path); | |
3261 | if (err) | |
3262 | return err; | |
3263 | ||
3264 | err = bpf_object__pin_programs(obj, path); | |
3265 | if (err) { | |
3266 | bpf_object__unpin_maps(obj, path); | |
3267 | return err; | |
3268 | } | |
3269 | ||
3270 | return 0; | |
3271 | } | |
3272 | ||
1a5e3fb1 WN |
3273 | void bpf_object__close(struct bpf_object *obj) |
3274 | { | |
a5b8bd47 WN |
3275 | size_t i; |
3276 | ||
1a5e3fb1 WN |
3277 | if (!obj) |
3278 | return; | |
3279 | ||
10931d24 WN |
3280 | if (obj->clear_priv) |
3281 | obj->clear_priv(obj, obj->priv); | |
3282 | ||
1a5e3fb1 | 3283 | bpf_object__elf_finish(obj); |
52d3352e | 3284 | bpf_object__unload(obj); |
8a138aed | 3285 | btf__free(obj->btf); |
2993e051 | 3286 | btf_ext__free(obj->btf_ext); |
1a5e3fb1 | 3287 | |
9d759a9b | 3288 | for (i = 0; i < obj->nr_maps; i++) { |
561bbcca | 3289 | zfree(&obj->maps[i].name); |
9d759a9b WN |
3290 | if (obj->maps[i].clear_priv) |
3291 | obj->maps[i].clear_priv(&obj->maps[i], | |
3292 | obj->maps[i].priv); | |
3293 | obj->maps[i].priv = NULL; | |
3294 | obj->maps[i].clear_priv = NULL; | |
3295 | } | |
d859900c DB |
3296 | |
3297 | zfree(&obj->sections.rodata); | |
3298 | zfree(&obj->sections.data); | |
9d759a9b WN |
3299 | zfree(&obj->maps); |
3300 | obj->nr_maps = 0; | |
a5b8bd47 WN |
3301 | |
3302 | if (obj->programs && obj->nr_programs) { | |
3303 | for (i = 0; i < obj->nr_programs; i++) | |
3304 | bpf_program__exit(&obj->programs[i]); | |
3305 | } | |
3306 | zfree(&obj->programs); | |
3307 | ||
9a208eff | 3308 | list_del(&obj->list); |
1a5e3fb1 WN |
3309 | free(obj); |
3310 | } | |
aa9b1ac3 | 3311 | |
9a208eff WN |
3312 | struct bpf_object * |
3313 | bpf_object__next(struct bpf_object *prev) | |
3314 | { | |
3315 | struct bpf_object *next; | |
3316 | ||
3317 | if (!prev) | |
3318 | next = list_first_entry(&bpf_objects_list, | |
3319 | struct bpf_object, | |
3320 | list); | |
3321 | else | |
3322 | next = list_next_entry(prev, list); | |
3323 | ||
3324 | /* Empty list is noticed here so don't need checking on entry. */ | |
3325 | if (&next->list == &bpf_objects_list) | |
3326 | return NULL; | |
3327 | ||
3328 | return next; | |
3329 | } | |
3330 | ||
a324aae3 | 3331 | const char *bpf_object__name(const struct bpf_object *obj) |
acf860ae | 3332 | { |
a7fe0450 | 3333 | return obj ? obj->path : ERR_PTR(-EINVAL); |
acf860ae WN |
3334 | } |
3335 | ||
a324aae3 | 3336 | unsigned int bpf_object__kversion(const struct bpf_object *obj) |
45825d8a | 3337 | { |
a7fe0450 | 3338 | return obj ? obj->kern_version : 0; |
45825d8a WN |
3339 | } |
3340 | ||
a324aae3 | 3341 | struct btf *bpf_object__btf(const struct bpf_object *obj) |
789f6bab AI |
3342 | { |
3343 | return obj ? obj->btf : NULL; | |
3344 | } | |
3345 | ||
8a138aed MKL |
3346 | int bpf_object__btf_fd(const struct bpf_object *obj) |
3347 | { | |
3348 | return obj->btf ? btf__fd(obj->btf) : -1; | |
3349 | } | |
3350 | ||
10931d24 WN |
3351 | int bpf_object__set_priv(struct bpf_object *obj, void *priv, |
3352 | bpf_object_clear_priv_t clear_priv) | |
3353 | { | |
3354 | if (obj->priv && obj->clear_priv) | |
3355 | obj->clear_priv(obj, obj->priv); | |
3356 | ||
3357 | obj->priv = priv; | |
3358 | obj->clear_priv = clear_priv; | |
3359 | return 0; | |
3360 | } | |
3361 | ||
a324aae3 | 3362 | void *bpf_object__priv(const struct bpf_object *obj) |
10931d24 WN |
3363 | { |
3364 | return obj ? obj->priv : ERR_PTR(-EINVAL); | |
3365 | } | |
3366 | ||
eac7d845 | 3367 | static struct bpf_program * |
a324aae3 AN |
3368 | __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, |
3369 | bool forward) | |
aa9b1ac3 | 3370 | { |
a83d6e76 | 3371 | size_t nr_programs = obj->nr_programs; |
0c19a9fb | 3372 | ssize_t idx; |
aa9b1ac3 | 3373 | |
a83d6e76 | 3374 | if (!nr_programs) |
aa9b1ac3 | 3375 | return NULL; |
aa9b1ac3 | 3376 | |
a83d6e76 MKL |
3377 | if (!p) |
3378 | /* Iter from the beginning */ | |
3379 | return forward ? &obj->programs[0] : | |
3380 | &obj->programs[nr_programs - 1]; | |
3381 | ||
0c19a9fb | 3382 | if (p->obj != obj) { |
aa9b1ac3 WN |
3383 | pr_warning("error: program handler doesn't match object\n"); |
3384 | return NULL; | |
3385 | } | |
3386 | ||
a83d6e76 | 3387 | idx = (p - obj->programs) + (forward ? 1 : -1); |
0c19a9fb | 3388 | if (idx >= obj->nr_programs || idx < 0) |
aa9b1ac3 WN |
3389 | return NULL; |
3390 | return &obj->programs[idx]; | |
3391 | } | |
3392 | ||
eac7d845 | 3393 | struct bpf_program * |
a324aae3 | 3394 | bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) |
eac7d845 JK |
3395 | { |
3396 | struct bpf_program *prog = prev; | |
3397 | ||
3398 | do { | |
a83d6e76 | 3399 | prog = __bpf_program__iter(prog, obj, true); |
0c19a9fb SF |
3400 | } while (prog && bpf_program__is_function_storage(prog, obj)); |
3401 | ||
3402 | return prog; | |
3403 | } | |
3404 | ||
3405 | struct bpf_program * | |
a324aae3 | 3406 | bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) |
0c19a9fb SF |
3407 | { |
3408 | struct bpf_program *prog = next; | |
3409 | ||
0c19a9fb | 3410 | do { |
a83d6e76 | 3411 | prog = __bpf_program__iter(prog, obj, false); |
eac7d845 JK |
3412 | } while (prog && bpf_program__is_function_storage(prog, obj)); |
3413 | ||
3414 | return prog; | |
3415 | } | |
3416 | ||
edb13ed4 ACM |
3417 | int bpf_program__set_priv(struct bpf_program *prog, void *priv, |
3418 | bpf_program_clear_priv_t clear_priv) | |
aa9b1ac3 WN |
3419 | { |
3420 | if (prog->priv && prog->clear_priv) | |
3421 | prog->clear_priv(prog, prog->priv); | |
3422 | ||
3423 | prog->priv = priv; | |
3424 | prog->clear_priv = clear_priv; | |
3425 | return 0; | |
3426 | } | |
3427 | ||
a324aae3 | 3428 | void *bpf_program__priv(const struct bpf_program *prog) |
aa9b1ac3 | 3429 | { |
be834ffb | 3430 | return prog ? prog->priv : ERR_PTR(-EINVAL); |
aa9b1ac3 WN |
3431 | } |
3432 | ||
9aba3613 JK |
3433 | void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) |
3434 | { | |
3435 | prog->prog_ifindex = ifindex; | |
3436 | } | |
3437 | ||
a324aae3 | 3438 | const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
3439 | { |
3440 | const char *title; | |
3441 | ||
3442 | title = prog->section_name; | |
715f8db9 | 3443 | if (needs_copy) { |
aa9b1ac3 WN |
3444 | title = strdup(title); |
3445 | if (!title) { | |
3446 | pr_warning("failed to strdup program title\n"); | |
6371ca3b | 3447 | return ERR_PTR(-ENOMEM); |
aa9b1ac3 WN |
3448 | } |
3449 | } | |
3450 | ||
3451 | return title; | |
3452 | } | |
3453 | ||
a324aae3 | 3454 | int bpf_program__fd(const struct bpf_program *prog) |
aa9b1ac3 | 3455 | { |
b580563e WN |
3456 | return bpf_program__nth_fd(prog, 0); |
3457 | } | |
3458 | ||
3459 | int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, | |
3460 | bpf_program_prep_t prep) | |
3461 | { | |
3462 | int *instances_fds; | |
3463 | ||
3464 | if (nr_instances <= 0 || !prep) | |
3465 | return -EINVAL; | |
3466 | ||
3467 | if (prog->instances.nr > 0 || prog->instances.fds) { | |
3468 | pr_warning("Can't set pre-processor after loading\n"); | |
3469 | return -EINVAL; | |
3470 | } | |
3471 | ||
3472 | instances_fds = malloc(sizeof(int) * nr_instances); | |
3473 | if (!instances_fds) { | |
3474 | pr_warning("alloc memory failed for fds\n"); | |
3475 | return -ENOMEM; | |
3476 | } | |
3477 | ||
3478 | /* fill all fd with -1 */ | |
3479 | memset(instances_fds, -1, sizeof(int) * nr_instances); | |
3480 | ||
3481 | prog->instances.nr = nr_instances; | |
3482 | prog->instances.fds = instances_fds; | |
3483 | prog->preprocessor = prep; | |
3484 | return 0; | |
3485 | } | |
3486 | ||
a324aae3 | 3487 | int bpf_program__nth_fd(const struct bpf_program *prog, int n) |
b580563e WN |
3488 | { |
3489 | int fd; | |
3490 | ||
1e960043 JK |
3491 | if (!prog) |
3492 | return -EINVAL; | |
3493 | ||
b580563e WN |
3494 | if (n >= prog->instances.nr || n < 0) { |
3495 | pr_warning("Can't get the %dth fd from program %s: only %d instances\n", | |
3496 | n, prog->section_name, prog->instances.nr); | |
3497 | return -EINVAL; | |
3498 | } | |
3499 | ||
3500 | fd = prog->instances.fds[n]; | |
3501 | if (fd < 0) { | |
3502 | pr_warning("%dth instance of program '%s' is invalid\n", | |
3503 | n, prog->section_name); | |
3504 | return -ENOENT; | |
3505 | } | |
3506 | ||
3507 | return fd; | |
aa9b1ac3 | 3508 | } |
9d759a9b | 3509 | |
dd26b7f5 | 3510 | void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) |
5f44e4c8 WN |
3511 | { |
3512 | prog->type = type; | |
3513 | } | |
3514 | ||
a324aae3 | 3515 | static bool bpf_program__is_type(const struct bpf_program *prog, |
5f44e4c8 WN |
3516 | enum bpf_prog_type type) |
3517 | { | |
3518 | return prog ? (prog->type == type) : false; | |
3519 | } | |
3520 | ||
a324aae3 AN |
3521 | #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ |
3522 | int bpf_program__set_##NAME(struct bpf_program *prog) \ | |
3523 | { \ | |
3524 | if (!prog) \ | |
3525 | return -EINVAL; \ | |
3526 | bpf_program__set_type(prog, TYPE); \ | |
3527 | return 0; \ | |
3528 | } \ | |
3529 | \ | |
3530 | bool bpf_program__is_##NAME(const struct bpf_program *prog) \ | |
3531 | { \ | |
3532 | return bpf_program__is_type(prog, TYPE); \ | |
3533 | } \ | |
ed794073 | 3534 | |
7803ba73 | 3535 | BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); |
ed794073 | 3536 | BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); |
7803ba73 JS |
3537 | BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); |
3538 | BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); | |
ed794073 | 3539 | BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); |
e14c93fd | 3540 | BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); |
7803ba73 JS |
3541 | BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); |
3542 | BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); | |
5f44e4c8 | 3543 | |
16962b24 JF |
3544 | void bpf_program__set_expected_attach_type(struct bpf_program *prog, |
3545 | enum bpf_attach_type type) | |
d7be143b AI |
3546 | { |
3547 | prog->expected_attach_type = type; | |
3548 | } | |
3549 | ||
36153532 AI |
3550 | #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ |
3551 | { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } | |
d7be143b | 3552 | |
956b620f | 3553 | /* Programs that can NOT be attached. */ |
36153532 | 3554 | #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) |
d7be143b | 3555 | |
956b620f AI |
3556 | /* Programs that can be attached. */ |
3557 | #define BPF_APROG_SEC(string, ptype, atype) \ | |
36153532 | 3558 | BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) |
81efee75 | 3559 | |
956b620f AI |
3560 | /* Programs that must specify expected attach type at load time. */ |
3561 | #define BPF_EAPROG_SEC(string, ptype, eatype) \ | |
36153532 | 3562 | BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) |
956b620f AI |
3563 | |
3564 | /* Programs that can be attached but attach type can't be identified by section | |
3565 | * name. Kept for backward compatibility. | |
3566 | */ | |
3567 | #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) | |
e50b0a6f | 3568 | |
583c9009 RG |
3569 | static const struct { |
3570 | const char *sec; | |
3571 | size_t len; | |
3572 | enum bpf_prog_type prog_type; | |
d7be143b | 3573 | enum bpf_attach_type expected_attach_type; |
36153532 | 3574 | int is_attachable; |
956b620f | 3575 | enum bpf_attach_type attach_type; |
583c9009 | 3576 | } section_names[] = { |
956b620f AI |
3577 | BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), |
3578 | BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), | |
3579 | BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), | |
3580 | BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), | |
3581 | BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), | |
3582 | BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), | |
3583 | BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), | |
3584 | BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), | |
3585 | BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), | |
3586 | BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), | |
3587 | BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), | |
3588 | BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), | |
3589 | BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), | |
bafa7afe AI |
3590 | BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, |
3591 | BPF_CGROUP_INET_INGRESS), | |
3592 | BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, | |
3593 | BPF_CGROUP_INET_EGRESS), | |
956b620f AI |
3594 | BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), |
3595 | BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, | |
3596 | BPF_CGROUP_INET_SOCK_CREATE), | |
3597 | BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, | |
3598 | BPF_CGROUP_INET4_POST_BIND), | |
3599 | BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, | |
3600 | BPF_CGROUP_INET6_POST_BIND), | |
3601 | BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, | |
3602 | BPF_CGROUP_DEVICE), | |
3603 | BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, | |
3604 | BPF_CGROUP_SOCK_OPS), | |
c6f6851b AI |
3605 | BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, |
3606 | BPF_SK_SKB_STREAM_PARSER), | |
3607 | BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, | |
3608 | BPF_SK_SKB_STREAM_VERDICT), | |
956b620f AI |
3609 | BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), |
3610 | BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, | |
3611 | BPF_SK_MSG_VERDICT), | |
3612 | BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, | |
3613 | BPF_LIRC_MODE2), | |
3614 | BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, | |
3615 | BPF_FLOW_DISSECTOR), | |
3616 | BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
3617 | BPF_CGROUP_INET4_BIND), | |
3618 | BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
3619 | BPF_CGROUP_INET6_BIND), | |
3620 | BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
3621 | BPF_CGROUP_INET4_CONNECT), | |
3622 | BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
3623 | BPF_CGROUP_INET6_CONNECT), | |
3624 | BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
3625 | BPF_CGROUP_UDP4_SENDMSG), | |
3626 | BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
3627 | BPF_CGROUP_UDP6_SENDMSG), | |
9bb59ac1 DB |
3628 | BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, |
3629 | BPF_CGROUP_UDP4_RECVMSG), | |
3630 | BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
3631 | BPF_CGROUP_UDP6_RECVMSG), | |
063cc9f0 AI |
3632 | BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, |
3633 | BPF_CGROUP_SYSCTL), | |
4cdbfb59 SF |
3634 | BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, |
3635 | BPF_CGROUP_GETSOCKOPT), | |
3636 | BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, | |
3637 | BPF_CGROUP_SETSOCKOPT), | |
583c9009 | 3638 | }; |
d7be143b | 3639 | |
956b620f | 3640 | #undef BPF_PROG_SEC_IMPL |
583c9009 | 3641 | #undef BPF_PROG_SEC |
956b620f AI |
3642 | #undef BPF_APROG_SEC |
3643 | #undef BPF_EAPROG_SEC | |
3644 | #undef BPF_APROG_COMPAT | |
583c9009 | 3645 | |
c76e4c22 TS |
3646 | #define MAX_TYPE_NAME_SIZE 32 |
3647 | ||
3648 | static char *libbpf_get_type_names(bool attach_type) | |
3649 | { | |
3650 | int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; | |
3651 | char *buf; | |
3652 | ||
3653 | buf = malloc(len); | |
3654 | if (!buf) | |
3655 | return NULL; | |
3656 | ||
3657 | buf[0] = '\0'; | |
3658 | /* Forge string buf with all available names */ | |
3659 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { | |
3660 | if (attach_type && !section_names[i].is_attachable) | |
3661 | continue; | |
3662 | ||
3663 | if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { | |
3664 | free(buf); | |
3665 | return NULL; | |
3666 | } | |
3667 | strcat(buf, " "); | |
3668 | strcat(buf, section_names[i].sec); | |
3669 | } | |
3670 | ||
3671 | return buf; | |
3672 | } | |
3673 | ||
b60df2a0 JK |
3674 | int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, |
3675 | enum bpf_attach_type *expected_attach_type) | |
583c9009 | 3676 | { |
c76e4c22 | 3677 | char *type_names; |
583c9009 RG |
3678 | int i; |
3679 | ||
b60df2a0 JK |
3680 | if (!name) |
3681 | return -EINVAL; | |
583c9009 | 3682 | |
b60df2a0 JK |
3683 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { |
3684 | if (strncmp(name, section_names[i].sec, section_names[i].len)) | |
3685 | continue; | |
3686 | *prog_type = section_names[i].prog_type; | |
3687 | *expected_attach_type = section_names[i].expected_attach_type; | |
3688 | return 0; | |
3689 | } | |
c76e4c22 TS |
3690 | pr_warning("failed to guess program type based on ELF section name '%s'\n", name); |
3691 | type_names = libbpf_get_type_names(false); | |
3692 | if (type_names != NULL) { | |
3693 | pr_info("supported section(type) names are:%s\n", type_names); | |
3694 | free(type_names); | |
3695 | } | |
3696 | ||
b60df2a0 JK |
3697 | return -EINVAL; |
3698 | } | |
583c9009 | 3699 | |
956b620f AI |
3700 | int libbpf_attach_type_by_name(const char *name, |
3701 | enum bpf_attach_type *attach_type) | |
3702 | { | |
c76e4c22 | 3703 | char *type_names; |
956b620f AI |
3704 | int i; |
3705 | ||
3706 | if (!name) | |
3707 | return -EINVAL; | |
3708 | ||
3709 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { | |
3710 | if (strncmp(name, section_names[i].sec, section_names[i].len)) | |
3711 | continue; | |
36153532 | 3712 | if (!section_names[i].is_attachable) |
956b620f AI |
3713 | return -EINVAL; |
3714 | *attach_type = section_names[i].attach_type; | |
3715 | return 0; | |
3716 | } | |
c76e4c22 TS |
3717 | pr_warning("failed to guess attach type based on ELF section name '%s'\n", name); |
3718 | type_names = libbpf_get_type_names(true); | |
3719 | if (type_names != NULL) { | |
3720 | pr_info("attachable section(type) names are:%s\n", type_names); | |
3721 | free(type_names); | |
3722 | } | |
3723 | ||
956b620f AI |
3724 | return -EINVAL; |
3725 | } | |
3726 | ||
b60df2a0 JK |
3727 | static int |
3728 | bpf_program__identify_section(struct bpf_program *prog, | |
3729 | enum bpf_prog_type *prog_type, | |
3730 | enum bpf_attach_type *expected_attach_type) | |
3731 | { | |
3732 | return libbpf_prog_type_by_name(prog->section_name, prog_type, | |
3733 | expected_attach_type); | |
583c9009 RG |
3734 | } |
3735 | ||
a324aae3 | 3736 | int bpf_map__fd(const struct bpf_map *map) |
9d759a9b | 3737 | { |
6e009e65 | 3738 | return map ? map->fd : -EINVAL; |
9d759a9b WN |
3739 | } |
3740 | ||
a324aae3 | 3741 | const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) |
9d759a9b | 3742 | { |
53897a78 | 3743 | return map ? &map->def : ERR_PTR(-EINVAL); |
9d759a9b WN |
3744 | } |
3745 | ||
a324aae3 | 3746 | const char *bpf_map__name(const struct bpf_map *map) |
561bbcca | 3747 | { |
009ad5d5 | 3748 | return map ? map->name : NULL; |
561bbcca WN |
3749 | } |
3750 | ||
5b891af7 | 3751 | __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) |
8a138aed | 3752 | { |
61746dbe | 3753 | return map ? map->btf_key_type_id : 0; |
8a138aed MKL |
3754 | } |
3755 | ||
5b891af7 | 3756 | __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) |
8a138aed | 3757 | { |
61746dbe | 3758 | return map ? map->btf_value_type_id : 0; |
8a138aed MKL |
3759 | } |
3760 | ||
edb13ed4 ACM |
3761 | int bpf_map__set_priv(struct bpf_map *map, void *priv, |
3762 | bpf_map_clear_priv_t clear_priv) | |
9d759a9b WN |
3763 | { |
3764 | if (!map) | |
3765 | return -EINVAL; | |
3766 | ||
3767 | if (map->priv) { | |
3768 | if (map->clear_priv) | |
3769 | map->clear_priv(map, map->priv); | |
3770 | } | |
3771 | ||
3772 | map->priv = priv; | |
3773 | map->clear_priv = clear_priv; | |
3774 | return 0; | |
3775 | } | |
3776 | ||
a324aae3 | 3777 | void *bpf_map__priv(const struct bpf_map *map) |
9d759a9b | 3778 | { |
b4cbfa56 | 3779 | return map ? map->priv : ERR_PTR(-EINVAL); |
9d759a9b WN |
3780 | } |
3781 | ||
a324aae3 | 3782 | bool bpf_map__is_offload_neutral(const struct bpf_map *map) |
f83fb22c JK |
3783 | { |
3784 | return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; | |
3785 | } | |
3786 | ||
a324aae3 | 3787 | bool bpf_map__is_internal(const struct bpf_map *map) |
d859900c DB |
3788 | { |
3789 | return map->libbpf_type != LIBBPF_MAP_UNSPEC; | |
3790 | } | |
3791 | ||
9aba3613 JK |
3792 | void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) |
3793 | { | |
3794 | map->map_ifindex = ifindex; | |
3795 | } | |
3796 | ||
addb9fc9 NS |
3797 | int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) |
3798 | { | |
3799 | if (!bpf_map_type__is_map_in_map(map->def.type)) { | |
3800 | pr_warning("error: unsupported map type\n"); | |
3801 | return -EINVAL; | |
3802 | } | |
3803 | if (map->inner_map_fd != -1) { | |
3804 | pr_warning("error: inner_map_fd already specified\n"); | |
3805 | return -EINVAL; | |
3806 | } | |
3807 | map->inner_map_fd = fd; | |
3808 | return 0; | |
3809 | } | |
3810 | ||
0c19a9fb | 3811 | static struct bpf_map * |
a324aae3 | 3812 | __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) |
9d759a9b | 3813 | { |
0c19a9fb | 3814 | ssize_t idx; |
9d759a9b WN |
3815 | struct bpf_map *s, *e; |
3816 | ||
3817 | if (!obj || !obj->maps) | |
3818 | return NULL; | |
3819 | ||
3820 | s = obj->maps; | |
3821 | e = obj->maps + obj->nr_maps; | |
3822 | ||
0c19a9fb | 3823 | if ((m < s) || (m >= e)) { |
9d759a9b WN |
3824 | pr_warning("error in %s: map handler doesn't belong to object\n", |
3825 | __func__); | |
3826 | return NULL; | |
3827 | } | |
3828 | ||
0c19a9fb SF |
3829 | idx = (m - obj->maps) + i; |
3830 | if (idx >= obj->nr_maps || idx < 0) | |
9d759a9b WN |
3831 | return NULL; |
3832 | return &obj->maps[idx]; | |
3833 | } | |
561bbcca | 3834 | |
0c19a9fb | 3835 | struct bpf_map * |
a324aae3 | 3836 | bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) |
0c19a9fb SF |
3837 | { |
3838 | if (prev == NULL) | |
3839 | return obj->maps; | |
3840 | ||
3841 | return __bpf_map__iter(prev, obj, 1); | |
3842 | } | |
3843 | ||
3844 | struct bpf_map * | |
a324aae3 | 3845 | bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) |
0c19a9fb SF |
3846 | { |
3847 | if (next == NULL) { | |
3848 | if (!obj->nr_maps) | |
3849 | return NULL; | |
3850 | return obj->maps + obj->nr_maps - 1; | |
3851 | } | |
3852 | ||
3853 | return __bpf_map__iter(next, obj, -1); | |
3854 | } | |
3855 | ||
561bbcca | 3856 | struct bpf_map * |
a324aae3 | 3857 | bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) |
561bbcca WN |
3858 | { |
3859 | struct bpf_map *pos; | |
3860 | ||
f74a53d9 | 3861 | bpf_object__for_each_map(pos, obj) { |
973170e6 | 3862 | if (pos->name && !strcmp(pos->name, name)) |
561bbcca WN |
3863 | return pos; |
3864 | } | |
3865 | return NULL; | |
3866 | } | |
5a6acad1 | 3867 | |
f3cea32d | 3868 | int |
a324aae3 | 3869 | bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) |
f3cea32d MF |
3870 | { |
3871 | return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); | |
3872 | } | |
3873 | ||
5a6acad1 WN |
3874 | struct bpf_map * |
3875 | bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) | |
3876 | { | |
db48814b | 3877 | return ERR_PTR(-ENOTSUP); |
5a6acad1 | 3878 | } |
e28ff1a8 JS |
3879 | |
3880 | long libbpf_get_error(const void *ptr) | |
3881 | { | |
d98363b5 | 3882 | return PTR_ERR_OR_ZERO(ptr); |
e28ff1a8 | 3883 | } |
6f6d33f3 JF |
3884 | |
3885 | int bpf_prog_load(const char *file, enum bpf_prog_type type, | |
3886 | struct bpf_object **pobj, int *prog_fd) | |
d7be143b AI |
3887 | { |
3888 | struct bpf_prog_load_attr attr; | |
3889 | ||
3890 | memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); | |
3891 | attr.file = file; | |
3892 | attr.prog_type = type; | |
3893 | attr.expected_attach_type = 0; | |
3894 | ||
3895 | return bpf_prog_load_xattr(&attr, pobj, prog_fd); | |
3896 | } | |
3897 | ||
3898 | int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, | |
3899 | struct bpf_object **pobj, int *prog_fd) | |
6f6d33f3 | 3900 | { |
33bae185 | 3901 | struct bpf_object_open_attr open_attr = {}; |
48cca7e4 | 3902 | struct bpf_program *prog, *first_prog = NULL; |
d7be143b AI |
3903 | enum bpf_attach_type expected_attach_type; |
3904 | enum bpf_prog_type prog_type; | |
6f6d33f3 | 3905 | struct bpf_object *obj; |
f0307a7e | 3906 | struct bpf_map *map; |
6f6d33f3 JF |
3907 | int err; |
3908 | ||
d7be143b AI |
3909 | if (!attr) |
3910 | return -EINVAL; | |
17387dd5 JK |
3911 | if (!attr->file) |
3912 | return -EINVAL; | |
d7be143b | 3913 | |
33bae185 LY |
3914 | open_attr.file = attr->file; |
3915 | open_attr.prog_type = attr->prog_type; | |
3916 | ||
07f2d4ea | 3917 | obj = bpf_object__open_xattr(&open_attr); |
3597683c | 3918 | if (IS_ERR_OR_NULL(obj)) |
6f6d33f3 JF |
3919 | return -ENOENT; |
3920 | ||
48cca7e4 AS |
3921 | bpf_object__for_each_program(prog, obj) { |
3922 | /* | |
3923 | * If type is not specified, try to guess it based on | |
3924 | * section name. | |
3925 | */ | |
d7be143b | 3926 | prog_type = attr->prog_type; |
f0307a7e | 3927 | prog->prog_ifindex = attr->ifindex; |
d7be143b AI |
3928 | expected_attach_type = attr->expected_attach_type; |
3929 | if (prog_type == BPF_PROG_TYPE_UNSPEC) { | |
b60df2a0 JK |
3930 | err = bpf_program__identify_section(prog, &prog_type, |
3931 | &expected_attach_type); | |
3932 | if (err < 0) { | |
48cca7e4 AS |
3933 | bpf_object__close(obj); |
3934 | return -EINVAL; | |
3935 | } | |
583c9009 | 3936 | } |
48cca7e4 | 3937 | |
d7be143b AI |
3938 | bpf_program__set_type(prog, prog_type); |
3939 | bpf_program__set_expected_attach_type(prog, | |
3940 | expected_attach_type); | |
3941 | ||
da11b417 | 3942 | prog->log_level = attr->log_level; |
04656198 | 3943 | prog->prog_flags = attr->prog_flags; |
69495d2a | 3944 | if (!first_prog) |
48cca7e4 AS |
3945 | first_prog = prog; |
3946 | } | |
3947 | ||
f74a53d9 | 3948 | bpf_object__for_each_map(map, obj) { |
f83fb22c JK |
3949 | if (!bpf_map__is_offload_neutral(map)) |
3950 | map->map_ifindex = attr->ifindex; | |
f0307a7e DB |
3951 | } |
3952 | ||
48cca7e4 AS |
3953 | if (!first_prog) { |
3954 | pr_warning("object file doesn't contain bpf program\n"); | |
3955 | bpf_object__close(obj); | |
3956 | return -ENOENT; | |
583c9009 RG |
3957 | } |
3958 | ||
6f6d33f3 JF |
3959 | err = bpf_object__load(obj); |
3960 | if (err) { | |
3961 | bpf_object__close(obj); | |
3962 | return -EINVAL; | |
3963 | } | |
3964 | ||
3965 | *pobj = obj; | |
48cca7e4 | 3966 | *prog_fd = bpf_program__fd(first_prog); |
6f6d33f3 JF |
3967 | return 0; |
3968 | } | |
d0cabbb0 | 3969 | |
1c2e9efc AN |
3970 | struct bpf_link { |
3971 | int (*destroy)(struct bpf_link *link); | |
3972 | }; | |
3973 | ||
3974 | int bpf_link__destroy(struct bpf_link *link) | |
3975 | { | |
3976 | int err; | |
3977 | ||
3978 | if (!link) | |
3979 | return 0; | |
3980 | ||
3981 | err = link->destroy(link); | |
3982 | free(link); | |
3983 | ||
3984 | return err; | |
3985 | } | |
3986 | ||
63f2f5ee AN |
3987 | struct bpf_link_fd { |
3988 | struct bpf_link link; /* has to be at the top of struct */ | |
3989 | int fd; /* hook FD */ | |
3990 | }; | |
3991 | ||
3992 | static int bpf_link__destroy_perf_event(struct bpf_link *link) | |
3993 | { | |
3994 | struct bpf_link_fd *l = (void *)link; | |
3995 | int err; | |
3996 | ||
3997 | err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0); | |
3998 | if (err) | |
3999 | err = -errno; | |
4000 | ||
4001 | close(l->fd); | |
4002 | return err; | |
4003 | } | |
4004 | ||
4005 | struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, | |
4006 | int pfd) | |
4007 | { | |
4008 | char errmsg[STRERR_BUFSIZE]; | |
4009 | struct bpf_link_fd *link; | |
4010 | int prog_fd, err; | |
4011 | ||
4012 | if (pfd < 0) { | |
4013 | pr_warning("program '%s': invalid perf event FD %d\n", | |
4014 | bpf_program__title(prog, false), pfd); | |
4015 | return ERR_PTR(-EINVAL); | |
4016 | } | |
4017 | prog_fd = bpf_program__fd(prog); | |
4018 | if (prog_fd < 0) { | |
4019 | pr_warning("program '%s': can't attach BPF program w/o FD (did you load it?)\n", | |
4020 | bpf_program__title(prog, false)); | |
4021 | return ERR_PTR(-EINVAL); | |
4022 | } | |
4023 | ||
4024 | link = malloc(sizeof(*link)); | |
4025 | if (!link) | |
4026 | return ERR_PTR(-ENOMEM); | |
4027 | link->link.destroy = &bpf_link__destroy_perf_event; | |
4028 | link->fd = pfd; | |
4029 | ||
4030 | if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { | |
4031 | err = -errno; | |
4032 | free(link); | |
4033 | pr_warning("program '%s': failed to attach to pfd %d: %s\n", | |
4034 | bpf_program__title(prog, false), pfd, | |
4035 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
4036 | return ERR_PTR(err); | |
4037 | } | |
4038 | if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { | |
4039 | err = -errno; | |
4040 | free(link); | |
4041 | pr_warning("program '%s': failed to enable pfd %d: %s\n", | |
4042 | bpf_program__title(prog, false), pfd, | |
4043 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
4044 | return ERR_PTR(err); | |
4045 | } | |
4046 | return (struct bpf_link *)link; | |
4047 | } | |
4048 | ||
b2650027 AN |
4049 | /* |
4050 | * this function is expected to parse integer in the range of [0, 2^31-1] from | |
4051 | * given file using scanf format string fmt. If actual parsed value is | |
4052 | * negative, the result might be indistinguishable from error | |
4053 | */ | |
4054 | static int parse_uint_from_file(const char *file, const char *fmt) | |
4055 | { | |
4056 | char buf[STRERR_BUFSIZE]; | |
4057 | int err, ret; | |
4058 | FILE *f; | |
4059 | ||
4060 | f = fopen(file, "r"); | |
4061 | if (!f) { | |
4062 | err = -errno; | |
4063 | pr_debug("failed to open '%s': %s\n", file, | |
4064 | libbpf_strerror_r(err, buf, sizeof(buf))); | |
4065 | return err; | |
4066 | } | |
4067 | err = fscanf(f, fmt, &ret); | |
4068 | if (err != 1) { | |
4069 | err = err == EOF ? -EIO : -errno; | |
4070 | pr_debug("failed to parse '%s': %s\n", file, | |
4071 | libbpf_strerror_r(err, buf, sizeof(buf))); | |
4072 | fclose(f); | |
4073 | return err; | |
4074 | } | |
4075 | fclose(f); | |
4076 | return ret; | |
4077 | } | |
4078 | ||
4079 | static int determine_kprobe_perf_type(void) | |
4080 | { | |
4081 | const char *file = "/sys/bus/event_source/devices/kprobe/type"; | |
4082 | ||
4083 | return parse_uint_from_file(file, "%d\n"); | |
4084 | } | |
4085 | ||
4086 | static int determine_uprobe_perf_type(void) | |
4087 | { | |
4088 | const char *file = "/sys/bus/event_source/devices/uprobe/type"; | |
4089 | ||
4090 | return parse_uint_from_file(file, "%d\n"); | |
4091 | } | |
4092 | ||
4093 | static int determine_kprobe_retprobe_bit(void) | |
4094 | { | |
4095 | const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; | |
4096 | ||
4097 | return parse_uint_from_file(file, "config:%d\n"); | |
4098 | } | |
4099 | ||
4100 | static int determine_uprobe_retprobe_bit(void) | |
4101 | { | |
4102 | const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; | |
4103 | ||
4104 | return parse_uint_from_file(file, "config:%d\n"); | |
4105 | } | |
4106 | ||
4107 | static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, | |
4108 | uint64_t offset, int pid) | |
4109 | { | |
4110 | struct perf_event_attr attr = {}; | |
4111 | char errmsg[STRERR_BUFSIZE]; | |
4112 | int type, pfd, err; | |
4113 | ||
4114 | type = uprobe ? determine_uprobe_perf_type() | |
4115 | : determine_kprobe_perf_type(); | |
4116 | if (type < 0) { | |
4117 | pr_warning("failed to determine %s perf type: %s\n", | |
4118 | uprobe ? "uprobe" : "kprobe", | |
4119 | libbpf_strerror_r(type, errmsg, sizeof(errmsg))); | |
4120 | return type; | |
4121 | } | |
4122 | if (retprobe) { | |
4123 | int bit = uprobe ? determine_uprobe_retprobe_bit() | |
4124 | : determine_kprobe_retprobe_bit(); | |
4125 | ||
4126 | if (bit < 0) { | |
4127 | pr_warning("failed to determine %s retprobe bit: %s\n", | |
4128 | uprobe ? "uprobe" : "kprobe", | |
4129 | libbpf_strerror_r(bit, errmsg, | |
4130 | sizeof(errmsg))); | |
4131 | return bit; | |
4132 | } | |
4133 | attr.config |= 1 << bit; | |
4134 | } | |
4135 | attr.size = sizeof(attr); | |
4136 | attr.type = type; | |
36db2a94 AN |
4137 | attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ |
4138 | attr.config2 = offset; /* kprobe_addr or probe_offset */ | |
b2650027 AN |
4139 | |
4140 | /* pid filter is meaningful only for uprobes */ | |
4141 | pfd = syscall(__NR_perf_event_open, &attr, | |
4142 | pid < 0 ? -1 : pid /* pid */, | |
4143 | pid == -1 ? 0 : -1 /* cpu */, | |
4144 | -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); | |
4145 | if (pfd < 0) { | |
4146 | err = -errno; | |
4147 | pr_warning("%s perf_event_open() failed: %s\n", | |
4148 | uprobe ? "uprobe" : "kprobe", | |
4149 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
4150 | return err; | |
4151 | } | |
4152 | return pfd; | |
4153 | } | |
4154 | ||
4155 | struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, | |
4156 | bool retprobe, | |
4157 | const char *func_name) | |
4158 | { | |
4159 | char errmsg[STRERR_BUFSIZE]; | |
4160 | struct bpf_link *link; | |
4161 | int pfd, err; | |
4162 | ||
4163 | pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, | |
4164 | 0 /* offset */, -1 /* pid */); | |
4165 | if (pfd < 0) { | |
4166 | pr_warning("program '%s': failed to create %s '%s' perf event: %s\n", | |
4167 | bpf_program__title(prog, false), | |
4168 | retprobe ? "kretprobe" : "kprobe", func_name, | |
4169 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
4170 | return ERR_PTR(pfd); | |
4171 | } | |
4172 | link = bpf_program__attach_perf_event(prog, pfd); | |
4173 | if (IS_ERR(link)) { | |
4174 | close(pfd); | |
4175 | err = PTR_ERR(link); | |
4176 | pr_warning("program '%s': failed to attach to %s '%s': %s\n", | |
4177 | bpf_program__title(prog, false), | |
4178 | retprobe ? "kretprobe" : "kprobe", func_name, | |
4179 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
4180 | return link; | |
4181 | } | |
4182 | return link; | |
4183 | } | |
4184 | ||
4185 | struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, | |
4186 | bool retprobe, pid_t pid, | |
4187 | const char *binary_path, | |
4188 | size_t func_offset) | |
4189 | { | |
4190 | char errmsg[STRERR_BUFSIZE]; | |
4191 | struct bpf_link *link; | |
4192 | int pfd, err; | |
4193 | ||
4194 | pfd = perf_event_open_probe(true /* uprobe */, retprobe, | |
4195 | binary_path, func_offset, pid); | |
4196 | if (pfd < 0) { | |
4197 | pr_warning("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", | |
4198 | bpf_program__title(prog, false), | |
4199 | retprobe ? "uretprobe" : "uprobe", | |
4200 | binary_path, func_offset, | |
4201 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
4202 | return ERR_PTR(pfd); | |
4203 | } | |
4204 | link = bpf_program__attach_perf_event(prog, pfd); | |
4205 | if (IS_ERR(link)) { | |
4206 | close(pfd); | |
4207 | err = PTR_ERR(link); | |
4208 | pr_warning("program '%s': failed to attach to %s '%s:0x%zx': %s\n", | |
4209 | bpf_program__title(prog, false), | |
4210 | retprobe ? "uretprobe" : "uprobe", | |
4211 | binary_path, func_offset, | |
4212 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
4213 | return link; | |
4214 | } | |
4215 | return link; | |
4216 | } | |
4217 | ||
f6de59c1 AN |
4218 | static int determine_tracepoint_id(const char *tp_category, |
4219 | const char *tp_name) | |
4220 | { | |
4221 | char file[PATH_MAX]; | |
4222 | int ret; | |
4223 | ||
4224 | ret = snprintf(file, sizeof(file), | |
4225 | "/sys/kernel/debug/tracing/events/%s/%s/id", | |
4226 | tp_category, tp_name); | |
4227 | if (ret < 0) | |
4228 | return -errno; | |
4229 | if (ret >= sizeof(file)) { | |
4230 | pr_debug("tracepoint %s/%s path is too long\n", | |
4231 | tp_category, tp_name); | |
4232 | return -E2BIG; | |
4233 | } | |
4234 | return parse_uint_from_file(file, "%d\n"); | |
4235 | } | |
4236 | ||
4237 | static int perf_event_open_tracepoint(const char *tp_category, | |
4238 | const char *tp_name) | |
4239 | { | |
4240 | struct perf_event_attr attr = {}; | |
4241 | char errmsg[STRERR_BUFSIZE]; | |
4242 | int tp_id, pfd, err; | |
4243 | ||
4244 | tp_id = determine_tracepoint_id(tp_category, tp_name); | |
4245 | if (tp_id < 0) { | |
4246 | pr_warning("failed to determine tracepoint '%s/%s' perf event ID: %s\n", | |
4247 | tp_category, tp_name, | |
4248 | libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); | |
4249 | return tp_id; | |
4250 | } | |
4251 | ||
4252 | attr.type = PERF_TYPE_TRACEPOINT; | |
4253 | attr.size = sizeof(attr); | |
4254 | attr.config = tp_id; | |
4255 | ||
4256 | pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, | |
4257 | -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); | |
4258 | if (pfd < 0) { | |
4259 | err = -errno; | |
4260 | pr_warning("tracepoint '%s/%s' perf_event_open() failed: %s\n", | |
4261 | tp_category, tp_name, | |
4262 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
4263 | return err; | |
4264 | } | |
4265 | return pfd; | |
4266 | } | |
4267 | ||
4268 | struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, | |
4269 | const char *tp_category, | |
4270 | const char *tp_name) | |
4271 | { | |
4272 | char errmsg[STRERR_BUFSIZE]; | |
4273 | struct bpf_link *link; | |
4274 | int pfd, err; | |
4275 | ||
4276 | pfd = perf_event_open_tracepoint(tp_category, tp_name); | |
4277 | if (pfd < 0) { | |
4278 | pr_warning("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", | |
4279 | bpf_program__title(prog, false), | |
4280 | tp_category, tp_name, | |
4281 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
4282 | return ERR_PTR(pfd); | |
4283 | } | |
4284 | link = bpf_program__attach_perf_event(prog, pfd); | |
4285 | if (IS_ERR(link)) { | |
4286 | close(pfd); | |
4287 | err = PTR_ERR(link); | |
4288 | pr_warning("program '%s': failed to attach to tracepoint '%s/%s': %s\n", | |
4289 | bpf_program__title(prog, false), | |
4290 | tp_category, tp_name, | |
4291 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
4292 | return link; | |
4293 | } | |
4294 | return link; | |
4295 | } | |
4296 | ||
84bf5e1f AN |
4297 | static int bpf_link__destroy_fd(struct bpf_link *link) |
4298 | { | |
4299 | struct bpf_link_fd *l = (void *)link; | |
4300 | ||
4301 | return close(l->fd); | |
4302 | } | |
4303 | ||
4304 | struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, | |
4305 | const char *tp_name) | |
4306 | { | |
4307 | char errmsg[STRERR_BUFSIZE]; | |
4308 | struct bpf_link_fd *link; | |
4309 | int prog_fd, pfd; | |
4310 | ||
4311 | prog_fd = bpf_program__fd(prog); | |
4312 | if (prog_fd < 0) { | |
4313 | pr_warning("program '%s': can't attach before loaded\n", | |
4314 | bpf_program__title(prog, false)); | |
4315 | return ERR_PTR(-EINVAL); | |
4316 | } | |
4317 | ||
4318 | link = malloc(sizeof(*link)); | |
4319 | if (!link) | |
4320 | return ERR_PTR(-ENOMEM); | |
4321 | link->link.destroy = &bpf_link__destroy_fd; | |
4322 | ||
4323 | pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); | |
4324 | if (pfd < 0) { | |
4325 | pfd = -errno; | |
4326 | free(link); | |
4327 | pr_warning("program '%s': failed to attach to raw tracepoint '%s': %s\n", | |
4328 | bpf_program__title(prog, false), tp_name, | |
4329 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
4330 | return ERR_PTR(pfd); | |
4331 | } | |
4332 | link->fd = pfd; | |
4333 | return (struct bpf_link *)link; | |
4334 | } | |
4335 | ||
d0cabbb0 | 4336 | enum bpf_perf_event_ret |
3dca2115 DB |
4337 | bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, |
4338 | void **copy_mem, size_t *copy_size, | |
4339 | bpf_perf_event_print_t fn, void *private_data) | |
d0cabbb0 | 4340 | { |
3dca2115 | 4341 | struct perf_event_mmap_page *header = mmap_mem; |
a64af0ef | 4342 | __u64 data_head = ring_buffer_read_head(header); |
d0cabbb0 | 4343 | __u64 data_tail = header->data_tail; |
3dca2115 DB |
4344 | void *base = ((__u8 *)header) + page_size; |
4345 | int ret = LIBBPF_PERF_EVENT_CONT; | |
4346 | struct perf_event_header *ehdr; | |
4347 | size_t ehdr_size; | |
4348 | ||
4349 | while (data_head != data_tail) { | |
4350 | ehdr = base + (data_tail & (mmap_size - 1)); | |
4351 | ehdr_size = ehdr->size; | |
4352 | ||
4353 | if (((void *)ehdr) + ehdr_size > base + mmap_size) { | |
4354 | void *copy_start = ehdr; | |
4355 | size_t len_first = base + mmap_size - copy_start; | |
4356 | size_t len_secnd = ehdr_size - len_first; | |
4357 | ||
4358 | if (*copy_size < ehdr_size) { | |
4359 | free(*copy_mem); | |
4360 | *copy_mem = malloc(ehdr_size); | |
4361 | if (!*copy_mem) { | |
4362 | *copy_size = 0; | |
d0cabbb0 JK |
4363 | ret = LIBBPF_PERF_EVENT_ERROR; |
4364 | break; | |
4365 | } | |
3dca2115 | 4366 | *copy_size = ehdr_size; |
d0cabbb0 JK |
4367 | } |
4368 | ||
3dca2115 DB |
4369 | memcpy(*copy_mem, copy_start, len_first); |
4370 | memcpy(*copy_mem + len_first, base, len_secnd); | |
4371 | ehdr = *copy_mem; | |
d0cabbb0 JK |
4372 | } |
4373 | ||
3dca2115 DB |
4374 | ret = fn(ehdr, private_data); |
4375 | data_tail += ehdr_size; | |
d0cabbb0 JK |
4376 | if (ret != LIBBPF_PERF_EVENT_CONT) |
4377 | break; | |
d0cabbb0 JK |
4378 | } |
4379 | ||
a64af0ef | 4380 | ring_buffer_write_tail(header, data_tail); |
d0cabbb0 JK |
4381 | return ret; |
4382 | } | |
34be1646 | 4383 | |
fb84b822 AN |
4384 | struct perf_buffer; |
4385 | ||
4386 | struct perf_buffer_params { | |
4387 | struct perf_event_attr *attr; | |
4388 | /* if event_cb is specified, it takes precendence */ | |
4389 | perf_buffer_event_fn event_cb; | |
4390 | /* sample_cb and lost_cb are higher-level common-case callbacks */ | |
4391 | perf_buffer_sample_fn sample_cb; | |
4392 | perf_buffer_lost_fn lost_cb; | |
4393 | void *ctx; | |
4394 | int cpu_cnt; | |
4395 | int *cpus; | |
4396 | int *map_keys; | |
4397 | }; | |
4398 | ||
4399 | struct perf_cpu_buf { | |
4400 | struct perf_buffer *pb; | |
4401 | void *base; /* mmap()'ed memory */ | |
4402 | void *buf; /* for reconstructing segmented data */ | |
4403 | size_t buf_size; | |
4404 | int fd; | |
4405 | int cpu; | |
4406 | int map_key; | |
4407 | }; | |
4408 | ||
4409 | struct perf_buffer { | |
4410 | perf_buffer_event_fn event_cb; | |
4411 | perf_buffer_sample_fn sample_cb; | |
4412 | perf_buffer_lost_fn lost_cb; | |
4413 | void *ctx; /* passed into callbacks */ | |
4414 | ||
4415 | size_t page_size; | |
4416 | size_t mmap_size; | |
4417 | struct perf_cpu_buf **cpu_bufs; | |
4418 | struct epoll_event *events; | |
4419 | int cpu_cnt; | |
4420 | int epoll_fd; /* perf event FD */ | |
4421 | int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ | |
4422 | }; | |
4423 | ||
4424 | static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, | |
4425 | struct perf_cpu_buf *cpu_buf) | |
4426 | { | |
4427 | if (!cpu_buf) | |
4428 | return; | |
4429 | if (cpu_buf->base && | |
4430 | munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) | |
4431 | pr_warning("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); | |
4432 | if (cpu_buf->fd >= 0) { | |
4433 | ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); | |
4434 | close(cpu_buf->fd); | |
4435 | } | |
4436 | free(cpu_buf->buf); | |
4437 | free(cpu_buf); | |
4438 | } | |
4439 | ||
4440 | void perf_buffer__free(struct perf_buffer *pb) | |
4441 | { | |
4442 | int i; | |
4443 | ||
4444 | if (!pb) | |
4445 | return; | |
4446 | if (pb->cpu_bufs) { | |
4447 | for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) { | |
4448 | struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; | |
4449 | ||
4450 | bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); | |
4451 | perf_buffer__free_cpu_buf(pb, cpu_buf); | |
4452 | } | |
4453 | free(pb->cpu_bufs); | |
4454 | } | |
4455 | if (pb->epoll_fd >= 0) | |
4456 | close(pb->epoll_fd); | |
4457 | free(pb->events); | |
4458 | free(pb); | |
4459 | } | |
4460 | ||
4461 | static struct perf_cpu_buf * | |
4462 | perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, | |
4463 | int cpu, int map_key) | |
4464 | { | |
4465 | struct perf_cpu_buf *cpu_buf; | |
4466 | char msg[STRERR_BUFSIZE]; | |
4467 | int err; | |
4468 | ||
4469 | cpu_buf = calloc(1, sizeof(*cpu_buf)); | |
4470 | if (!cpu_buf) | |
4471 | return ERR_PTR(-ENOMEM); | |
4472 | ||
4473 | cpu_buf->pb = pb; | |
4474 | cpu_buf->cpu = cpu; | |
4475 | cpu_buf->map_key = map_key; | |
4476 | ||
4477 | cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, | |
4478 | -1, PERF_FLAG_FD_CLOEXEC); | |
4479 | if (cpu_buf->fd < 0) { | |
4480 | err = -errno; | |
4481 | pr_warning("failed to open perf buffer event on cpu #%d: %s\n", | |
4482 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
4483 | goto error; | |
4484 | } | |
4485 | ||
4486 | cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, | |
4487 | PROT_READ | PROT_WRITE, MAP_SHARED, | |
4488 | cpu_buf->fd, 0); | |
4489 | if (cpu_buf->base == MAP_FAILED) { | |
4490 | cpu_buf->base = NULL; | |
4491 | err = -errno; | |
4492 | pr_warning("failed to mmap perf buffer on cpu #%d: %s\n", | |
4493 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
4494 | goto error; | |
4495 | } | |
4496 | ||
4497 | if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { | |
4498 | err = -errno; | |
4499 | pr_warning("failed to enable perf buffer event on cpu #%d: %s\n", | |
4500 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
4501 | goto error; | |
4502 | } | |
4503 | ||
4504 | return cpu_buf; | |
4505 | ||
4506 | error: | |
4507 | perf_buffer__free_cpu_buf(pb, cpu_buf); | |
4508 | return (struct perf_cpu_buf *)ERR_PTR(err); | |
4509 | } | |
4510 | ||
4511 | static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, | |
4512 | struct perf_buffer_params *p); | |
4513 | ||
4514 | struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, | |
4515 | const struct perf_buffer_opts *opts) | |
4516 | { | |
4517 | struct perf_buffer_params p = {}; | |
4518 | struct perf_event_attr attr = { | |
4519 | .config = PERF_COUNT_SW_BPF_OUTPUT, | |
4520 | .type = PERF_TYPE_SOFTWARE, | |
4521 | .sample_type = PERF_SAMPLE_RAW, | |
4522 | .sample_period = 1, | |
4523 | .wakeup_events = 1, | |
4524 | }; | |
4525 | ||
4526 | p.attr = &attr; | |
4527 | p.sample_cb = opts ? opts->sample_cb : NULL; | |
4528 | p.lost_cb = opts ? opts->lost_cb : NULL; | |
4529 | p.ctx = opts ? opts->ctx : NULL; | |
4530 | ||
4531 | return __perf_buffer__new(map_fd, page_cnt, &p); | |
4532 | } | |
4533 | ||
4534 | struct perf_buffer * | |
4535 | perf_buffer__new_raw(int map_fd, size_t page_cnt, | |
4536 | const struct perf_buffer_raw_opts *opts) | |
4537 | { | |
4538 | struct perf_buffer_params p = {}; | |
4539 | ||
4540 | p.attr = opts->attr; | |
4541 | p.event_cb = opts->event_cb; | |
4542 | p.ctx = opts->ctx; | |
4543 | p.cpu_cnt = opts->cpu_cnt; | |
4544 | p.cpus = opts->cpus; | |
4545 | p.map_keys = opts->map_keys; | |
4546 | ||
4547 | return __perf_buffer__new(map_fd, page_cnt, &p); | |
4548 | } | |
4549 | ||
4550 | static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, | |
4551 | struct perf_buffer_params *p) | |
4552 | { | |
4553 | struct bpf_map_info map = {}; | |
4554 | char msg[STRERR_BUFSIZE]; | |
4555 | struct perf_buffer *pb; | |
4556 | __u32 map_info_len; | |
4557 | int err, i; | |
4558 | ||
4559 | if (page_cnt & (page_cnt - 1)) { | |
4560 | pr_warning("page count should be power of two, but is %zu\n", | |
4561 | page_cnt); | |
4562 | return ERR_PTR(-EINVAL); | |
4563 | } | |
4564 | ||
4565 | map_info_len = sizeof(map); | |
4566 | err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); | |
4567 | if (err) { | |
4568 | err = -errno; | |
4569 | pr_warning("failed to get map info for map FD %d: %s\n", | |
4570 | map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); | |
4571 | return ERR_PTR(err); | |
4572 | } | |
4573 | ||
4574 | if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { | |
4575 | pr_warning("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", | |
4576 | map.name); | |
4577 | return ERR_PTR(-EINVAL); | |
4578 | } | |
4579 | ||
4580 | pb = calloc(1, sizeof(*pb)); | |
4581 | if (!pb) | |
4582 | return ERR_PTR(-ENOMEM); | |
4583 | ||
4584 | pb->event_cb = p->event_cb; | |
4585 | pb->sample_cb = p->sample_cb; | |
4586 | pb->lost_cb = p->lost_cb; | |
4587 | pb->ctx = p->ctx; | |
4588 | ||
4589 | pb->page_size = getpagesize(); | |
4590 | pb->mmap_size = pb->page_size * page_cnt; | |
4591 | pb->map_fd = map_fd; | |
4592 | ||
4593 | pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); | |
4594 | if (pb->epoll_fd < 0) { | |
4595 | err = -errno; | |
4596 | pr_warning("failed to create epoll instance: %s\n", | |
4597 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
4598 | goto error; | |
4599 | } | |
4600 | ||
4601 | if (p->cpu_cnt > 0) { | |
4602 | pb->cpu_cnt = p->cpu_cnt; | |
4603 | } else { | |
4604 | pb->cpu_cnt = libbpf_num_possible_cpus(); | |
4605 | if (pb->cpu_cnt < 0) { | |
4606 | err = pb->cpu_cnt; | |
4607 | goto error; | |
4608 | } | |
4609 | if (map.max_entries < pb->cpu_cnt) | |
4610 | pb->cpu_cnt = map.max_entries; | |
4611 | } | |
4612 | ||
4613 | pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); | |
4614 | if (!pb->events) { | |
4615 | err = -ENOMEM; | |
4616 | pr_warning("failed to allocate events: out of memory\n"); | |
4617 | goto error; | |
4618 | } | |
4619 | pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); | |
4620 | if (!pb->cpu_bufs) { | |
4621 | err = -ENOMEM; | |
4622 | pr_warning("failed to allocate buffers: out of memory\n"); | |
4623 | goto error; | |
4624 | } | |
4625 | ||
4626 | for (i = 0; i < pb->cpu_cnt; i++) { | |
4627 | struct perf_cpu_buf *cpu_buf; | |
4628 | int cpu, map_key; | |
4629 | ||
4630 | cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; | |
4631 | map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; | |
4632 | ||
4633 | cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); | |
4634 | if (IS_ERR(cpu_buf)) { | |
4635 | err = PTR_ERR(cpu_buf); | |
4636 | goto error; | |
4637 | } | |
4638 | ||
4639 | pb->cpu_bufs[i] = cpu_buf; | |
4640 | ||
4641 | err = bpf_map_update_elem(pb->map_fd, &map_key, | |
4642 | &cpu_buf->fd, 0); | |
4643 | if (err) { | |
4644 | err = -errno; | |
4645 | pr_warning("failed to set cpu #%d, key %d -> perf FD %d: %s\n", | |
4646 | cpu, map_key, cpu_buf->fd, | |
4647 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
4648 | goto error; | |
4649 | } | |
4650 | ||
4651 | pb->events[i].events = EPOLLIN; | |
4652 | pb->events[i].data.ptr = cpu_buf; | |
4653 | if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, | |
4654 | &pb->events[i]) < 0) { | |
4655 | err = -errno; | |
4656 | pr_warning("failed to epoll_ctl cpu #%d perf FD %d: %s\n", | |
4657 | cpu, cpu_buf->fd, | |
4658 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
4659 | goto error; | |
4660 | } | |
4661 | } | |
4662 | ||
4663 | return pb; | |
4664 | ||
4665 | error: | |
4666 | if (pb) | |
4667 | perf_buffer__free(pb); | |
4668 | return ERR_PTR(err); | |
4669 | } | |
4670 | ||
4671 | struct perf_sample_raw { | |
4672 | struct perf_event_header header; | |
4673 | uint32_t size; | |
4674 | char data[0]; | |
4675 | }; | |
4676 | ||
4677 | struct perf_sample_lost { | |
4678 | struct perf_event_header header; | |
4679 | uint64_t id; | |
4680 | uint64_t lost; | |
4681 | uint64_t sample_id; | |
4682 | }; | |
4683 | ||
4684 | static enum bpf_perf_event_ret | |
4685 | perf_buffer__process_record(struct perf_event_header *e, void *ctx) | |
4686 | { | |
4687 | struct perf_cpu_buf *cpu_buf = ctx; | |
4688 | struct perf_buffer *pb = cpu_buf->pb; | |
4689 | void *data = e; | |
4690 | ||
4691 | /* user wants full control over parsing perf event */ | |
4692 | if (pb->event_cb) | |
4693 | return pb->event_cb(pb->ctx, cpu_buf->cpu, e); | |
4694 | ||
4695 | switch (e->type) { | |
4696 | case PERF_RECORD_SAMPLE: { | |
4697 | struct perf_sample_raw *s = data; | |
4698 | ||
4699 | if (pb->sample_cb) | |
4700 | pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); | |
4701 | break; | |
4702 | } | |
4703 | case PERF_RECORD_LOST: { | |
4704 | struct perf_sample_lost *s = data; | |
4705 | ||
4706 | if (pb->lost_cb) | |
4707 | pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); | |
4708 | break; | |
4709 | } | |
4710 | default: | |
4711 | pr_warning("unknown perf sample type %d\n", e->type); | |
4712 | return LIBBPF_PERF_EVENT_ERROR; | |
4713 | } | |
4714 | return LIBBPF_PERF_EVENT_CONT; | |
4715 | } | |
4716 | ||
4717 | static int perf_buffer__process_records(struct perf_buffer *pb, | |
4718 | struct perf_cpu_buf *cpu_buf) | |
4719 | { | |
4720 | enum bpf_perf_event_ret ret; | |
4721 | ||
4722 | ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, | |
4723 | pb->page_size, &cpu_buf->buf, | |
4724 | &cpu_buf->buf_size, | |
4725 | perf_buffer__process_record, cpu_buf); | |
4726 | if (ret != LIBBPF_PERF_EVENT_CONT) | |
4727 | return ret; | |
4728 | return 0; | |
4729 | } | |
4730 | ||
4731 | int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) | |
4732 | { | |
4733 | int i, cnt, err; | |
4734 | ||
4735 | cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); | |
4736 | for (i = 0; i < cnt; i++) { | |
4737 | struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; | |
4738 | ||
4739 | err = perf_buffer__process_records(pb, cpu_buf); | |
4740 | if (err) { | |
4741 | pr_warning("error while processing records: %d\n", err); | |
4742 | return err; | |
4743 | } | |
4744 | } | |
4745 | return cnt < 0 ? -errno : cnt; | |
4746 | } | |
4747 | ||
34be1646 SL |
4748 | struct bpf_prog_info_array_desc { |
4749 | int array_offset; /* e.g. offset of jited_prog_insns */ | |
4750 | int count_offset; /* e.g. offset of jited_prog_len */ | |
4751 | int size_offset; /* > 0: offset of rec size, | |
4752 | * < 0: fix size of -size_offset | |
4753 | */ | |
4754 | }; | |
4755 | ||
4756 | static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { | |
4757 | [BPF_PROG_INFO_JITED_INSNS] = { | |
4758 | offsetof(struct bpf_prog_info, jited_prog_insns), | |
4759 | offsetof(struct bpf_prog_info, jited_prog_len), | |
4760 | -1, | |
4761 | }, | |
4762 | [BPF_PROG_INFO_XLATED_INSNS] = { | |
4763 | offsetof(struct bpf_prog_info, xlated_prog_insns), | |
4764 | offsetof(struct bpf_prog_info, xlated_prog_len), | |
4765 | -1, | |
4766 | }, | |
4767 | [BPF_PROG_INFO_MAP_IDS] = { | |
4768 | offsetof(struct bpf_prog_info, map_ids), | |
4769 | offsetof(struct bpf_prog_info, nr_map_ids), | |
4770 | -(int)sizeof(__u32), | |
4771 | }, | |
4772 | [BPF_PROG_INFO_JITED_KSYMS] = { | |
4773 | offsetof(struct bpf_prog_info, jited_ksyms), | |
4774 | offsetof(struct bpf_prog_info, nr_jited_ksyms), | |
4775 | -(int)sizeof(__u64), | |
4776 | }, | |
4777 | [BPF_PROG_INFO_JITED_FUNC_LENS] = { | |
4778 | offsetof(struct bpf_prog_info, jited_func_lens), | |
4779 | offsetof(struct bpf_prog_info, nr_jited_func_lens), | |
4780 | -(int)sizeof(__u32), | |
4781 | }, | |
4782 | [BPF_PROG_INFO_FUNC_INFO] = { | |
4783 | offsetof(struct bpf_prog_info, func_info), | |
4784 | offsetof(struct bpf_prog_info, nr_func_info), | |
4785 | offsetof(struct bpf_prog_info, func_info_rec_size), | |
4786 | }, | |
4787 | [BPF_PROG_INFO_LINE_INFO] = { | |
4788 | offsetof(struct bpf_prog_info, line_info), | |
4789 | offsetof(struct bpf_prog_info, nr_line_info), | |
4790 | offsetof(struct bpf_prog_info, line_info_rec_size), | |
4791 | }, | |
4792 | [BPF_PROG_INFO_JITED_LINE_INFO] = { | |
4793 | offsetof(struct bpf_prog_info, jited_line_info), | |
4794 | offsetof(struct bpf_prog_info, nr_jited_line_info), | |
4795 | offsetof(struct bpf_prog_info, jited_line_info_rec_size), | |
4796 | }, | |
4797 | [BPF_PROG_INFO_PROG_TAGS] = { | |
4798 | offsetof(struct bpf_prog_info, prog_tags), | |
4799 | offsetof(struct bpf_prog_info, nr_prog_tags), | |
4800 | -(int)sizeof(__u8) * BPF_TAG_SIZE, | |
4801 | }, | |
4802 | ||
4803 | }; | |
4804 | ||
4805 | static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) | |
4806 | { | |
4807 | __u32 *array = (__u32 *)info; | |
4808 | ||
4809 | if (offset >= 0) | |
4810 | return array[offset / sizeof(__u32)]; | |
4811 | return -(int)offset; | |
4812 | } | |
4813 | ||
4814 | static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) | |
4815 | { | |
4816 | __u64 *array = (__u64 *)info; | |
4817 | ||
4818 | if (offset >= 0) | |
4819 | return array[offset / sizeof(__u64)]; | |
4820 | return -(int)offset; | |
4821 | } | |
4822 | ||
4823 | static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, | |
4824 | __u32 val) | |
4825 | { | |
4826 | __u32 *array = (__u32 *)info; | |
4827 | ||
4828 | if (offset >= 0) | |
4829 | array[offset / sizeof(__u32)] = val; | |
4830 | } | |
4831 | ||
4832 | static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, | |
4833 | __u64 val) | |
4834 | { | |
4835 | __u64 *array = (__u64 *)info; | |
4836 | ||
4837 | if (offset >= 0) | |
4838 | array[offset / sizeof(__u64)] = val; | |
4839 | } | |
4840 | ||
4841 | struct bpf_prog_info_linear * | |
4842 | bpf_program__get_prog_info_linear(int fd, __u64 arrays) | |
4843 | { | |
4844 | struct bpf_prog_info_linear *info_linear; | |
4845 | struct bpf_prog_info info = {}; | |
4846 | __u32 info_len = sizeof(info); | |
4847 | __u32 data_len = 0; | |
4848 | int i, err; | |
4849 | void *ptr; | |
4850 | ||
4851 | if (arrays >> BPF_PROG_INFO_LAST_ARRAY) | |
4852 | return ERR_PTR(-EINVAL); | |
4853 | ||
4854 | /* step 1: get array dimensions */ | |
4855 | err = bpf_obj_get_info_by_fd(fd, &info, &info_len); | |
4856 | if (err) { | |
4857 | pr_debug("can't get prog info: %s", strerror(errno)); | |
4858 | return ERR_PTR(-EFAULT); | |
4859 | } | |
4860 | ||
4861 | /* step 2: calculate total size of all arrays */ | |
4862 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
4863 | bool include_array = (arrays & (1UL << i)) > 0; | |
4864 | struct bpf_prog_info_array_desc *desc; | |
4865 | __u32 count, size; | |
4866 | ||
4867 | desc = bpf_prog_info_array_desc + i; | |
4868 | ||
4869 | /* kernel is too old to support this field */ | |
4870 | if (info_len < desc->array_offset + sizeof(__u32) || | |
4871 | info_len < desc->count_offset + sizeof(__u32) || | |
4872 | (desc->size_offset > 0 && info_len < desc->size_offset)) | |
4873 | include_array = false; | |
4874 | ||
4875 | if (!include_array) { | |
4876 | arrays &= ~(1UL << i); /* clear the bit */ | |
4877 | continue; | |
4878 | } | |
4879 | ||
4880 | count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
4881 | size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
4882 | ||
4883 | data_len += count * size; | |
4884 | } | |
4885 | ||
4886 | /* step 3: allocate continuous memory */ | |
4887 | data_len = roundup(data_len, sizeof(__u64)); | |
4888 | info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); | |
4889 | if (!info_linear) | |
4890 | return ERR_PTR(-ENOMEM); | |
4891 | ||
4892 | /* step 4: fill data to info_linear->info */ | |
4893 | info_linear->arrays = arrays; | |
4894 | memset(&info_linear->info, 0, sizeof(info)); | |
4895 | ptr = info_linear->data; | |
4896 | ||
4897 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
4898 | struct bpf_prog_info_array_desc *desc; | |
4899 | __u32 count, size; | |
4900 | ||
4901 | if ((arrays & (1UL << i)) == 0) | |
4902 | continue; | |
4903 | ||
4904 | desc = bpf_prog_info_array_desc + i; | |
4905 | count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
4906 | size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
4907 | bpf_prog_info_set_offset_u32(&info_linear->info, | |
4908 | desc->count_offset, count); | |
4909 | bpf_prog_info_set_offset_u32(&info_linear->info, | |
4910 | desc->size_offset, size); | |
4911 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
4912 | desc->array_offset, | |
4913 | ptr_to_u64(ptr)); | |
4914 | ptr += count * size; | |
4915 | } | |
4916 | ||
4917 | /* step 5: call syscall again to get required arrays */ | |
4918 | err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); | |
4919 | if (err) { | |
4920 | pr_debug("can't get prog info: %s", strerror(errno)); | |
4921 | free(info_linear); | |
4922 | return ERR_PTR(-EFAULT); | |
4923 | } | |
4924 | ||
4925 | /* step 6: verify the data */ | |
4926 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
4927 | struct bpf_prog_info_array_desc *desc; | |
4928 | __u32 v1, v2; | |
4929 | ||
4930 | if ((arrays & (1UL << i)) == 0) | |
4931 | continue; | |
4932 | ||
4933 | desc = bpf_prog_info_array_desc + i; | |
4934 | v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
4935 | v2 = bpf_prog_info_read_offset_u32(&info_linear->info, | |
4936 | desc->count_offset); | |
4937 | if (v1 != v2) | |
4938 | pr_warning("%s: mismatch in element count\n", __func__); | |
4939 | ||
4940 | v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
4941 | v2 = bpf_prog_info_read_offset_u32(&info_linear->info, | |
4942 | desc->size_offset); | |
4943 | if (v1 != v2) | |
4944 | pr_warning("%s: mismatch in rec size\n", __func__); | |
4945 | } | |
4946 | ||
4947 | /* step 7: update info_len and data_len */ | |
4948 | info_linear->info_len = sizeof(struct bpf_prog_info); | |
4949 | info_linear->data_len = data_len; | |
4950 | ||
4951 | return info_linear; | |
4952 | } | |
4953 | ||
4954 | void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) | |
4955 | { | |
4956 | int i; | |
4957 | ||
4958 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
4959 | struct bpf_prog_info_array_desc *desc; | |
4960 | __u64 addr, offs; | |
4961 | ||
4962 | if ((info_linear->arrays & (1UL << i)) == 0) | |
4963 | continue; | |
4964 | ||
4965 | desc = bpf_prog_info_array_desc + i; | |
4966 | addr = bpf_prog_info_read_offset_u64(&info_linear->info, | |
4967 | desc->array_offset); | |
4968 | offs = addr - ptr_to_u64(info_linear->data); | |
4969 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
4970 | desc->array_offset, offs); | |
4971 | } | |
4972 | } | |
4973 | ||
4974 | void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) | |
4975 | { | |
4976 | int i; | |
4977 | ||
4978 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
4979 | struct bpf_prog_info_array_desc *desc; | |
4980 | __u64 addr, offs; | |
4981 | ||
4982 | if ((info_linear->arrays & (1UL << i)) == 0) | |
4983 | continue; | |
4984 | ||
4985 | desc = bpf_prog_info_array_desc + i; | |
4986 | offs = bpf_prog_info_read_offset_u64(&info_linear->info, | |
4987 | desc->array_offset); | |
4988 | addr = offs + ptr_to_u64(info_linear->data); | |
4989 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
4990 | desc->array_offset, addr); | |
4991 | } | |
4992 | } | |
6446b315 HL |
4993 | |
4994 | int libbpf_num_possible_cpus(void) | |
4995 | { | |
4996 | static const char *fcpu = "/sys/devices/system/cpu/possible"; | |
4997 | int len = 0, n = 0, il = 0, ir = 0; | |
4998 | unsigned int start = 0, end = 0; | |
4999 | static int cpus; | |
5000 | char buf[128]; | |
5001 | int error = 0; | |
5002 | int fd = -1; | |
5003 | ||
5004 | if (cpus > 0) | |
5005 | return cpus; | |
5006 | ||
5007 | fd = open(fcpu, O_RDONLY); | |
5008 | if (fd < 0) { | |
5009 | error = errno; | |
5010 | pr_warning("Failed to open file %s: %s\n", | |
5011 | fcpu, strerror(error)); | |
5012 | return -error; | |
5013 | } | |
5014 | len = read(fd, buf, sizeof(buf)); | |
5015 | close(fd); | |
5016 | if (len <= 0) { | |
5017 | error = len ? errno : EINVAL; | |
5018 | pr_warning("Failed to read # of possible cpus from %s: %s\n", | |
5019 | fcpu, strerror(error)); | |
5020 | return -error; | |
5021 | } | |
5022 | if (len == sizeof(buf)) { | |
5023 | pr_warning("File %s size overflow\n", fcpu); | |
5024 | return -EOVERFLOW; | |
5025 | } | |
5026 | buf[len] = '\0'; | |
5027 | ||
5028 | for (ir = 0, cpus = 0; ir <= len; ir++) { | |
5029 | /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ | |
5030 | if (buf[ir] == ',' || buf[ir] == '\0') { | |
5031 | buf[ir] = '\0'; | |
5032 | n = sscanf(&buf[il], "%u-%u", &start, &end); | |
5033 | if (n <= 0) { | |
5034 | pr_warning("Failed to get # CPUs from %s\n", | |
5035 | &buf[il]); | |
5036 | return -EINVAL; | |
5037 | } else if (n == 1) { | |
5038 | end = start; | |
5039 | } | |
5040 | cpus += end - start + 1; | |
5041 | il = ir + 1; | |
5042 | } | |
5043 | } | |
5044 | if (cpus <= 0) { | |
5045 | pr_warning("Invalid #CPUs %d from %s\n", cpus, fcpu); | |
5046 | return -EINVAL; | |
5047 | } | |
5048 | return cpus; | |
5049 | } |