Commit | Line | Data |
---|---|---|
1bc38b8f | 1 | // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) |
6061a3d6 | 2 | |
1b76c13e WN |
3 | /* |
4 | * Common eBPF ELF object loading operations. | |
5 | * | |
6 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
7 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
8 | * Copyright (C) 2015 Huawei Inc. | |
f367540c | 9 | * Copyright (C) 2017 Nicira, Inc. |
d859900c | 10 | * Copyright (C) 2019 Isovalent, Inc. |
1b76c13e WN |
11 | */ |
12 | ||
b4269954 | 13 | #ifndef _GNU_SOURCE |
531b014e | 14 | #define _GNU_SOURCE |
b4269954 | 15 | #endif |
1b76c13e | 16 | #include <stdlib.h> |
b3f59d66 WN |
17 | #include <stdio.h> |
18 | #include <stdarg.h> | |
f367540c | 19 | #include <libgen.h> |
34090915 | 20 | #include <inttypes.h> |
b3f59d66 | 21 | #include <string.h> |
1b76c13e | 22 | #include <unistd.h> |
cdb2f920 | 23 | #include <endian.h> |
1a5e3fb1 WN |
24 | #include <fcntl.h> |
25 | #include <errno.h> | |
1b76c13e | 26 | #include <asm/unistd.h> |
e28ff1a8 | 27 | #include <linux/err.h> |
cb1e5e96 | 28 | #include <linux/kernel.h> |
1b76c13e | 29 | #include <linux/bpf.h> |
38d5d3b3 | 30 | #include <linux/btf.h> |
47eff617 | 31 | #include <linux/filter.h> |
9a208eff | 32 | #include <linux/list.h> |
f367540c | 33 | #include <linux/limits.h> |
438363c0 | 34 | #include <linux/perf_event.h> |
a64af0ef | 35 | #include <linux/ring_buffer.h> |
5e61f270 | 36 | #include <linux/version.h> |
fb84b822 | 37 | #include <sys/epoll.h> |
63f2f5ee | 38 | #include <sys/ioctl.h> |
fb84b822 | 39 | #include <sys/mman.h> |
f367540c JS |
40 | #include <sys/stat.h> |
41 | #include <sys/types.h> | |
42 | #include <sys/vfs.h> | |
ddc7c304 | 43 | #include <sys/utsname.h> |
531b014e | 44 | #include <tools/libc_compat.h> |
1a5e3fb1 WN |
45 | #include <libelf.h> |
46 | #include <gelf.h> | |
1b76c13e WN |
47 | |
48 | #include "libbpf.h" | |
52d3352e | 49 | #include "bpf.h" |
8a138aed | 50 | #include "btf.h" |
6d41907c | 51 | #include "str_error.h" |
d7c4b398 | 52 | #include "libbpf_internal.h" |
ddc7c304 | 53 | #include "hashmap.h" |
b3f59d66 | 54 | |
9b16137a WN |
55 | #ifndef EM_BPF |
56 | #define EM_BPF 247 | |
57 | #endif | |
58 | ||
f367540c JS |
59 | #ifndef BPF_FS_MAGIC |
60 | #define BPF_FS_MAGIC 0xcafe4a11 | |
61 | #endif | |
62 | ||
ff466b58 AI |
63 | /* vsprintf() in __base_pr() uses nonliteral format string. It may break |
64 | * compilation if user enables corresponding warning. Disable it explicitly. | |
65 | */ | |
66 | #pragma GCC diagnostic ignored "-Wformat-nonliteral" | |
67 | ||
b3f59d66 WN |
68 | #define __printf(a, b) __attribute__((format(printf, a, b))) |
69 | ||
a8a1f7d0 SF |
70 | static int __base_pr(enum libbpf_print_level level, const char *format, |
71 | va_list args) | |
b3f59d66 | 72 | { |
6f1ae8b6 YS |
73 | if (level == LIBBPF_DEBUG) |
74 | return 0; | |
75 | ||
a8a1f7d0 | 76 | return vfprintf(stderr, format, args); |
b3f59d66 WN |
77 | } |
78 | ||
a8a1f7d0 | 79 | static libbpf_print_fn_t __libbpf_pr = __base_pr; |
b3f59d66 | 80 | |
e87fd8ba | 81 | libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) |
b3f59d66 | 82 | { |
e87fd8ba AN |
83 | libbpf_print_fn_t old_print_fn = __libbpf_pr; |
84 | ||
6f1ae8b6 | 85 | __libbpf_pr = fn; |
e87fd8ba | 86 | return old_print_fn; |
b3f59d66 | 87 | } |
1a5e3fb1 | 88 | |
8461ef8b YS |
89 | __printf(2, 3) |
90 | void libbpf_print(enum libbpf_print_level level, const char *format, ...) | |
91 | { | |
92 | va_list args; | |
93 | ||
6f1ae8b6 YS |
94 | if (!__libbpf_pr) |
95 | return; | |
96 | ||
8461ef8b | 97 | va_start(args, format); |
6f1ae8b6 | 98 | __libbpf_pr(level, format, args); |
8461ef8b YS |
99 | va_end(args); |
100 | } | |
101 | ||
6371ca3b WN |
102 | #define STRERR_BUFSIZE 128 |
103 | ||
6371ca3b WN |
104 | #define CHECK_ERR(action, err, out) do { \ |
105 | err = action; \ | |
106 | if (err) \ | |
107 | goto out; \ | |
108 | } while(0) | |
109 | ||
110 | ||
1a5e3fb1 WN |
111 | /* Copied from tools/perf/util/util.h */ |
112 | #ifndef zfree | |
113 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
114 | #endif | |
115 | ||
116 | #ifndef zclose | |
117 | # define zclose(fd) ({ \ | |
118 | int ___err = 0; \ | |
119 | if ((fd) >= 0) \ | |
120 | ___err = close((fd)); \ | |
121 | fd = -1; \ | |
122 | ___err; }) | |
123 | #endif | |
124 | ||
125 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
126 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
127 | #else | |
128 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
129 | #endif | |
130 | ||
34be1646 SL |
131 | static inline __u64 ptr_to_u64(const void *ptr) |
132 | { | |
133 | return (__u64) (unsigned long) ptr; | |
134 | } | |
135 | ||
47eff617 SF |
136 | struct bpf_capabilities { |
137 | /* v4.14: kernel support for program & map names. */ | |
138 | __u32 name:1; | |
8837fe5d DB |
139 | /* v5.2: kernel support for global data sections. */ |
140 | __u32 global_data:1; | |
d7c4b398 AN |
141 | /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ |
142 | __u32 btf_func:1; | |
143 | /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ | |
144 | __u32 btf_datasec:1; | |
47eff617 SF |
145 | }; |
146 | ||
a5b8bd47 WN |
147 | /* |
148 | * bpf_prog should be a better name but it has been used in | |
149 | * linux/filter.h. | |
150 | */ | |
151 | struct bpf_program { | |
152 | /* Index in elf obj file, for relocation use. */ | |
153 | int idx; | |
88cda1c9 | 154 | char *name; |
f0307a7e | 155 | int prog_ifindex; |
a5b8bd47 | 156 | char *section_name; |
33a2c75c SF |
157 | /* section_name with / replaced by _; makes recursive pinning |
158 | * in bpf_object__pin_programs easier | |
159 | */ | |
160 | char *pin_name; | |
a5b8bd47 | 161 | struct bpf_insn *insns; |
48cca7e4 | 162 | size_t insns_cnt, main_prog_cnt; |
5f44e4c8 | 163 | enum bpf_prog_type type; |
34090915 | 164 | |
48cca7e4 AS |
165 | struct reloc_desc { |
166 | enum { | |
167 | RELO_LD64, | |
168 | RELO_CALL, | |
d859900c | 169 | RELO_DATA, |
48cca7e4 | 170 | } type; |
34090915 | 171 | int insn_idx; |
48cca7e4 AS |
172 | union { |
173 | int map_idx; | |
174 | int text_off; | |
175 | }; | |
34090915 WN |
176 | } *reloc_desc; |
177 | int nr_reloc; | |
da11b417 | 178 | int log_level; |
55cffde2 | 179 | |
b580563e WN |
180 | struct { |
181 | int nr; | |
182 | int *fds; | |
183 | } instances; | |
184 | bpf_program_prep_t preprocessor; | |
aa9b1ac3 WN |
185 | |
186 | struct bpf_object *obj; | |
187 | void *priv; | |
188 | bpf_program_clear_priv_t clear_priv; | |
d7be143b AI |
189 | |
190 | enum bpf_attach_type expected_attach_type; | |
2993e051 YS |
191 | void *func_info; |
192 | __u32 func_info_rec_size; | |
f0187f0b | 193 | __u32 func_info_cnt; |
47eff617 SF |
194 | |
195 | struct bpf_capabilities *caps; | |
3d650141 MKL |
196 | |
197 | void *line_info; | |
198 | __u32 line_info_rec_size; | |
199 | __u32 line_info_cnt; | |
04656198 | 200 | __u32 prog_flags; |
a5b8bd47 WN |
201 | }; |
202 | ||
d859900c DB |
203 | enum libbpf_map_type { |
204 | LIBBPF_MAP_UNSPEC, | |
205 | LIBBPF_MAP_DATA, | |
206 | LIBBPF_MAP_BSS, | |
207 | LIBBPF_MAP_RODATA, | |
208 | }; | |
209 | ||
210 | static const char * const libbpf_type_to_btf_name[] = { | |
211 | [LIBBPF_MAP_DATA] = ".data", | |
212 | [LIBBPF_MAP_BSS] = ".bss", | |
213 | [LIBBPF_MAP_RODATA] = ".rodata", | |
214 | }; | |
215 | ||
9d759a9b WN |
216 | struct bpf_map { |
217 | int fd; | |
561bbcca | 218 | char *name; |
db48814b AN |
219 | int sec_idx; |
220 | size_t sec_offset; | |
f0307a7e | 221 | int map_ifindex; |
addb9fc9 | 222 | int inner_map_fd; |
9d759a9b | 223 | struct bpf_map_def def; |
5b891af7 MKL |
224 | __u32 btf_key_type_id; |
225 | __u32 btf_value_type_id; | |
9d759a9b WN |
226 | void *priv; |
227 | bpf_map_clear_priv_t clear_priv; | |
d859900c DB |
228 | enum libbpf_map_type libbpf_type; |
229 | }; | |
230 | ||
231 | struct bpf_secdata { | |
232 | void *rodata; | |
233 | void *data; | |
9d759a9b WN |
234 | }; |
235 | ||
9a208eff WN |
236 | static LIST_HEAD(bpf_objects_list); |
237 | ||
1a5e3fb1 | 238 | struct bpf_object { |
d859900c | 239 | char name[BPF_OBJ_NAME_LEN]; |
cb1e5e96 | 240 | char license[64]; |
438363c0 | 241 | __u32 kern_version; |
0b3d1efa | 242 | |
a5b8bd47 WN |
243 | struct bpf_program *programs; |
244 | size_t nr_programs; | |
9d759a9b WN |
245 | struct bpf_map *maps; |
246 | size_t nr_maps; | |
bf829271 | 247 | size_t maps_cap; |
d859900c | 248 | struct bpf_secdata sections; |
9d759a9b | 249 | |
52d3352e | 250 | bool loaded; |
9a94f277 | 251 | bool has_pseudo_calls; |
62561eb4 | 252 | bool relaxed_core_relocs; |
a5b8bd47 | 253 | |
1a5e3fb1 WN |
254 | /* |
255 | * Information when doing elf related work. Only valid if fd | |
256 | * is valid. | |
257 | */ | |
258 | struct { | |
259 | int fd; | |
5e61f270 | 260 | const void *obj_buf; |
6c956392 | 261 | size_t obj_buf_sz; |
1a5e3fb1 WN |
262 | Elf *elf; |
263 | GElf_Ehdr ehdr; | |
bec7d68c | 264 | Elf_Data *symbols; |
d859900c DB |
265 | Elf_Data *data; |
266 | Elf_Data *rodata; | |
267 | Elf_Data *bss; | |
77ba9a5b | 268 | size_t strtabidx; |
b62f06e8 WN |
269 | struct { |
270 | GElf_Shdr shdr; | |
271 | Elf_Data *data; | |
272 | } *reloc; | |
273 | int nr_reloc; | |
666810e8 | 274 | int maps_shndx; |
abd29c93 | 275 | int btf_maps_shndx; |
48cca7e4 | 276 | int text_shndx; |
d859900c DB |
277 | int data_shndx; |
278 | int rodata_shndx; | |
279 | int bss_shndx; | |
1a5e3fb1 | 280 | } efile; |
9a208eff WN |
281 | /* |
282 | * All loaded bpf_object is linked in a list, which is | |
283 | * hidden to caller. bpf_objects__<func> handlers deal with | |
284 | * all objects. | |
285 | */ | |
286 | struct list_head list; | |
10931d24 | 287 | |
8a138aed | 288 | struct btf *btf; |
2993e051 | 289 | struct btf_ext *btf_ext; |
8a138aed | 290 | |
10931d24 WN |
291 | void *priv; |
292 | bpf_object_clear_priv_t clear_priv; | |
293 | ||
47eff617 SF |
294 | struct bpf_capabilities caps; |
295 | ||
1a5e3fb1 WN |
296 | char path[]; |
297 | }; | |
298 | #define obj_elf_valid(o) ((o)->efile.elf) | |
299 | ||
29cd77f4 | 300 | void bpf_program__unload(struct bpf_program *prog) |
55cffde2 | 301 | { |
b580563e WN |
302 | int i; |
303 | ||
55cffde2 WN |
304 | if (!prog) |
305 | return; | |
306 | ||
b580563e WN |
307 | /* |
308 | * If the object is opened but the program was never loaded, | |
309 | * it is possible that prog->instances.nr == -1. | |
310 | */ | |
311 | if (prog->instances.nr > 0) { | |
312 | for (i = 0; i < prog->instances.nr; i++) | |
313 | zclose(prog->instances.fds[i]); | |
314 | } else if (prog->instances.nr != -1) { | |
be18010e KW |
315 | pr_warn("Internal error: instances.nr is %d\n", |
316 | prog->instances.nr); | |
b580563e WN |
317 | } |
318 | ||
319 | prog->instances.nr = -1; | |
320 | zfree(&prog->instances.fds); | |
2993e051 | 321 | |
2993e051 | 322 | zfree(&prog->func_info); |
07a09d1b | 323 | zfree(&prog->line_info); |
55cffde2 WN |
324 | } |
325 | ||
a5b8bd47 WN |
326 | static void bpf_program__exit(struct bpf_program *prog) |
327 | { | |
328 | if (!prog) | |
329 | return; | |
330 | ||
aa9b1ac3 WN |
331 | if (prog->clear_priv) |
332 | prog->clear_priv(prog, prog->priv); | |
333 | ||
334 | prog->priv = NULL; | |
335 | prog->clear_priv = NULL; | |
336 | ||
55cffde2 | 337 | bpf_program__unload(prog); |
88cda1c9 | 338 | zfree(&prog->name); |
a5b8bd47 | 339 | zfree(&prog->section_name); |
33a2c75c | 340 | zfree(&prog->pin_name); |
a5b8bd47 | 341 | zfree(&prog->insns); |
34090915 WN |
342 | zfree(&prog->reloc_desc); |
343 | ||
344 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
345 | prog->insns_cnt = 0; |
346 | prog->idx = -1; | |
347 | } | |
348 | ||
33a2c75c SF |
349 | static char *__bpf_program__pin_name(struct bpf_program *prog) |
350 | { | |
351 | char *name, *p; | |
352 | ||
353 | name = p = strdup(prog->section_name); | |
354 | while ((p = strchr(p, '/'))) | |
355 | *p = '_'; | |
356 | ||
357 | return name; | |
358 | } | |
359 | ||
a5b8bd47 | 360 | static int |
88cda1c9 MKL |
361 | bpf_program__init(void *data, size_t size, char *section_name, int idx, |
362 | struct bpf_program *prog) | |
a5b8bd47 | 363 | { |
8ca990ce AN |
364 | const size_t bpf_insn_sz = sizeof(struct bpf_insn); |
365 | ||
366 | if (size == 0 || size % bpf_insn_sz) { | |
be18010e KW |
367 | pr_warn("corrupted section '%s', size: %zu\n", |
368 | section_name, size); | |
a5b8bd47 WN |
369 | return -EINVAL; |
370 | } | |
371 | ||
1ad9cbb8 | 372 | memset(prog, 0, sizeof(*prog)); |
a5b8bd47 | 373 | |
88cda1c9 | 374 | prog->section_name = strdup(section_name); |
a5b8bd47 | 375 | if (!prog->section_name) { |
be18010e KW |
376 | pr_warn("failed to alloc name for prog under section(%d) %s\n", |
377 | idx, section_name); | |
a5b8bd47 WN |
378 | goto errout; |
379 | } | |
380 | ||
33a2c75c SF |
381 | prog->pin_name = __bpf_program__pin_name(prog); |
382 | if (!prog->pin_name) { | |
be18010e KW |
383 | pr_warn("failed to alloc pin name for prog under section(%d) %s\n", |
384 | idx, section_name); | |
33a2c75c SF |
385 | goto errout; |
386 | } | |
387 | ||
a5b8bd47 WN |
388 | prog->insns = malloc(size); |
389 | if (!prog->insns) { | |
be18010e KW |
390 | pr_warn("failed to alloc insns for prog under section %s\n", |
391 | section_name); | |
a5b8bd47 WN |
392 | goto errout; |
393 | } | |
8ca990ce AN |
394 | prog->insns_cnt = size / bpf_insn_sz; |
395 | memcpy(prog->insns, data, size); | |
a5b8bd47 | 396 | prog->idx = idx; |
b580563e WN |
397 | prog->instances.fds = NULL; |
398 | prog->instances.nr = -1; | |
47ae7e3d | 399 | prog->type = BPF_PROG_TYPE_UNSPEC; |
a5b8bd47 WN |
400 | |
401 | return 0; | |
402 | errout: | |
403 | bpf_program__exit(prog); | |
404 | return -ENOMEM; | |
405 | } | |
406 | ||
407 | static int | |
408 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
88cda1c9 | 409 | char *section_name, int idx) |
a5b8bd47 WN |
410 | { |
411 | struct bpf_program prog, *progs; | |
412 | int nr_progs, err; | |
413 | ||
88cda1c9 | 414 | err = bpf_program__init(data, size, section_name, idx, &prog); |
a5b8bd47 WN |
415 | if (err) |
416 | return err; | |
417 | ||
47eff617 | 418 | prog.caps = &obj->caps; |
a5b8bd47 WN |
419 | progs = obj->programs; |
420 | nr_progs = obj->nr_programs; | |
421 | ||
531b014e | 422 | progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); |
a5b8bd47 WN |
423 | if (!progs) { |
424 | /* | |
425 | * In this case the original obj->programs | |
426 | * is still valid, so don't need special treat for | |
427 | * bpf_close_object(). | |
428 | */ | |
be18010e KW |
429 | pr_warn("failed to alloc a new program under section '%s'\n", |
430 | section_name); | |
a5b8bd47 WN |
431 | bpf_program__exit(&prog); |
432 | return -ENOMEM; | |
433 | } | |
434 | ||
435 | pr_debug("found program %s\n", prog.section_name); | |
436 | obj->programs = progs; | |
437 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 438 | prog.obj = obj; |
a5b8bd47 WN |
439 | progs[nr_progs] = prog; |
440 | return 0; | |
441 | } | |
442 | ||
88cda1c9 MKL |
443 | static int |
444 | bpf_object__init_prog_names(struct bpf_object *obj) | |
445 | { | |
446 | Elf_Data *symbols = obj->efile.symbols; | |
447 | struct bpf_program *prog; | |
448 | size_t pi, si; | |
449 | ||
450 | for (pi = 0; pi < obj->nr_programs; pi++) { | |
48cca7e4 | 451 | const char *name = NULL; |
88cda1c9 MKL |
452 | |
453 | prog = &obj->programs[pi]; | |
454 | ||
455 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; | |
456 | si++) { | |
457 | GElf_Sym sym; | |
458 | ||
459 | if (!gelf_getsym(symbols, si, &sym)) | |
460 | continue; | |
461 | if (sym.st_shndx != prog->idx) | |
462 | continue; | |
fe4d44b2 RG |
463 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) |
464 | continue; | |
88cda1c9 MKL |
465 | |
466 | name = elf_strptr(obj->efile.elf, | |
467 | obj->efile.strtabidx, | |
468 | sym.st_name); | |
469 | if (!name) { | |
be18010e KW |
470 | pr_warn("failed to get sym name string for prog %s\n", |
471 | prog->section_name); | |
88cda1c9 MKL |
472 | return -LIBBPF_ERRNO__LIBELF; |
473 | } | |
474 | } | |
475 | ||
9a94f277 JK |
476 | if (!name && prog->idx == obj->efile.text_shndx) |
477 | name = ".text"; | |
478 | ||
88cda1c9 | 479 | if (!name) { |
be18010e KW |
480 | pr_warn("failed to find sym for prog %s\n", |
481 | prog->section_name); | |
88cda1c9 MKL |
482 | return -EINVAL; |
483 | } | |
9a94f277 | 484 | |
88cda1c9 MKL |
485 | prog->name = strdup(name); |
486 | if (!prog->name) { | |
be18010e KW |
487 | pr_warn("failed to allocate memory for prog sym %s\n", |
488 | name); | |
88cda1c9 MKL |
489 | return -ENOMEM; |
490 | } | |
491 | } | |
492 | ||
493 | return 0; | |
494 | } | |
495 | ||
5e61f270 AN |
496 | static __u32 get_kernel_version(void) |
497 | { | |
498 | __u32 major, minor, patch; | |
499 | struct utsname info; | |
500 | ||
501 | uname(&info); | |
502 | if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) | |
503 | return 0; | |
504 | return KERNEL_VERSION(major, minor, patch); | |
505 | } | |
506 | ||
6c956392 | 507 | static struct bpf_object *bpf_object__new(const char *path, |
5e61f270 | 508 | const void *obj_buf, |
2ce8450e AN |
509 | size_t obj_buf_sz, |
510 | const char *obj_name) | |
1a5e3fb1 WN |
511 | { |
512 | struct bpf_object *obj; | |
d859900c | 513 | char *end; |
1a5e3fb1 WN |
514 | |
515 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
516 | if (!obj) { | |
be18010e | 517 | pr_warn("alloc memory failed for %s\n", path); |
6371ca3b | 518 | return ERR_PTR(-ENOMEM); |
1a5e3fb1 WN |
519 | } |
520 | ||
521 | strcpy(obj->path, path); | |
2ce8450e AN |
522 | if (obj_name) { |
523 | strncpy(obj->name, obj_name, sizeof(obj->name) - 1); | |
524 | obj->name[sizeof(obj->name) - 1] = 0; | |
525 | } else { | |
526 | /* Using basename() GNU version which doesn't modify arg. */ | |
527 | strncpy(obj->name, basename((void *)path), | |
528 | sizeof(obj->name) - 1); | |
529 | end = strchr(obj->name, '.'); | |
530 | if (end) | |
531 | *end = 0; | |
532 | } | |
6c956392 | 533 | |
d859900c | 534 | obj->efile.fd = -1; |
6c956392 | 535 | /* |
76e1022b | 536 | * Caller of this function should also call |
6c956392 WN |
537 | * bpf_object__elf_finish() after data collection to return |
538 | * obj_buf to user. If not, we should duplicate the buffer to | |
539 | * avoid user freeing them before elf finish. | |
540 | */ | |
541 | obj->efile.obj_buf = obj_buf; | |
542 | obj->efile.obj_buf_sz = obj_buf_sz; | |
666810e8 | 543 | obj->efile.maps_shndx = -1; |
abd29c93 | 544 | obj->efile.btf_maps_shndx = -1; |
d859900c DB |
545 | obj->efile.data_shndx = -1; |
546 | obj->efile.rodata_shndx = -1; | |
547 | obj->efile.bss_shndx = -1; | |
6c956392 | 548 | |
5e61f270 | 549 | obj->kern_version = get_kernel_version(); |
52d3352e | 550 | obj->loaded = false; |
9a208eff WN |
551 | |
552 | INIT_LIST_HEAD(&obj->list); | |
553 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
554 | return obj; |
555 | } | |
556 | ||
557 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
558 | { | |
559 | if (!obj_elf_valid(obj)) | |
560 | return; | |
561 | ||
562 | if (obj->efile.elf) { | |
563 | elf_end(obj->efile.elf); | |
564 | obj->efile.elf = NULL; | |
565 | } | |
bec7d68c | 566 | obj->efile.symbols = NULL; |
d859900c DB |
567 | obj->efile.data = NULL; |
568 | obj->efile.rodata = NULL; | |
569 | obj->efile.bss = NULL; | |
b62f06e8 WN |
570 | |
571 | zfree(&obj->efile.reloc); | |
572 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 573 | zclose(obj->efile.fd); |
6c956392 WN |
574 | obj->efile.obj_buf = NULL; |
575 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
576 | } |
577 | ||
578 | static int bpf_object__elf_init(struct bpf_object *obj) | |
579 | { | |
580 | int err = 0; | |
581 | GElf_Ehdr *ep; | |
582 | ||
583 | if (obj_elf_valid(obj)) { | |
be18010e | 584 | pr_warn("elf init: internal error\n"); |
6371ca3b | 585 | return -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
586 | } |
587 | ||
6c956392 WN |
588 | if (obj->efile.obj_buf_sz > 0) { |
589 | /* | |
590 | * obj_buf should have been validated by | |
591 | * bpf_object__open_buffer(). | |
592 | */ | |
5e61f270 | 593 | obj->efile.elf = elf_memory((char *)obj->efile.obj_buf, |
6c956392 WN |
594 | obj->efile.obj_buf_sz); |
595 | } else { | |
596 | obj->efile.fd = open(obj->path, O_RDONLY); | |
597 | if (obj->efile.fd < 0) { | |
be5c5d4e | 598 | char errmsg[STRERR_BUFSIZE], *cp; |
1ce6a9fc | 599 | |
be5c5d4e AN |
600 | err = -errno; |
601 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
be18010e | 602 | pr_warn("failed to open %s: %s\n", obj->path, cp); |
be5c5d4e | 603 | return err; |
6c956392 WN |
604 | } |
605 | ||
606 | obj->efile.elf = elf_begin(obj->efile.fd, | |
76e1022b | 607 | LIBBPF_ELF_C_READ_MMAP, NULL); |
1a5e3fb1 WN |
608 | } |
609 | ||
1a5e3fb1 | 610 | if (!obj->efile.elf) { |
be18010e | 611 | pr_warn("failed to open %s as ELF file\n", obj->path); |
6371ca3b | 612 | err = -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
613 | goto errout; |
614 | } | |
615 | ||
616 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
be18010e | 617 | pr_warn("failed to get EHDR from %s\n", obj->path); |
6371ca3b | 618 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
619 | goto errout; |
620 | } | |
621 | ep = &obj->efile.ehdr; | |
622 | ||
9b16137a | 623 | /* Old LLVM set e_machine to EM_NONE */ |
76e1022b AN |
624 | if (ep->e_type != ET_REL || |
625 | (ep->e_machine && ep->e_machine != EM_BPF)) { | |
be18010e | 626 | pr_warn("%s is not an eBPF object file\n", obj->path); |
6371ca3b | 627 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
628 | goto errout; |
629 | } | |
630 | ||
631 | return 0; | |
632 | errout: | |
633 | bpf_object__elf_finish(obj); | |
634 | return err; | |
635 | } | |
636 | ||
12ef5634 | 637 | static int bpf_object__check_endianness(struct bpf_object *obj) |
cc4228d5 | 638 | { |
cdb2f920 | 639 | #if __BYTE_ORDER == __LITTLE_ENDIAN |
12ef5634 AN |
640 | if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) |
641 | return 0; | |
cdb2f920 | 642 | #elif __BYTE_ORDER == __BIG_ENDIAN |
12ef5634 AN |
643 | if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) |
644 | return 0; | |
645 | #else | |
646 | # error "Unrecognized __BYTE_ORDER__" | |
647 | #endif | |
be18010e | 648 | pr_warn("endianness mismatch.\n"); |
6371ca3b | 649 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
650 | } |
651 | ||
cb1e5e96 | 652 | static int |
399dc65e | 653 | bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) |
cb1e5e96 | 654 | { |
399dc65e | 655 | memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); |
cb1e5e96 WN |
656 | pr_debug("license of %s is %s\n", obj->path, obj->license); |
657 | return 0; | |
658 | } | |
659 | ||
54b8625c JF |
660 | static int |
661 | bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) | |
662 | { | |
663 | __u32 kver; | |
664 | ||
665 | if (size != sizeof(kver)) { | |
be18010e | 666 | pr_warn("invalid kver section in %s\n", obj->path); |
54b8625c JF |
667 | return -LIBBPF_ERRNO__FORMAT; |
668 | } | |
669 | memcpy(&kver, data, sizeof(kver)); | |
670 | obj->kern_version = kver; | |
671 | pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); | |
672 | return 0; | |
673 | } | |
674 | ||
4708bbda EL |
675 | static int compare_bpf_map(const void *_a, const void *_b) |
676 | { | |
677 | const struct bpf_map *a = _a; | |
678 | const struct bpf_map *b = _b; | |
9d759a9b | 679 | |
db48814b AN |
680 | if (a->sec_idx != b->sec_idx) |
681 | return a->sec_idx - b->sec_idx; | |
682 | return a->sec_offset - b->sec_offset; | |
0b3d1efa WN |
683 | } |
684 | ||
addb9fc9 NS |
685 | static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) |
686 | { | |
687 | if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || | |
688 | type == BPF_MAP_TYPE_HASH_OF_MAPS) | |
689 | return true; | |
690 | return false; | |
691 | } | |
692 | ||
1713d68b DB |
693 | static int bpf_object_search_section_size(const struct bpf_object *obj, |
694 | const char *name, size_t *d_size) | |
695 | { | |
696 | const GElf_Ehdr *ep = &obj->efile.ehdr; | |
697 | Elf *elf = obj->efile.elf; | |
698 | Elf_Scn *scn = NULL; | |
699 | int idx = 0; | |
700 | ||
701 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
702 | const char *sec_name; | |
703 | Elf_Data *data; | |
704 | GElf_Shdr sh; | |
705 | ||
706 | idx++; | |
707 | if (gelf_getshdr(scn, &sh) != &sh) { | |
be18010e KW |
708 | pr_warn("failed to get section(%d) header from %s\n", |
709 | idx, obj->path); | |
1713d68b DB |
710 | return -EIO; |
711 | } | |
712 | ||
713 | sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
714 | if (!sec_name) { | |
be18010e KW |
715 | pr_warn("failed to get section(%d) name from %s\n", |
716 | idx, obj->path); | |
1713d68b DB |
717 | return -EIO; |
718 | } | |
719 | ||
720 | if (strcmp(name, sec_name)) | |
721 | continue; | |
722 | ||
723 | data = elf_getdata(scn, 0); | |
724 | if (!data) { | |
be18010e KW |
725 | pr_warn("failed to get section(%d) data from %s(%s)\n", |
726 | idx, name, obj->path); | |
1713d68b DB |
727 | return -EIO; |
728 | } | |
729 | ||
730 | *d_size = data->d_size; | |
731 | return 0; | |
732 | } | |
733 | ||
734 | return -ENOENT; | |
735 | } | |
736 | ||
737 | int bpf_object__section_size(const struct bpf_object *obj, const char *name, | |
738 | __u32 *size) | |
739 | { | |
740 | int ret = -ENOENT; | |
741 | size_t d_size; | |
742 | ||
743 | *size = 0; | |
744 | if (!name) { | |
745 | return -EINVAL; | |
746 | } else if (!strcmp(name, ".data")) { | |
747 | if (obj->efile.data) | |
748 | *size = obj->efile.data->d_size; | |
749 | } else if (!strcmp(name, ".bss")) { | |
750 | if (obj->efile.bss) | |
751 | *size = obj->efile.bss->d_size; | |
752 | } else if (!strcmp(name, ".rodata")) { | |
753 | if (obj->efile.rodata) | |
754 | *size = obj->efile.rodata->d_size; | |
755 | } else { | |
756 | ret = bpf_object_search_section_size(obj, name, &d_size); | |
757 | if (!ret) | |
758 | *size = d_size; | |
759 | } | |
760 | ||
761 | return *size ? 0 : ret; | |
762 | } | |
763 | ||
764 | int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, | |
765 | __u32 *off) | |
766 | { | |
767 | Elf_Data *symbols = obj->efile.symbols; | |
768 | const char *sname; | |
769 | size_t si; | |
770 | ||
771 | if (!name || !off) | |
772 | return -EINVAL; | |
773 | ||
774 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { | |
775 | GElf_Sym sym; | |
776 | ||
777 | if (!gelf_getsym(symbols, si, &sym)) | |
778 | continue; | |
779 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || | |
780 | GELF_ST_TYPE(sym.st_info) != STT_OBJECT) | |
781 | continue; | |
782 | ||
783 | sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
784 | sym.st_name); | |
785 | if (!sname) { | |
be18010e KW |
786 | pr_warn("failed to get sym name string for var %s\n", |
787 | name); | |
1713d68b DB |
788 | return -EIO; |
789 | } | |
790 | if (strcmp(name, sname) == 0) { | |
791 | *off = sym.st_value; | |
792 | return 0; | |
793 | } | |
794 | } | |
795 | ||
796 | return -ENOENT; | |
797 | } | |
798 | ||
bf829271 | 799 | static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) |
d859900c | 800 | { |
bf829271 AN |
801 | struct bpf_map *new_maps; |
802 | size_t new_cap; | |
803 | int i; | |
804 | ||
805 | if (obj->nr_maps < obj->maps_cap) | |
806 | return &obj->maps[obj->nr_maps++]; | |
807 | ||
95064979 | 808 | new_cap = max((size_t)4, obj->maps_cap * 3 / 2); |
bf829271 AN |
809 | new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); |
810 | if (!new_maps) { | |
be18010e | 811 | pr_warn("alloc maps for object failed\n"); |
bf829271 AN |
812 | return ERR_PTR(-ENOMEM); |
813 | } | |
814 | ||
815 | obj->maps_cap = new_cap; | |
816 | obj->maps = new_maps; | |
817 | ||
818 | /* zero out new maps */ | |
819 | memset(obj->maps + obj->nr_maps, 0, | |
820 | (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); | |
821 | /* | |
822 | * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) | |
823 | * when failure (zclose won't close negative fd)). | |
824 | */ | |
825 | for (i = obj->nr_maps; i < obj->maps_cap; i++) { | |
826 | obj->maps[i].fd = -1; | |
827 | obj->maps[i].inner_map_fd = -1; | |
828 | } | |
829 | ||
830 | return &obj->maps[obj->nr_maps++]; | |
d859900c DB |
831 | } |
832 | ||
833 | static int | |
bf829271 | 834 | bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, |
db48814b | 835 | int sec_idx, Elf_Data *data, void **data_buff) |
d859900c | 836 | { |
d859900c | 837 | char map_name[BPF_OBJ_NAME_LEN]; |
bf829271 AN |
838 | struct bpf_map_def *def; |
839 | struct bpf_map *map; | |
840 | ||
841 | map = bpf_object__add_map(obj); | |
842 | if (IS_ERR(map)) | |
843 | return PTR_ERR(map); | |
d859900c DB |
844 | |
845 | map->libbpf_type = type; | |
db48814b AN |
846 | map->sec_idx = sec_idx; |
847 | map->sec_offset = 0; | |
d859900c DB |
848 | snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name, |
849 | libbpf_type_to_btf_name[type]); | |
850 | map->name = strdup(map_name); | |
851 | if (!map->name) { | |
be18010e | 852 | pr_warn("failed to alloc map name\n"); |
d859900c DB |
853 | return -ENOMEM; |
854 | } | |
db48814b AN |
855 | pr_debug("map '%s' (global data): at sec_idx %d, offset %zu.\n", |
856 | map_name, map->sec_idx, map->sec_offset); | |
d859900c | 857 | |
bf829271 | 858 | def = &map->def; |
d859900c DB |
859 | def->type = BPF_MAP_TYPE_ARRAY; |
860 | def->key_size = sizeof(int); | |
861 | def->value_size = data->d_size; | |
862 | def->max_entries = 1; | |
399dc65e | 863 | def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0; |
d859900c DB |
864 | if (data_buff) { |
865 | *data_buff = malloc(data->d_size); | |
866 | if (!*data_buff) { | |
867 | zfree(&map->name); | |
be18010e | 868 | pr_warn("failed to alloc map content buffer\n"); |
d859900c DB |
869 | return -ENOMEM; |
870 | } | |
871 | memcpy(*data_buff, data->d_buf, data->d_size); | |
872 | } | |
873 | ||
e1d1dc46 | 874 | pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); |
d859900c DB |
875 | return 0; |
876 | } | |
877 | ||
bf829271 AN |
878 | static int bpf_object__init_global_data_maps(struct bpf_object *obj) |
879 | { | |
880 | int err; | |
881 | ||
882 | if (!obj->caps.global_data) | |
883 | return 0; | |
884 | /* | |
885 | * Populate obj->maps with libbpf internal maps. | |
886 | */ | |
887 | if (obj->efile.data_shndx >= 0) { | |
888 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, | |
db48814b | 889 | obj->efile.data_shndx, |
bf829271 AN |
890 | obj->efile.data, |
891 | &obj->sections.data); | |
892 | if (err) | |
893 | return err; | |
894 | } | |
895 | if (obj->efile.rodata_shndx >= 0) { | |
896 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, | |
db48814b | 897 | obj->efile.rodata_shndx, |
bf829271 AN |
898 | obj->efile.rodata, |
899 | &obj->sections.rodata); | |
900 | if (err) | |
901 | return err; | |
902 | } | |
903 | if (obj->efile.bss_shndx >= 0) { | |
904 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, | |
db48814b | 905 | obj->efile.bss_shndx, |
bf829271 AN |
906 | obj->efile.bss, NULL); |
907 | if (err) | |
908 | return err; | |
909 | } | |
910 | return 0; | |
911 | } | |
912 | ||
913 | static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) | |
561bbcca | 914 | { |
561bbcca | 915 | Elf_Data *symbols = obj->efile.symbols; |
bf829271 | 916 | int i, map_def_sz = 0, nr_maps = 0, nr_syms; |
d859900c | 917 | Elf_Data *data = NULL; |
bf829271 AN |
918 | Elf_Scn *scn; |
919 | ||
920 | if (obj->efile.maps_shndx < 0) | |
921 | return 0; | |
561bbcca | 922 | |
4708bbda EL |
923 | if (!symbols) |
924 | return -EINVAL; | |
925 | ||
bf829271 AN |
926 | scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); |
927 | if (scn) | |
928 | data = elf_getdata(scn, NULL); | |
929 | if (!scn || !data) { | |
be18010e KW |
930 | pr_warn("failed to get Elf_Data from map section %d\n", |
931 | obj->efile.maps_shndx); | |
bf829271 | 932 | return -EINVAL; |
4708bbda | 933 | } |
561bbcca | 934 | |
4708bbda EL |
935 | /* |
936 | * Count number of maps. Each map has a name. | |
937 | * Array of maps is not supported: only the first element is | |
938 | * considered. | |
939 | * | |
940 | * TODO: Detect array of map and report error. | |
941 | */ | |
bf829271 AN |
942 | nr_syms = symbols->d_size / sizeof(GElf_Sym); |
943 | for (i = 0; i < nr_syms; i++) { | |
561bbcca | 944 | GElf_Sym sym; |
4708bbda EL |
945 | |
946 | if (!gelf_getsym(symbols, i, &sym)) | |
947 | continue; | |
948 | if (sym.st_shndx != obj->efile.maps_shndx) | |
949 | continue; | |
950 | nr_maps++; | |
951 | } | |
b13c5c14 | 952 | /* Assume equally sized map definitions */ |
bf829271 AN |
953 | pr_debug("maps in %s: %d maps in %zd bytes\n", |
954 | obj->path, nr_maps, data->d_size); | |
955 | ||
956 | map_def_sz = data->d_size / nr_maps; | |
957 | if (!data->d_size || (data->d_size % nr_maps) != 0) { | |
be18010e KW |
958 | pr_warn("unable to determine map definition size " |
959 | "section %s, %d maps in %zd bytes\n", | |
960 | obj->path, nr_maps, data->d_size); | |
bf829271 | 961 | return -EINVAL; |
addb9fc9 | 962 | } |
4708bbda | 963 | |
bf829271 AN |
964 | /* Fill obj->maps using data in "maps" section. */ |
965 | for (i = 0; i < nr_syms; i++) { | |
4708bbda | 966 | GElf_Sym sym; |
561bbcca | 967 | const char *map_name; |
4708bbda | 968 | struct bpf_map_def *def; |
bf829271 | 969 | struct bpf_map *map; |
561bbcca WN |
970 | |
971 | if (!gelf_getsym(symbols, i, &sym)) | |
972 | continue; | |
666810e8 | 973 | if (sym.st_shndx != obj->efile.maps_shndx) |
561bbcca WN |
974 | continue; |
975 | ||
bf829271 AN |
976 | map = bpf_object__add_map(obj); |
977 | if (IS_ERR(map)) | |
978 | return PTR_ERR(map); | |
979 | ||
980 | map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
561bbcca | 981 | sym.st_name); |
c51829bb | 982 | if (!map_name) { |
be18010e KW |
983 | pr_warn("failed to get map #%d name sym string for obj %s\n", |
984 | i, obj->path); | |
c51829bb AN |
985 | return -LIBBPF_ERRNO__FORMAT; |
986 | } | |
d859900c | 987 | |
bf829271 | 988 | map->libbpf_type = LIBBPF_MAP_UNSPEC; |
db48814b AN |
989 | map->sec_idx = sym.st_shndx; |
990 | map->sec_offset = sym.st_value; | |
991 | pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", | |
992 | map_name, map->sec_idx, map->sec_offset); | |
b13c5c14 | 993 | if (sym.st_value + map_def_sz > data->d_size) { |
be18010e KW |
994 | pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", |
995 | obj->path, map_name); | |
4708bbda | 996 | return -EINVAL; |
561bbcca | 997 | } |
4708bbda | 998 | |
bf829271 AN |
999 | map->name = strdup(map_name); |
1000 | if (!map->name) { | |
be18010e | 1001 | pr_warn("failed to alloc map name\n"); |
973170e6 WN |
1002 | return -ENOMEM; |
1003 | } | |
bf829271 | 1004 | pr_debug("map %d is \"%s\"\n", i, map->name); |
4708bbda | 1005 | def = (struct bpf_map_def *)(data->d_buf + sym.st_value); |
b13c5c14 CG |
1006 | /* |
1007 | * If the definition of the map in the object file fits in | |
1008 | * bpf_map_def, copy it. Any extra fields in our version | |
1009 | * of bpf_map_def will default to zero as a result of the | |
1010 | * calloc above. | |
1011 | */ | |
1012 | if (map_def_sz <= sizeof(struct bpf_map_def)) { | |
bf829271 | 1013 | memcpy(&map->def, def, map_def_sz); |
b13c5c14 CG |
1014 | } else { |
1015 | /* | |
1016 | * Here the map structure being read is bigger than what | |
1017 | * we expect, truncate if the excess bits are all zero. | |
1018 | * If they are not zero, reject this map as | |
1019 | * incompatible. | |
1020 | */ | |
1021 | char *b; | |
1022 | for (b = ((char *)def) + sizeof(struct bpf_map_def); | |
1023 | b < ((char *)def) + map_def_sz; b++) { | |
1024 | if (*b != 0) { | |
be18010e KW |
1025 | pr_warn("maps section in %s: \"%s\" " |
1026 | "has unrecognized, non-zero " | |
1027 | "options\n", | |
1028 | obj->path, map_name); | |
c034a177 JF |
1029 | if (strict) |
1030 | return -EINVAL; | |
b13c5c14 CG |
1031 | } |
1032 | } | |
bf829271 | 1033 | memcpy(&map->def, def, sizeof(struct bpf_map_def)); |
b13c5c14 | 1034 | } |
561bbcca | 1035 | } |
bf829271 AN |
1036 | return 0; |
1037 | } | |
4708bbda | 1038 | |
ddc7c304 AN |
1039 | static const struct btf_type * |
1040 | skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) | |
abd29c93 AN |
1041 | { |
1042 | const struct btf_type *t = btf__type_by_id(btf, id); | |
8837fe5d | 1043 | |
ddc7c304 AN |
1044 | if (res_id) |
1045 | *res_id = id; | |
1046 | ||
1047 | while (btf_is_mod(t) || btf_is_typedef(t)) { | |
1048 | if (res_id) | |
1049 | *res_id = t->type; | |
1050 | t = btf__type_by_id(btf, t->type); | |
abd29c93 | 1051 | } |
ddc7c304 AN |
1052 | |
1053 | return t; | |
abd29c93 AN |
1054 | } |
1055 | ||
ef99b02b AN |
1056 | /* |
1057 | * Fetch integer attribute of BTF map definition. Such attributes are | |
1058 | * represented using a pointer to an array, in which dimensionality of array | |
1059 | * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; | |
1060 | * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF | |
1061 | * type definition, while using only sizeof(void *) space in ELF data section. | |
1062 | */ | |
1063 | static bool get_map_field_int(const char *map_name, const struct btf *btf, | |
abd29c93 | 1064 | const struct btf_type *def, |
ef99b02b | 1065 | const struct btf_member *m, __u32 *res) { |
ddc7c304 | 1066 | const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); |
abd29c93 | 1067 | const char *name = btf__name_by_offset(btf, m->name_off); |
ef99b02b AN |
1068 | const struct btf_array *arr_info; |
1069 | const struct btf_type *arr_t; | |
abd29c93 | 1070 | |
b03bc685 | 1071 | if (!btf_is_ptr(t)) { |
be18010e KW |
1072 | pr_warn("map '%s': attr '%s': expected PTR, got %u.\n", |
1073 | map_name, name, btf_kind(t)); | |
abd29c93 AN |
1074 | return false; |
1075 | } | |
ef99b02b AN |
1076 | |
1077 | arr_t = btf__type_by_id(btf, t->type); | |
1078 | if (!arr_t) { | |
be18010e KW |
1079 | pr_warn("map '%s': attr '%s': type [%u] not found.\n", |
1080 | map_name, name, t->type); | |
abd29c93 AN |
1081 | return false; |
1082 | } | |
b03bc685 | 1083 | if (!btf_is_array(arr_t)) { |
be18010e KW |
1084 | pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n", |
1085 | map_name, name, btf_kind(arr_t)); | |
abd29c93 AN |
1086 | return false; |
1087 | } | |
b03bc685 | 1088 | arr_info = btf_array(arr_t); |
ef99b02b | 1089 | *res = arr_info->nelems; |
abd29c93 AN |
1090 | return true; |
1091 | } | |
1092 | ||
1093 | static int bpf_object__init_user_btf_map(struct bpf_object *obj, | |
1094 | const struct btf_type *sec, | |
1095 | int var_idx, int sec_idx, | |
1096 | const Elf_Data *data, bool strict) | |
1097 | { | |
1098 | const struct btf_type *var, *def, *t; | |
1099 | const struct btf_var_secinfo *vi; | |
1100 | const struct btf_var *var_extra; | |
1101 | const struct btf_member *m; | |
abd29c93 AN |
1102 | const char *map_name; |
1103 | struct bpf_map *map; | |
1104 | int vlen, i; | |
1105 | ||
b03bc685 | 1106 | vi = btf_var_secinfos(sec) + var_idx; |
abd29c93 | 1107 | var = btf__type_by_id(obj->btf, vi->type); |
b03bc685 | 1108 | var_extra = btf_var(var); |
abd29c93 | 1109 | map_name = btf__name_by_offset(obj->btf, var->name_off); |
b03bc685 | 1110 | vlen = btf_vlen(var); |
abd29c93 AN |
1111 | |
1112 | if (map_name == NULL || map_name[0] == '\0') { | |
be18010e | 1113 | pr_warn("map #%d: empty name.\n", var_idx); |
abd29c93 AN |
1114 | return -EINVAL; |
1115 | } | |
1116 | if ((__u64)vi->offset + vi->size > data->d_size) { | |
be18010e | 1117 | pr_warn("map '%s' BTF data is corrupted.\n", map_name); |
abd29c93 AN |
1118 | return -EINVAL; |
1119 | } | |
b03bc685 | 1120 | if (!btf_is_var(var)) { |
be18010e KW |
1121 | pr_warn("map '%s': unexpected var kind %u.\n", |
1122 | map_name, btf_kind(var)); | |
abd29c93 AN |
1123 | return -EINVAL; |
1124 | } | |
1125 | if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && | |
1126 | var_extra->linkage != BTF_VAR_STATIC) { | |
be18010e KW |
1127 | pr_warn("map '%s': unsupported var linkage %u.\n", |
1128 | map_name, var_extra->linkage); | |
abd29c93 AN |
1129 | return -EOPNOTSUPP; |
1130 | } | |
1131 | ||
ddc7c304 | 1132 | def = skip_mods_and_typedefs(obj->btf, var->type, NULL); |
b03bc685 | 1133 | if (!btf_is_struct(def)) { |
be18010e KW |
1134 | pr_warn("map '%s': unexpected def kind %u.\n", |
1135 | map_name, btf_kind(var)); | |
abd29c93 AN |
1136 | return -EINVAL; |
1137 | } | |
1138 | if (def->size > vi->size) { | |
be18010e | 1139 | pr_warn("map '%s': invalid def size.\n", map_name); |
abd29c93 AN |
1140 | return -EINVAL; |
1141 | } | |
1142 | ||
1143 | map = bpf_object__add_map(obj); | |
1144 | if (IS_ERR(map)) | |
1145 | return PTR_ERR(map); | |
1146 | map->name = strdup(map_name); | |
1147 | if (!map->name) { | |
be18010e | 1148 | pr_warn("map '%s': failed to alloc map name.\n", map_name); |
abd29c93 AN |
1149 | return -ENOMEM; |
1150 | } | |
1151 | map->libbpf_type = LIBBPF_MAP_UNSPEC; | |
1152 | map->def.type = BPF_MAP_TYPE_UNSPEC; | |
1153 | map->sec_idx = sec_idx; | |
1154 | map->sec_offset = vi->offset; | |
1155 | pr_debug("map '%s': at sec_idx %d, offset %zu.\n", | |
1156 | map_name, map->sec_idx, map->sec_offset); | |
1157 | ||
b03bc685 AN |
1158 | vlen = btf_vlen(def); |
1159 | m = btf_members(def); | |
abd29c93 AN |
1160 | for (i = 0; i < vlen; i++, m++) { |
1161 | const char *name = btf__name_by_offset(obj->btf, m->name_off); | |
1162 | ||
1163 | if (!name) { | |
be18010e | 1164 | pr_warn("map '%s': invalid field #%d.\n", map_name, i); |
abd29c93 AN |
1165 | return -EINVAL; |
1166 | } | |
1167 | if (strcmp(name, "type") == 0) { | |
1168 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1169 | &map->def.type)) |
abd29c93 AN |
1170 | return -EINVAL; |
1171 | pr_debug("map '%s': found type = %u.\n", | |
1172 | map_name, map->def.type); | |
1173 | } else if (strcmp(name, "max_entries") == 0) { | |
1174 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1175 | &map->def.max_entries)) |
abd29c93 AN |
1176 | return -EINVAL; |
1177 | pr_debug("map '%s': found max_entries = %u.\n", | |
1178 | map_name, map->def.max_entries); | |
1179 | } else if (strcmp(name, "map_flags") == 0) { | |
1180 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1181 | &map->def.map_flags)) |
abd29c93 AN |
1182 | return -EINVAL; |
1183 | pr_debug("map '%s': found map_flags = %u.\n", | |
1184 | map_name, map->def.map_flags); | |
1185 | } else if (strcmp(name, "key_size") == 0) { | |
1186 | __u32 sz; | |
1187 | ||
1188 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1189 | &sz)) |
abd29c93 AN |
1190 | return -EINVAL; |
1191 | pr_debug("map '%s': found key_size = %u.\n", | |
1192 | map_name, sz); | |
1193 | if (map->def.key_size && map->def.key_size != sz) { | |
be18010e KW |
1194 | pr_warn("map '%s': conflicting key size %u != %u.\n", |
1195 | map_name, map->def.key_size, sz); | |
abd29c93 AN |
1196 | return -EINVAL; |
1197 | } | |
1198 | map->def.key_size = sz; | |
1199 | } else if (strcmp(name, "key") == 0) { | |
1200 | __s64 sz; | |
1201 | ||
1202 | t = btf__type_by_id(obj->btf, m->type); | |
1203 | if (!t) { | |
be18010e KW |
1204 | pr_warn("map '%s': key type [%d] not found.\n", |
1205 | map_name, m->type); | |
abd29c93 AN |
1206 | return -EINVAL; |
1207 | } | |
b03bc685 | 1208 | if (!btf_is_ptr(t)) { |
be18010e KW |
1209 | pr_warn("map '%s': key spec is not PTR: %u.\n", |
1210 | map_name, btf_kind(t)); | |
abd29c93 AN |
1211 | return -EINVAL; |
1212 | } | |
1213 | sz = btf__resolve_size(obj->btf, t->type); | |
1214 | if (sz < 0) { | |
be18010e KW |
1215 | pr_warn("map '%s': can't determine key size for type [%u]: %lld.\n", |
1216 | map_name, t->type, sz); | |
abd29c93 AN |
1217 | return sz; |
1218 | } | |
1219 | pr_debug("map '%s': found key [%u], sz = %lld.\n", | |
1220 | map_name, t->type, sz); | |
1221 | if (map->def.key_size && map->def.key_size != sz) { | |
be18010e KW |
1222 | pr_warn("map '%s': conflicting key size %u != %lld.\n", |
1223 | map_name, map->def.key_size, sz); | |
abd29c93 AN |
1224 | return -EINVAL; |
1225 | } | |
1226 | map->def.key_size = sz; | |
1227 | map->btf_key_type_id = t->type; | |
1228 | } else if (strcmp(name, "value_size") == 0) { | |
1229 | __u32 sz; | |
1230 | ||
1231 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1232 | &sz)) |
abd29c93 AN |
1233 | return -EINVAL; |
1234 | pr_debug("map '%s': found value_size = %u.\n", | |
1235 | map_name, sz); | |
1236 | if (map->def.value_size && map->def.value_size != sz) { | |
be18010e KW |
1237 | pr_warn("map '%s': conflicting value size %u != %u.\n", |
1238 | map_name, map->def.value_size, sz); | |
abd29c93 AN |
1239 | return -EINVAL; |
1240 | } | |
1241 | map->def.value_size = sz; | |
1242 | } else if (strcmp(name, "value") == 0) { | |
1243 | __s64 sz; | |
1244 | ||
1245 | t = btf__type_by_id(obj->btf, m->type); | |
1246 | if (!t) { | |
be18010e KW |
1247 | pr_warn("map '%s': value type [%d] not found.\n", |
1248 | map_name, m->type); | |
abd29c93 AN |
1249 | return -EINVAL; |
1250 | } | |
b03bc685 | 1251 | if (!btf_is_ptr(t)) { |
be18010e KW |
1252 | pr_warn("map '%s': value spec is not PTR: %u.\n", |
1253 | map_name, btf_kind(t)); | |
abd29c93 AN |
1254 | return -EINVAL; |
1255 | } | |
1256 | sz = btf__resolve_size(obj->btf, t->type); | |
1257 | if (sz < 0) { | |
be18010e KW |
1258 | pr_warn("map '%s': can't determine value size for type [%u]: %lld.\n", |
1259 | map_name, t->type, sz); | |
abd29c93 AN |
1260 | return sz; |
1261 | } | |
1262 | pr_debug("map '%s': found value [%u], sz = %lld.\n", | |
1263 | map_name, t->type, sz); | |
1264 | if (map->def.value_size && map->def.value_size != sz) { | |
be18010e KW |
1265 | pr_warn("map '%s': conflicting value size %u != %lld.\n", |
1266 | map_name, map->def.value_size, sz); | |
abd29c93 AN |
1267 | return -EINVAL; |
1268 | } | |
1269 | map->def.value_size = sz; | |
1270 | map->btf_value_type_id = t->type; | |
1271 | } else { | |
1272 | if (strict) { | |
be18010e KW |
1273 | pr_warn("map '%s': unknown field '%s'.\n", |
1274 | map_name, name); | |
abd29c93 AN |
1275 | return -ENOTSUP; |
1276 | } | |
1277 | pr_debug("map '%s': ignoring unknown field '%s'.\n", | |
1278 | map_name, name); | |
1279 | } | |
1280 | } | |
1281 | ||
1282 | if (map->def.type == BPF_MAP_TYPE_UNSPEC) { | |
be18010e | 1283 | pr_warn("map '%s': map type isn't specified.\n", map_name); |
abd29c93 AN |
1284 | return -EINVAL; |
1285 | } | |
1286 | ||
1287 | return 0; | |
1288 | } | |
1289 | ||
1290 | static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict) | |
1291 | { | |
1292 | const struct btf_type *sec = NULL; | |
1293 | int nr_types, i, vlen, err; | |
1294 | const struct btf_type *t; | |
1295 | const char *name; | |
1296 | Elf_Data *data; | |
1297 | Elf_Scn *scn; | |
1298 | ||
1299 | if (obj->efile.btf_maps_shndx < 0) | |
1300 | return 0; | |
1301 | ||
1302 | scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); | |
1303 | if (scn) | |
1304 | data = elf_getdata(scn, NULL); | |
1305 | if (!scn || !data) { | |
be18010e KW |
1306 | pr_warn("failed to get Elf_Data from map section %d (%s)\n", |
1307 | obj->efile.maps_shndx, MAPS_ELF_SEC); | |
abd29c93 AN |
1308 | return -EINVAL; |
1309 | } | |
1310 | ||
1311 | nr_types = btf__get_nr_types(obj->btf); | |
1312 | for (i = 1; i <= nr_types; i++) { | |
1313 | t = btf__type_by_id(obj->btf, i); | |
b03bc685 | 1314 | if (!btf_is_datasec(t)) |
abd29c93 AN |
1315 | continue; |
1316 | name = btf__name_by_offset(obj->btf, t->name_off); | |
1317 | if (strcmp(name, MAPS_ELF_SEC) == 0) { | |
1318 | sec = t; | |
1319 | break; | |
1320 | } | |
1321 | } | |
1322 | ||
1323 | if (!sec) { | |
be18010e | 1324 | pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); |
abd29c93 AN |
1325 | return -ENOENT; |
1326 | } | |
1327 | ||
b03bc685 | 1328 | vlen = btf_vlen(sec); |
abd29c93 AN |
1329 | for (i = 0; i < vlen; i++) { |
1330 | err = bpf_object__init_user_btf_map(obj, sec, i, | |
1331 | obj->efile.btf_maps_shndx, | |
1332 | data, strict); | |
1333 | if (err) | |
1334 | return err; | |
1335 | } | |
1336 | ||
1337 | return 0; | |
1338 | } | |
1339 | ||
291ee02b | 1340 | static int bpf_object__init_maps(struct bpf_object *obj, bool relaxed_maps) |
bf829271 | 1341 | { |
291ee02b | 1342 | bool strict = !relaxed_maps; |
bf829271 | 1343 | int err; |
8837fe5d | 1344 | |
bf829271 AN |
1345 | err = bpf_object__init_user_maps(obj, strict); |
1346 | if (err) | |
1347 | return err; | |
1348 | ||
abd29c93 AN |
1349 | err = bpf_object__init_user_btf_maps(obj, strict); |
1350 | if (err) | |
1351 | return err; | |
1352 | ||
bf829271 AN |
1353 | err = bpf_object__init_global_data_maps(obj); |
1354 | if (err) | |
1355 | return err; | |
1356 | ||
1357 | if (obj->nr_maps) { | |
d859900c DB |
1358 | qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), |
1359 | compare_bpf_map); | |
bf829271 AN |
1360 | } |
1361 | return 0; | |
561bbcca WN |
1362 | } |
1363 | ||
e3d91b0c JDB |
1364 | static bool section_have_execinstr(struct bpf_object *obj, int idx) |
1365 | { | |
1366 | Elf_Scn *scn; | |
1367 | GElf_Shdr sh; | |
1368 | ||
1369 | scn = elf_getscn(obj->efile.elf, idx); | |
1370 | if (!scn) | |
1371 | return false; | |
1372 | ||
1373 | if (gelf_getshdr(scn, &sh) != &sh) | |
1374 | return false; | |
1375 | ||
1376 | if (sh.sh_flags & SHF_EXECINSTR) | |
1377 | return true; | |
1378 | ||
1379 | return false; | |
1380 | } | |
1381 | ||
d7c4b398 AN |
1382 | static void bpf_object__sanitize_btf(struct bpf_object *obj) |
1383 | { | |
1384 | bool has_datasec = obj->caps.btf_datasec; | |
1385 | bool has_func = obj->caps.btf_func; | |
1386 | struct btf *btf = obj->btf; | |
1387 | struct btf_type *t; | |
1388 | int i, j, vlen; | |
d7c4b398 AN |
1389 | |
1390 | if (!obj->btf || (has_func && has_datasec)) | |
1391 | return; | |
1392 | ||
1393 | for (i = 1; i <= btf__get_nr_types(btf); i++) { | |
1394 | t = (struct btf_type *)btf__type_by_id(btf, i); | |
d7c4b398 | 1395 | |
b03bc685 | 1396 | if (!has_datasec && btf_is_var(t)) { |
d7c4b398 AN |
1397 | /* replace VAR with INT */ |
1398 | t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); | |
1d4126c4 AN |
1399 | /* |
1400 | * using size = 1 is the safest choice, 4 will be too | |
1401 | * big and cause kernel BTF validation failure if | |
1402 | * original variable took less than 4 bytes | |
1403 | */ | |
1404 | t->size = 1; | |
708852dc | 1405 | *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); |
b03bc685 | 1406 | } else if (!has_datasec && btf_is_datasec(t)) { |
d7c4b398 | 1407 | /* replace DATASEC with STRUCT */ |
b03bc685 AN |
1408 | const struct btf_var_secinfo *v = btf_var_secinfos(t); |
1409 | struct btf_member *m = btf_members(t); | |
d7c4b398 AN |
1410 | struct btf_type *vt; |
1411 | char *name; | |
1412 | ||
1413 | name = (char *)btf__name_by_offset(btf, t->name_off); | |
1414 | while (*name) { | |
1415 | if (*name == '.') | |
1416 | *name = '_'; | |
1417 | name++; | |
1418 | } | |
1419 | ||
b03bc685 | 1420 | vlen = btf_vlen(t); |
d7c4b398 AN |
1421 | t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); |
1422 | for (j = 0; j < vlen; j++, v++, m++) { | |
1423 | /* order of field assignments is important */ | |
1424 | m->offset = v->offset * 8; | |
1425 | m->type = v->type; | |
1426 | /* preserve variable name as member name */ | |
1427 | vt = (void *)btf__type_by_id(btf, v->type); | |
1428 | m->name_off = vt->name_off; | |
1429 | } | |
b03bc685 | 1430 | } else if (!has_func && btf_is_func_proto(t)) { |
d7c4b398 | 1431 | /* replace FUNC_PROTO with ENUM */ |
b03bc685 | 1432 | vlen = btf_vlen(t); |
d7c4b398 AN |
1433 | t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); |
1434 | t->size = sizeof(__u32); /* kernel enforced */ | |
b03bc685 | 1435 | } else if (!has_func && btf_is_func(t)) { |
d7c4b398 AN |
1436 | /* replace FUNC with TYPEDEF */ |
1437 | t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); | |
1438 | } | |
1439 | } | |
1440 | } | |
1441 | ||
1442 | static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) | |
1443 | { | |
1444 | if (!obj->btf_ext) | |
1445 | return; | |
1446 | ||
1447 | if (!obj->caps.btf_func) { | |
1448 | btf_ext__free(obj->btf_ext); | |
1449 | obj->btf_ext = NULL; | |
1450 | } | |
1451 | } | |
1452 | ||
abd29c93 AN |
1453 | static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj) |
1454 | { | |
1455 | return obj->efile.btf_maps_shndx >= 0; | |
1456 | } | |
1457 | ||
063183bf | 1458 | static int bpf_object__init_btf(struct bpf_object *obj, |
9c6660d0 AN |
1459 | Elf_Data *btf_data, |
1460 | Elf_Data *btf_ext_data) | |
1461 | { | |
abd29c93 | 1462 | bool btf_required = bpf_object__is_btf_mandatory(obj); |
9c6660d0 AN |
1463 | int err = 0; |
1464 | ||
1465 | if (btf_data) { | |
1466 | obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); | |
1467 | if (IS_ERR(obj->btf)) { | |
be18010e KW |
1468 | pr_warn("Error loading ELF section %s: %d.\n", |
1469 | BTF_ELF_SEC, err); | |
9c6660d0 AN |
1470 | goto out; |
1471 | } | |
1472 | err = btf__finalize_data(obj, obj->btf); | |
1473 | if (err) { | |
be18010e | 1474 | pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err); |
9c6660d0 AN |
1475 | goto out; |
1476 | } | |
9c6660d0 AN |
1477 | } |
1478 | if (btf_ext_data) { | |
1479 | if (!obj->btf) { | |
1480 | pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", | |
1481 | BTF_EXT_ELF_SEC, BTF_ELF_SEC); | |
1482 | goto out; | |
1483 | } | |
1484 | obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, | |
1485 | btf_ext_data->d_size); | |
1486 | if (IS_ERR(obj->btf_ext)) { | |
be18010e KW |
1487 | pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n", |
1488 | BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); | |
9c6660d0 AN |
1489 | obj->btf_ext = NULL; |
1490 | goto out; | |
1491 | } | |
9c6660d0 AN |
1492 | } |
1493 | out: | |
1494 | if (err || IS_ERR(obj->btf)) { | |
abd29c93 AN |
1495 | if (btf_required) |
1496 | err = err ? : PTR_ERR(obj->btf); | |
1497 | else | |
1498 | err = 0; | |
9c6660d0 AN |
1499 | if (!IS_ERR_OR_NULL(obj->btf)) |
1500 | btf__free(obj->btf); | |
1501 | obj->btf = NULL; | |
1502 | } | |
abd29c93 | 1503 | if (btf_required && !obj->btf) { |
be18010e | 1504 | pr_warn("BTF is required, but is missing or corrupted.\n"); |
abd29c93 AN |
1505 | return err == 0 ? -ENOENT : err; |
1506 | } | |
9c6660d0 AN |
1507 | return 0; |
1508 | } | |
1509 | ||
063183bf AN |
1510 | static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) |
1511 | { | |
1512 | int err = 0; | |
1513 | ||
1514 | if (!obj->btf) | |
1515 | return 0; | |
1516 | ||
1517 | bpf_object__sanitize_btf(obj); | |
1518 | bpf_object__sanitize_btf_ext(obj); | |
1519 | ||
1520 | err = btf__load(obj->btf); | |
1521 | if (err) { | |
be18010e KW |
1522 | pr_warn("Error loading %s into kernel: %d.\n", |
1523 | BTF_ELF_SEC, err); | |
063183bf AN |
1524 | btf__free(obj->btf); |
1525 | obj->btf = NULL; | |
04efe591 AN |
1526 | /* btf_ext can't exist without btf, so free it as well */ |
1527 | if (obj->btf_ext) { | |
1528 | btf_ext__free(obj->btf_ext); | |
1529 | obj->btf_ext = NULL; | |
1530 | } | |
1531 | ||
abd29c93 AN |
1532 | if (bpf_object__is_btf_mandatory(obj)) |
1533 | return err; | |
063183bf AN |
1534 | } |
1535 | return 0; | |
1536 | } | |
1537 | ||
291ee02b | 1538 | static int bpf_object__elf_collect(struct bpf_object *obj, bool relaxed_maps) |
29603665 WN |
1539 | { |
1540 | Elf *elf = obj->efile.elf; | |
1541 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
f0187f0b | 1542 | Elf_Data *btf_ext_data = NULL; |
1713d68b | 1543 | Elf_Data *btf_data = NULL; |
29603665 | 1544 | Elf_Scn *scn = NULL; |
666810e8 | 1545 | int idx = 0, err = 0; |
29603665 WN |
1546 | |
1547 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
1548 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
be18010e | 1549 | pr_warn("failed to get e_shstrndx from %s\n", obj->path); |
6371ca3b | 1550 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
1551 | } |
1552 | ||
1553 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
1554 | char *name; | |
1555 | GElf_Shdr sh; | |
1556 | Elf_Data *data; | |
1557 | ||
1558 | idx++; | |
1559 | if (gelf_getshdr(scn, &sh) != &sh) { | |
be18010e KW |
1560 | pr_warn("failed to get section(%d) header from %s\n", |
1561 | idx, obj->path); | |
01b29d1d | 1562 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
1563 | } |
1564 | ||
1565 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
1566 | if (!name) { | |
be18010e KW |
1567 | pr_warn("failed to get section(%d) name from %s\n", |
1568 | idx, obj->path); | |
01b29d1d | 1569 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
1570 | } |
1571 | ||
1572 | data = elf_getdata(scn, 0); | |
1573 | if (!data) { | |
be18010e KW |
1574 | pr_warn("failed to get section(%d) data from %s(%s)\n", |
1575 | idx, name, obj->path); | |
01b29d1d | 1576 | return -LIBBPF_ERRNO__FORMAT; |
29603665 | 1577 | } |
077c066a JDB |
1578 | pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", |
1579 | idx, name, (unsigned long)data->d_size, | |
29603665 WN |
1580 | (int)sh.sh_link, (unsigned long)sh.sh_flags, |
1581 | (int)sh.sh_type); | |
cb1e5e96 | 1582 | |
1713d68b | 1583 | if (strcmp(name, "license") == 0) { |
cb1e5e96 WN |
1584 | err = bpf_object__init_license(obj, |
1585 | data->d_buf, | |
1586 | data->d_size); | |
01b29d1d AN |
1587 | if (err) |
1588 | return err; | |
1713d68b | 1589 | } else if (strcmp(name, "version") == 0) { |
54b8625c JF |
1590 | err = bpf_object__init_kversion(obj, |
1591 | data->d_buf, | |
1592 | data->d_size); | |
1593 | if (err) | |
1594 | return err; | |
1713d68b | 1595 | } else if (strcmp(name, "maps") == 0) { |
666810e8 | 1596 | obj->efile.maps_shndx = idx; |
abd29c93 AN |
1597 | } else if (strcmp(name, MAPS_ELF_SEC) == 0) { |
1598 | obj->efile.btf_maps_shndx = idx; | |
1713d68b DB |
1599 | } else if (strcmp(name, BTF_ELF_SEC) == 0) { |
1600 | btf_data = data; | |
2993e051 | 1601 | } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { |
f0187f0b | 1602 | btf_ext_data = data; |
8a138aed | 1603 | } else if (sh.sh_type == SHT_SYMTAB) { |
bec7d68c | 1604 | if (obj->efile.symbols) { |
be18010e KW |
1605 | pr_warn("bpf: multiple SYMTAB in %s\n", |
1606 | obj->path); | |
01b29d1d | 1607 | return -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 1608 | } |
01b29d1d AN |
1609 | obj->efile.symbols = data; |
1610 | obj->efile.strtabidx = sh.sh_link; | |
f8c7a4d4 JS |
1611 | } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { |
1612 | if (sh.sh_flags & SHF_EXECINSTR) { | |
1613 | if (strcmp(name, ".text") == 0) | |
1614 | obj->efile.text_shndx = idx; | |
1615 | err = bpf_object__add_program(obj, data->d_buf, | |
1616 | data->d_size, name, idx); | |
1617 | if (err) { | |
1618 | char errmsg[STRERR_BUFSIZE]; | |
1619 | char *cp = libbpf_strerror_r(-err, errmsg, | |
1620 | sizeof(errmsg)); | |
1621 | ||
be18010e KW |
1622 | pr_warn("failed to alloc program %s (%s): %s", |
1623 | name, obj->path, cp); | |
01b29d1d | 1624 | return err; |
f8c7a4d4 | 1625 | } |
d859900c DB |
1626 | } else if (strcmp(name, ".data") == 0) { |
1627 | obj->efile.data = data; | |
1628 | obj->efile.data_shndx = idx; | |
1629 | } else if (strcmp(name, ".rodata") == 0) { | |
1630 | obj->efile.rodata = data; | |
1631 | obj->efile.rodata_shndx = idx; | |
1632 | } else { | |
1633 | pr_debug("skip section(%d) %s\n", idx, name); | |
a5b8bd47 | 1634 | } |
b62f06e8 | 1635 | } else if (sh.sh_type == SHT_REL) { |
01b29d1d | 1636 | int nr_reloc = obj->efile.nr_reloc; |
b62f06e8 | 1637 | void *reloc = obj->efile.reloc; |
e3d91b0c JDB |
1638 | int sec = sh.sh_info; /* points to other section */ |
1639 | ||
1640 | /* Only do relo for section with exec instructions */ | |
1641 | if (!section_have_execinstr(obj, sec)) { | |
1642 | pr_debug("skip relo %s(%d) for section(%d)\n", | |
1643 | name, idx, sec); | |
1644 | continue; | |
1645 | } | |
b62f06e8 | 1646 | |
01b29d1d | 1647 | reloc = reallocarray(reloc, nr_reloc + 1, |
531b014e | 1648 | sizeof(*obj->efile.reloc)); |
b62f06e8 | 1649 | if (!reloc) { |
be18010e | 1650 | pr_warn("realloc failed\n"); |
01b29d1d AN |
1651 | return -ENOMEM; |
1652 | } | |
b62f06e8 | 1653 | |
01b29d1d AN |
1654 | obj->efile.reloc = reloc; |
1655 | obj->efile.nr_reloc++; | |
b62f06e8 | 1656 | |
01b29d1d AN |
1657 | obj->efile.reloc[nr_reloc].shdr = sh; |
1658 | obj->efile.reloc[nr_reloc].data = data; | |
d859900c DB |
1659 | } else if (sh.sh_type == SHT_NOBITS && strcmp(name, ".bss") == 0) { |
1660 | obj->efile.bss = data; | |
1661 | obj->efile.bss_shndx = idx; | |
077c066a JDB |
1662 | } else { |
1663 | pr_debug("skip section(%d) %s\n", idx, name); | |
bec7d68c | 1664 | } |
29603665 | 1665 | } |
561bbcca | 1666 | |
77ba9a5b | 1667 | if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { |
be18010e | 1668 | pr_warn("Corrupted ELF file: index of strtab invalid\n"); |
f102154d | 1669 | return -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 1670 | } |
063183bf | 1671 | err = bpf_object__init_btf(obj, btf_data, btf_ext_data); |
bf829271 | 1672 | if (!err) |
291ee02b | 1673 | err = bpf_object__init_maps(obj, relaxed_maps); |
063183bf AN |
1674 | if (!err) |
1675 | err = bpf_object__sanitize_and_load_btf(obj); | |
bf829271 AN |
1676 | if (!err) |
1677 | err = bpf_object__init_prog_names(obj); | |
29603665 WN |
1678 | return err; |
1679 | } | |
1680 | ||
34090915 WN |
1681 | static struct bpf_program * |
1682 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
1683 | { | |
1684 | struct bpf_program *prog; | |
1685 | size_t i; | |
1686 | ||
1687 | for (i = 0; i < obj->nr_programs; i++) { | |
1688 | prog = &obj->programs[i]; | |
1689 | if (prog->idx == idx) | |
1690 | return prog; | |
1691 | } | |
1692 | return NULL; | |
1693 | } | |
1694 | ||
6d4b198b | 1695 | struct bpf_program * |
a324aae3 AN |
1696 | bpf_object__find_program_by_title(const struct bpf_object *obj, |
1697 | const char *title) | |
6d4b198b JK |
1698 | { |
1699 | struct bpf_program *pos; | |
1700 | ||
1701 | bpf_object__for_each_program(pos, obj) { | |
1702 | if (pos->section_name && !strcmp(pos->section_name, title)) | |
1703 | return pos; | |
1704 | } | |
1705 | return NULL; | |
1706 | } | |
1707 | ||
d859900c DB |
1708 | static bool bpf_object__shndx_is_data(const struct bpf_object *obj, |
1709 | int shndx) | |
1710 | { | |
1711 | return shndx == obj->efile.data_shndx || | |
1712 | shndx == obj->efile.bss_shndx || | |
1713 | shndx == obj->efile.rodata_shndx; | |
1714 | } | |
1715 | ||
1716 | static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, | |
1717 | int shndx) | |
1718 | { | |
abd29c93 AN |
1719 | return shndx == obj->efile.maps_shndx || |
1720 | shndx == obj->efile.btf_maps_shndx; | |
d859900c DB |
1721 | } |
1722 | ||
1723 | static bool bpf_object__relo_in_known_section(const struct bpf_object *obj, | |
1724 | int shndx) | |
1725 | { | |
1726 | return shndx == obj->efile.text_shndx || | |
1727 | bpf_object__shndx_is_maps(obj, shndx) || | |
1728 | bpf_object__shndx_is_data(obj, shndx); | |
1729 | } | |
1730 | ||
1731 | static enum libbpf_map_type | |
1732 | bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) | |
1733 | { | |
1734 | if (shndx == obj->efile.data_shndx) | |
1735 | return LIBBPF_MAP_DATA; | |
1736 | else if (shndx == obj->efile.bss_shndx) | |
1737 | return LIBBPF_MAP_BSS; | |
1738 | else if (shndx == obj->efile.rodata_shndx) | |
1739 | return LIBBPF_MAP_RODATA; | |
1740 | else | |
1741 | return LIBBPF_MAP_UNSPEC; | |
1742 | } | |
1743 | ||
34090915 | 1744 | static int |
48cca7e4 AS |
1745 | bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, |
1746 | Elf_Data *data, struct bpf_object *obj) | |
34090915 | 1747 | { |
48cca7e4 | 1748 | Elf_Data *symbols = obj->efile.symbols; |
48cca7e4 AS |
1749 | struct bpf_map *maps = obj->maps; |
1750 | size_t nr_maps = obj->nr_maps; | |
34090915 WN |
1751 | int i, nrels; |
1752 | ||
399dc65e | 1753 | pr_debug("collecting relocating info for: '%s'\n", prog->section_name); |
34090915 WN |
1754 | nrels = shdr->sh_size / shdr->sh_entsize; |
1755 | ||
1756 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
1757 | if (!prog->reloc_desc) { | |
be18010e | 1758 | pr_warn("failed to alloc memory in relocation\n"); |
34090915 WN |
1759 | return -ENOMEM; |
1760 | } | |
1761 | prog->nr_reloc = nrels; | |
1762 | ||
1763 | for (i = 0; i < nrels; i++) { | |
34090915 | 1764 | struct bpf_insn *insns = prog->insns; |
d859900c | 1765 | enum libbpf_map_type type; |
abd29c93 AN |
1766 | unsigned int insn_idx; |
1767 | unsigned int shdr_idx; | |
d859900c | 1768 | const char *name; |
34090915 | 1769 | size_t map_idx; |
abd29c93 AN |
1770 | GElf_Sym sym; |
1771 | GElf_Rel rel; | |
34090915 WN |
1772 | |
1773 | if (!gelf_getrel(data, i, &rel)) { | |
be18010e | 1774 | pr_warn("relocation: failed to get %d reloc\n", i); |
6371ca3b | 1775 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
1776 | } |
1777 | ||
399dc65e | 1778 | if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { |
be18010e KW |
1779 | pr_warn("relocation: symbol %"PRIx64" not found\n", |
1780 | GELF_R_SYM(rel.r_info)); | |
6371ca3b | 1781 | return -LIBBPF_ERRNO__FORMAT; |
34090915 | 1782 | } |
d859900c DB |
1783 | |
1784 | name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
1785 | sym.st_name) ? : "<?>"; | |
1786 | ||
1787 | pr_debug("relo for %lld value %lld name %d (\'%s\')\n", | |
7d9890ef | 1788 | (long long) (rel.r_info >> 32), |
d859900c | 1789 | (long long) sym.st_value, sym.st_name, name); |
34090915 | 1790 | |
d859900c | 1791 | shdr_idx = sym.st_shndx; |
f2a3e4e9 AN |
1792 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); |
1793 | pr_debug("relocation: insn_idx=%u, shdr_idx=%u\n", | |
1794 | insn_idx, shdr_idx); | |
1795 | ||
1796 | if (shdr_idx >= SHN_LORESERVE) { | |
be18010e KW |
1797 | pr_warn("relocation: not yet supported relo for non-static global \'%s\' variable in special section (0x%x) found in insns[%d].code 0x%x\n", |
1798 | name, shdr_idx, insn_idx, | |
1799 | insns[insn_idx].code); | |
f2a3e4e9 AN |
1800 | return -LIBBPF_ERRNO__RELOC; |
1801 | } | |
d859900c | 1802 | if (!bpf_object__relo_in_known_section(obj, shdr_idx)) { |
be18010e KW |
1803 | pr_warn("Program '%s' contains unrecognized relo data pointing to section %u\n", |
1804 | prog->section_name, shdr_idx); | |
666810e8 WN |
1805 | return -LIBBPF_ERRNO__RELOC; |
1806 | } | |
1807 | ||
48cca7e4 AS |
1808 | if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { |
1809 | if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { | |
be18010e | 1810 | pr_warn("incorrect bpf_call opcode\n"); |
48cca7e4 AS |
1811 | return -LIBBPF_ERRNO__RELOC; |
1812 | } | |
1813 | prog->reloc_desc[i].type = RELO_CALL; | |
1814 | prog->reloc_desc[i].insn_idx = insn_idx; | |
1815 | prog->reloc_desc[i].text_off = sym.st_value; | |
9a94f277 | 1816 | obj->has_pseudo_calls = true; |
48cca7e4 AS |
1817 | continue; |
1818 | } | |
1819 | ||
34090915 | 1820 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { |
be18010e KW |
1821 | pr_warn("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", |
1822 | insn_idx, insns[insn_idx].code); | |
6371ca3b | 1823 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
1824 | } |
1825 | ||
d859900c DB |
1826 | if (bpf_object__shndx_is_maps(obj, shdr_idx) || |
1827 | bpf_object__shndx_is_data(obj, shdr_idx)) { | |
1828 | type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); | |
8837fe5d DB |
1829 | if (type != LIBBPF_MAP_UNSPEC) { |
1830 | if (GELF_ST_BIND(sym.st_info) == STB_GLOBAL) { | |
be18010e KW |
1831 | pr_warn("bpf: relocation: not yet supported relo for non-static global \'%s\' variable found in insns[%d].code 0x%x\n", |
1832 | name, insn_idx, insns[insn_idx].code); | |
8837fe5d DB |
1833 | return -LIBBPF_ERRNO__RELOC; |
1834 | } | |
1835 | if (!obj->caps.global_data) { | |
be18010e KW |
1836 | pr_warn("bpf: relocation: kernel does not support global \'%s\' variable access in insns[%d]\n", |
1837 | name, insn_idx); | |
8837fe5d DB |
1838 | return -LIBBPF_ERRNO__RELOC; |
1839 | } | |
d859900c DB |
1840 | } |
1841 | ||
f8c7a4d4 | 1842 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { |
d859900c DB |
1843 | if (maps[map_idx].libbpf_type != type) |
1844 | continue; | |
1845 | if (type != LIBBPF_MAP_UNSPEC || | |
db48814b AN |
1846 | (maps[map_idx].sec_idx == sym.st_shndx && |
1847 | maps[map_idx].sec_offset == sym.st_value)) { | |
1848 | pr_debug("relocation: found map %zd (%s, sec_idx %d, offset %zu) for insn %u\n", | |
1849 | map_idx, maps[map_idx].name, | |
1850 | maps[map_idx].sec_idx, | |
1851 | maps[map_idx].sec_offset, | |
1852 | insn_idx); | |
f8c7a4d4 JS |
1853 | break; |
1854 | } | |
94e5adec | 1855 | } |
94e5adec | 1856 | |
f8c7a4d4 | 1857 | if (map_idx >= nr_maps) { |
be18010e KW |
1858 | pr_warn("bpf relocation: map_idx %d larger than %d\n", |
1859 | (int)map_idx, (int)nr_maps - 1); | |
f8c7a4d4 JS |
1860 | return -LIBBPF_ERRNO__RELOC; |
1861 | } | |
34090915 | 1862 | |
d859900c DB |
1863 | prog->reloc_desc[i].type = type != LIBBPF_MAP_UNSPEC ? |
1864 | RELO_DATA : RELO_LD64; | |
f8c7a4d4 JS |
1865 | prog->reloc_desc[i].insn_idx = insn_idx; |
1866 | prog->reloc_desc[i].map_idx = map_idx; | |
1867 | } | |
34090915 WN |
1868 | } |
1869 | return 0; | |
1870 | } | |
1871 | ||
abd29c93 | 1872 | static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) |
8a138aed MKL |
1873 | { |
1874 | struct bpf_map_def *def = &map->def; | |
d859900c | 1875 | __u32 key_type_id = 0, value_type_id = 0; |
96408c43 | 1876 | int ret; |
8a138aed | 1877 | |
abd29c93 AN |
1878 | /* if it's BTF-defined map, we don't need to search for type IDs */ |
1879 | if (map->sec_idx == obj->efile.btf_maps_shndx) | |
1880 | return 0; | |
1881 | ||
d859900c | 1882 | if (!bpf_map__is_internal(map)) { |
abd29c93 | 1883 | ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, |
d859900c DB |
1884 | def->value_size, &key_type_id, |
1885 | &value_type_id); | |
1886 | } else { | |
1887 | /* | |
1888 | * LLVM annotates global data differently in BTF, that is, | |
1889 | * only as '.data', '.bss' or '.rodata'. | |
1890 | */ | |
abd29c93 | 1891 | ret = btf__find_by_name(obj->btf, |
d859900c DB |
1892 | libbpf_type_to_btf_name[map->libbpf_type]); |
1893 | } | |
1894 | if (ret < 0) | |
96408c43 | 1895 | return ret; |
8a138aed | 1896 | |
96408c43 | 1897 | map->btf_key_type_id = key_type_id; |
d859900c DB |
1898 | map->btf_value_type_id = bpf_map__is_internal(map) ? |
1899 | ret : value_type_id; | |
8a138aed MKL |
1900 | return 0; |
1901 | } | |
1902 | ||
26736eb9 JK |
1903 | int bpf_map__reuse_fd(struct bpf_map *map, int fd) |
1904 | { | |
1905 | struct bpf_map_info info = {}; | |
1906 | __u32 len = sizeof(info); | |
1907 | int new_fd, err; | |
1908 | char *new_name; | |
1909 | ||
1910 | err = bpf_obj_get_info_by_fd(fd, &info, &len); | |
1911 | if (err) | |
1912 | return err; | |
1913 | ||
1914 | new_name = strdup(info.name); | |
1915 | if (!new_name) | |
1916 | return -errno; | |
1917 | ||
1918 | new_fd = open("/", O_RDONLY | O_CLOEXEC); | |
1919 | if (new_fd < 0) | |
1920 | goto err_free_new_name; | |
1921 | ||
1922 | new_fd = dup3(fd, new_fd, O_CLOEXEC); | |
1923 | if (new_fd < 0) | |
1924 | goto err_close_new_fd; | |
1925 | ||
1926 | err = zclose(map->fd); | |
1927 | if (err) | |
1928 | goto err_close_new_fd; | |
1929 | free(map->name); | |
1930 | ||
1931 | map->fd = new_fd; | |
1932 | map->name = new_name; | |
1933 | map->def.type = info.type; | |
1934 | map->def.key_size = info.key_size; | |
1935 | map->def.value_size = info.value_size; | |
1936 | map->def.max_entries = info.max_entries; | |
1937 | map->def.map_flags = info.map_flags; | |
1938 | map->btf_key_type_id = info.btf_key_type_id; | |
1939 | map->btf_value_type_id = info.btf_value_type_id; | |
1940 | ||
1941 | return 0; | |
1942 | ||
1943 | err_close_new_fd: | |
1944 | close(new_fd); | |
1945 | err_free_new_name: | |
1946 | free(new_name); | |
1947 | return -errno; | |
1948 | } | |
1949 | ||
1a11a4c7 AI |
1950 | int bpf_map__resize(struct bpf_map *map, __u32 max_entries) |
1951 | { | |
1952 | if (!map || !max_entries) | |
1953 | return -EINVAL; | |
1954 | ||
1955 | /* If map already created, its attributes can't be changed. */ | |
1956 | if (map->fd >= 0) | |
1957 | return -EBUSY; | |
1958 | ||
1959 | map->def.max_entries = max_entries; | |
1960 | ||
1961 | return 0; | |
1962 | } | |
1963 | ||
47eff617 SF |
1964 | static int |
1965 | bpf_object__probe_name(struct bpf_object *obj) | |
1966 | { | |
1967 | struct bpf_load_program_attr attr; | |
1968 | char *cp, errmsg[STRERR_BUFSIZE]; | |
1969 | struct bpf_insn insns[] = { | |
1970 | BPF_MOV64_IMM(BPF_REG_0, 0), | |
1971 | BPF_EXIT_INSN(), | |
1972 | }; | |
1973 | int ret; | |
1974 | ||
1975 | /* make sure basic loading works */ | |
1976 | ||
1977 | memset(&attr, 0, sizeof(attr)); | |
1978 | attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; | |
1979 | attr.insns = insns; | |
1980 | attr.insns_cnt = ARRAY_SIZE(insns); | |
1981 | attr.license = "GPL"; | |
1982 | ||
1983 | ret = bpf_load_program_xattr(&attr, NULL, 0); | |
1984 | if (ret < 0) { | |
1985 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
be18010e KW |
1986 | pr_warn("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", |
1987 | __func__, cp, errno); | |
47eff617 SF |
1988 | return -errno; |
1989 | } | |
1990 | close(ret); | |
1991 | ||
1992 | /* now try the same program, but with the name */ | |
1993 | ||
1994 | attr.name = "test"; | |
1995 | ret = bpf_load_program_xattr(&attr, NULL, 0); | |
1996 | if (ret >= 0) { | |
1997 | obj->caps.name = 1; | |
1998 | close(ret); | |
1999 | } | |
2000 | ||
2001 | return 0; | |
2002 | } | |
2003 | ||
8837fe5d DB |
2004 | static int |
2005 | bpf_object__probe_global_data(struct bpf_object *obj) | |
2006 | { | |
2007 | struct bpf_load_program_attr prg_attr; | |
2008 | struct bpf_create_map_attr map_attr; | |
2009 | char *cp, errmsg[STRERR_BUFSIZE]; | |
2010 | struct bpf_insn insns[] = { | |
2011 | BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), | |
2012 | BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), | |
2013 | BPF_MOV64_IMM(BPF_REG_0, 0), | |
2014 | BPF_EXIT_INSN(), | |
2015 | }; | |
2016 | int ret, map; | |
2017 | ||
2018 | memset(&map_attr, 0, sizeof(map_attr)); | |
2019 | map_attr.map_type = BPF_MAP_TYPE_ARRAY; | |
2020 | map_attr.key_size = sizeof(int); | |
2021 | map_attr.value_size = 32; | |
2022 | map_attr.max_entries = 1; | |
2023 | ||
2024 | map = bpf_create_map_xattr(&map_attr); | |
2025 | if (map < 0) { | |
2026 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
be18010e KW |
2027 | pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", |
2028 | __func__, cp, errno); | |
8837fe5d DB |
2029 | return -errno; |
2030 | } | |
2031 | ||
2032 | insns[0].imm = map; | |
2033 | ||
2034 | memset(&prg_attr, 0, sizeof(prg_attr)); | |
2035 | prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; | |
2036 | prg_attr.insns = insns; | |
2037 | prg_attr.insns_cnt = ARRAY_SIZE(insns); | |
2038 | prg_attr.license = "GPL"; | |
2039 | ||
2040 | ret = bpf_load_program_xattr(&prg_attr, NULL, 0); | |
2041 | if (ret >= 0) { | |
2042 | obj->caps.global_data = 1; | |
2043 | close(ret); | |
2044 | } | |
2045 | ||
2046 | close(map); | |
2047 | return 0; | |
2048 | } | |
2049 | ||
d7c4b398 AN |
2050 | static int bpf_object__probe_btf_func(struct bpf_object *obj) |
2051 | { | |
2052 | const char strs[] = "\0int\0x\0a"; | |
2053 | /* void x(int a) {} */ | |
2054 | __u32 types[] = { | |
2055 | /* int */ | |
2056 | BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ | |
2057 | /* FUNC_PROTO */ /* [2] */ | |
2058 | BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), | |
2059 | BTF_PARAM_ENC(7, 1), | |
2060 | /* FUNC x */ /* [3] */ | |
2061 | BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), | |
2062 | }; | |
cfd49210 | 2063 | int btf_fd; |
d7c4b398 | 2064 | |
cfd49210 MR |
2065 | btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), |
2066 | strs, sizeof(strs)); | |
2067 | if (btf_fd >= 0) { | |
d7c4b398 | 2068 | obj->caps.btf_func = 1; |
cfd49210 MR |
2069 | close(btf_fd); |
2070 | return 1; | |
2071 | } | |
2072 | ||
d7c4b398 AN |
2073 | return 0; |
2074 | } | |
2075 | ||
2076 | static int bpf_object__probe_btf_datasec(struct bpf_object *obj) | |
2077 | { | |
2078 | const char strs[] = "\0x\0.data"; | |
2079 | /* static int a; */ | |
2080 | __u32 types[] = { | |
2081 | /* int */ | |
2082 | BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ | |
2083 | /* VAR x */ /* [2] */ | |
2084 | BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), | |
2085 | BTF_VAR_STATIC, | |
2086 | /* DATASEC val */ /* [3] */ | |
2087 | BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), | |
2088 | BTF_VAR_SECINFO_ENC(2, 0, 4), | |
2089 | }; | |
cfd49210 | 2090 | int btf_fd; |
d7c4b398 | 2091 | |
cfd49210 MR |
2092 | btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), |
2093 | strs, sizeof(strs)); | |
2094 | if (btf_fd >= 0) { | |
d7c4b398 | 2095 | obj->caps.btf_datasec = 1; |
cfd49210 MR |
2096 | close(btf_fd); |
2097 | return 1; | |
2098 | } | |
2099 | ||
d7c4b398 AN |
2100 | return 0; |
2101 | } | |
2102 | ||
47eff617 SF |
2103 | static int |
2104 | bpf_object__probe_caps(struct bpf_object *obj) | |
2105 | { | |
8837fe5d DB |
2106 | int (*probe_fn[])(struct bpf_object *obj) = { |
2107 | bpf_object__probe_name, | |
2108 | bpf_object__probe_global_data, | |
d7c4b398 AN |
2109 | bpf_object__probe_btf_func, |
2110 | bpf_object__probe_btf_datasec, | |
8837fe5d DB |
2111 | }; |
2112 | int i, ret; | |
2113 | ||
2114 | for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { | |
2115 | ret = probe_fn[i](obj); | |
2116 | if (ret < 0) | |
15ea164e | 2117 | pr_debug("Probe #%d failed with %d.\n", i, ret); |
8837fe5d DB |
2118 | } |
2119 | ||
2120 | return 0; | |
47eff617 SF |
2121 | } |
2122 | ||
d859900c DB |
2123 | static int |
2124 | bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) | |
2125 | { | |
2126 | char *cp, errmsg[STRERR_BUFSIZE]; | |
2127 | int err, zero = 0; | |
2128 | __u8 *data; | |
2129 | ||
2130 | /* Nothing to do here since kernel already zero-initializes .bss map. */ | |
2131 | if (map->libbpf_type == LIBBPF_MAP_BSS) | |
2132 | return 0; | |
2133 | ||
2134 | data = map->libbpf_type == LIBBPF_MAP_DATA ? | |
2135 | obj->sections.data : obj->sections.rodata; | |
2136 | ||
2137 | err = bpf_map_update_elem(map->fd, &zero, data, 0); | |
2138 | /* Freeze .rodata map as read-only from syscall side. */ | |
2139 | if (!err && map->libbpf_type == LIBBPF_MAP_RODATA) { | |
2140 | err = bpf_map_freeze(map->fd); | |
2141 | if (err) { | |
2142 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
be18010e KW |
2143 | pr_warn("Error freezing map(%s) as read-only: %s\n", |
2144 | map->name, cp); | |
d859900c DB |
2145 | err = 0; |
2146 | } | |
2147 | } | |
2148 | return err; | |
2149 | } | |
2150 | ||
52d3352e WN |
2151 | static int |
2152 | bpf_object__create_maps(struct bpf_object *obj) | |
2153 | { | |
8a138aed | 2154 | struct bpf_create_map_attr create_attr = {}; |
d7ff34d5 | 2155 | int nr_cpus = 0; |
52d3352e | 2156 | unsigned int i; |
8a138aed | 2157 | int err; |
52d3352e | 2158 | |
9d759a9b | 2159 | for (i = 0; i < obj->nr_maps; i++) { |
8a138aed MKL |
2160 | struct bpf_map *map = &obj->maps[i]; |
2161 | struct bpf_map_def *def = &map->def; | |
1ce6a9fc | 2162 | char *cp, errmsg[STRERR_BUFSIZE]; |
8a138aed MKL |
2163 | int *pfd = &map->fd; |
2164 | ||
26736eb9 JK |
2165 | if (map->fd >= 0) { |
2166 | pr_debug("skip map create (preset) %s: fd=%d\n", | |
2167 | map->name, map->fd); | |
2168 | continue; | |
2169 | } | |
2170 | ||
94cb310c SF |
2171 | if (obj->caps.name) |
2172 | create_attr.name = map->name; | |
f0307a7e | 2173 | create_attr.map_ifindex = map->map_ifindex; |
8a138aed MKL |
2174 | create_attr.map_type = def->type; |
2175 | create_attr.map_flags = def->map_flags; | |
2176 | create_attr.key_size = def->key_size; | |
2177 | create_attr.value_size = def->value_size; | |
d7ff34d5 AN |
2178 | if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && |
2179 | !def->max_entries) { | |
2180 | if (!nr_cpus) | |
2181 | nr_cpus = libbpf_num_possible_cpus(); | |
2182 | if (nr_cpus < 0) { | |
be18010e KW |
2183 | pr_warn("failed to determine number of system CPUs: %d\n", |
2184 | nr_cpus); | |
d7ff34d5 AN |
2185 | err = nr_cpus; |
2186 | goto err_out; | |
2187 | } | |
2188 | pr_debug("map '%s': setting size to %d\n", | |
2189 | map->name, nr_cpus); | |
2190 | create_attr.max_entries = nr_cpus; | |
2191 | } else { | |
2192 | create_attr.max_entries = def->max_entries; | |
2193 | } | |
e55d54f4 | 2194 | create_attr.btf_fd = 0; |
61746dbe MKL |
2195 | create_attr.btf_key_type_id = 0; |
2196 | create_attr.btf_value_type_id = 0; | |
addb9fc9 NS |
2197 | if (bpf_map_type__is_map_in_map(def->type) && |
2198 | map->inner_map_fd >= 0) | |
2199 | create_attr.inner_map_fd = map->inner_map_fd; | |
8a138aed | 2200 | |
abd29c93 | 2201 | if (obj->btf && !bpf_map_find_btf_info(obj, map)) { |
8a138aed | 2202 | create_attr.btf_fd = btf__fd(obj->btf); |
61746dbe MKL |
2203 | create_attr.btf_key_type_id = map->btf_key_type_id; |
2204 | create_attr.btf_value_type_id = map->btf_value_type_id; | |
8a138aed MKL |
2205 | } |
2206 | ||
2207 | *pfd = bpf_create_map_xattr(&create_attr); | |
e55d54f4 AN |
2208 | if (*pfd < 0 && (create_attr.btf_key_type_id || |
2209 | create_attr.btf_value_type_id)) { | |
d7ff34d5 AN |
2210 | err = -errno; |
2211 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
be18010e KW |
2212 | pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", |
2213 | map->name, cp, err); | |
e55d54f4 | 2214 | create_attr.btf_fd = 0; |
61746dbe MKL |
2215 | create_attr.btf_key_type_id = 0; |
2216 | create_attr.btf_value_type_id = 0; | |
2217 | map->btf_key_type_id = 0; | |
2218 | map->btf_value_type_id = 0; | |
8a138aed MKL |
2219 | *pfd = bpf_create_map_xattr(&create_attr); |
2220 | } | |
2221 | ||
52d3352e WN |
2222 | if (*pfd < 0) { |
2223 | size_t j; | |
52d3352e | 2224 | |
d7ff34d5 | 2225 | err = -errno; |
d859900c | 2226 | err_out: |
d7ff34d5 | 2227 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); |
be18010e KW |
2228 | pr_warn("failed to create map (name: '%s'): %s(%d)\n", |
2229 | map->name, cp, err); | |
52d3352e | 2230 | for (j = 0; j < i; j++) |
9d759a9b | 2231 | zclose(obj->maps[j].fd); |
52d3352e WN |
2232 | return err; |
2233 | } | |
d859900c DB |
2234 | |
2235 | if (bpf_map__is_internal(map)) { | |
2236 | err = bpf_object__populate_internal_map(obj, map); | |
2237 | if (err < 0) { | |
2238 | zclose(*pfd); | |
2239 | goto err_out; | |
2240 | } | |
2241 | } | |
2242 | ||
76e1022b | 2243 | pr_debug("created map %s: fd=%d\n", map->name, *pfd); |
52d3352e WN |
2244 | } |
2245 | ||
52d3352e WN |
2246 | return 0; |
2247 | } | |
2248 | ||
f0187f0b MKL |
2249 | static int |
2250 | check_btf_ext_reloc_err(struct bpf_program *prog, int err, | |
2251 | void *btf_prog_info, const char *info_name) | |
2252 | { | |
2253 | if (err != -ENOENT) { | |
be18010e KW |
2254 | pr_warn("Error in loading %s for sec %s.\n", |
2255 | info_name, prog->section_name); | |
f0187f0b MKL |
2256 | return err; |
2257 | } | |
2258 | ||
2259 | /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ | |
2260 | ||
2261 | if (btf_prog_info) { | |
2262 | /* | |
2263 | * Some info has already been found but has problem | |
399dc65e | 2264 | * in the last btf_ext reloc. Must have to error out. |
f0187f0b | 2265 | */ |
be18010e KW |
2266 | pr_warn("Error in relocating %s for sec %s.\n", |
2267 | info_name, prog->section_name); | |
f0187f0b MKL |
2268 | return err; |
2269 | } | |
2270 | ||
399dc65e | 2271 | /* Have problem loading the very first info. Ignore the rest. */ |
be18010e KW |
2272 | pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n", |
2273 | info_name, prog->section_name, info_name); | |
f0187f0b MKL |
2274 | return 0; |
2275 | } | |
2276 | ||
2277 | static int | |
2278 | bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, | |
2279 | const char *section_name, __u32 insn_offset) | |
2280 | { | |
2281 | int err; | |
2282 | ||
2283 | if (!insn_offset || prog->func_info) { | |
2284 | /* | |
2285 | * !insn_offset => main program | |
2286 | * | |
2287 | * For sub prog, the main program's func_info has to | |
2288 | * be loaded first (i.e. prog->func_info != NULL) | |
2289 | */ | |
2290 | err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, | |
2291 | section_name, insn_offset, | |
2292 | &prog->func_info, | |
2293 | &prog->func_info_cnt); | |
2294 | if (err) | |
2295 | return check_btf_ext_reloc_err(prog, err, | |
2296 | prog->func_info, | |
2297 | "bpf_func_info"); | |
2298 | ||
2299 | prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); | |
2300 | } | |
2301 | ||
3d650141 MKL |
2302 | if (!insn_offset || prog->line_info) { |
2303 | err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, | |
2304 | section_name, insn_offset, | |
2305 | &prog->line_info, | |
2306 | &prog->line_info_cnt); | |
2307 | if (err) | |
2308 | return check_btf_ext_reloc_err(prog, err, | |
2309 | prog->line_info, | |
2310 | "bpf_line_info"); | |
2311 | ||
2312 | prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); | |
2313 | } | |
2314 | ||
f0187f0b MKL |
2315 | return 0; |
2316 | } | |
2317 | ||
ddc7c304 AN |
2318 | #define BPF_CORE_SPEC_MAX_LEN 64 |
2319 | ||
2320 | /* represents BPF CO-RE field or array element accessor */ | |
2321 | struct bpf_core_accessor { | |
2322 | __u32 type_id; /* struct/union type or array element type */ | |
2323 | __u32 idx; /* field index or array index */ | |
2324 | const char *name; /* field name or NULL for array accessor */ | |
2325 | }; | |
2326 | ||
2327 | struct bpf_core_spec { | |
2328 | const struct btf *btf; | |
2329 | /* high-level spec: named fields and array indices only */ | |
2330 | struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN]; | |
2331 | /* high-level spec length */ | |
2332 | int len; | |
2333 | /* raw, low-level spec: 1-to-1 with accessor spec string */ | |
2334 | int raw_spec[BPF_CORE_SPEC_MAX_LEN]; | |
2335 | /* raw spec length */ | |
2336 | int raw_len; | |
2337 | /* field byte offset represented by spec */ | |
2338 | __u32 offset; | |
2339 | }; | |
2340 | ||
2341 | static bool str_is_empty(const char *s) | |
2342 | { | |
2343 | return !s || !s[0]; | |
2344 | } | |
2345 | ||
2346 | /* | |
511bb008 | 2347 | * Turn bpf_field_reloc into a low- and high-level spec representation, |
ddc7c304 AN |
2348 | * validating correctness along the way, as well as calculating resulting |
2349 | * field offset (in bytes), specified by accessor string. Low-level spec | |
2350 | * captures every single level of nestedness, including traversing anonymous | |
2351 | * struct/union members. High-level one only captures semantically meaningful | |
2352 | * "turning points": named fields and array indicies. | |
2353 | * E.g., for this case: | |
2354 | * | |
2355 | * struct sample { | |
2356 | * int __unimportant; | |
2357 | * struct { | |
2358 | * int __1; | |
2359 | * int __2; | |
2360 | * int a[7]; | |
2361 | * }; | |
2362 | * }; | |
2363 | * | |
2364 | * struct sample *s = ...; | |
2365 | * | |
2366 | * int x = &s->a[3]; // access string = '0:1:2:3' | |
2367 | * | |
2368 | * Low-level spec has 1:1 mapping with each element of access string (it's | |
2369 | * just a parsed access string representation): [0, 1, 2, 3]. | |
2370 | * | |
2371 | * High-level spec will capture only 3 points: | |
2372 | * - intial zero-index access by pointer (&s->... is the same as &s[0]...); | |
2373 | * - field 'a' access (corresponds to '2' in low-level spec); | |
2374 | * - array element #3 access (corresponds to '3' in low-level spec). | |
2375 | * | |
2376 | */ | |
2377 | static int bpf_core_spec_parse(const struct btf *btf, | |
2378 | __u32 type_id, | |
2379 | const char *spec_str, | |
2380 | struct bpf_core_spec *spec) | |
2381 | { | |
2382 | int access_idx, parsed_len, i; | |
2383 | const struct btf_type *t; | |
2384 | const char *name; | |
2385 | __u32 id; | |
2386 | __s64 sz; | |
2387 | ||
2388 | if (str_is_empty(spec_str) || *spec_str == ':') | |
2389 | return -EINVAL; | |
2390 | ||
2391 | memset(spec, 0, sizeof(*spec)); | |
2392 | spec->btf = btf; | |
2393 | ||
2394 | /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */ | |
2395 | while (*spec_str) { | |
2396 | if (*spec_str == ':') | |
2397 | ++spec_str; | |
2398 | if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) | |
2399 | return -EINVAL; | |
2400 | if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) | |
2401 | return -E2BIG; | |
2402 | spec_str += parsed_len; | |
2403 | spec->raw_spec[spec->raw_len++] = access_idx; | |
2404 | } | |
2405 | ||
2406 | if (spec->raw_len == 0) | |
2407 | return -EINVAL; | |
2408 | ||
2409 | /* first spec value is always reloc type array index */ | |
2410 | t = skip_mods_and_typedefs(btf, type_id, &id); | |
2411 | if (!t) | |
2412 | return -EINVAL; | |
2413 | ||
2414 | access_idx = spec->raw_spec[0]; | |
2415 | spec->spec[0].type_id = id; | |
2416 | spec->spec[0].idx = access_idx; | |
2417 | spec->len++; | |
2418 | ||
2419 | sz = btf__resolve_size(btf, id); | |
2420 | if (sz < 0) | |
2421 | return sz; | |
2422 | spec->offset = access_idx * sz; | |
2423 | ||
2424 | for (i = 1; i < spec->raw_len; i++) { | |
2425 | t = skip_mods_and_typedefs(btf, id, &id); | |
2426 | if (!t) | |
2427 | return -EINVAL; | |
2428 | ||
2429 | access_idx = spec->raw_spec[i]; | |
2430 | ||
2431 | if (btf_is_composite(t)) { | |
2432 | const struct btf_member *m; | |
2433 | __u32 offset; | |
2434 | ||
2435 | if (access_idx >= btf_vlen(t)) | |
2436 | return -EINVAL; | |
2437 | if (btf_member_bitfield_size(t, access_idx)) | |
2438 | return -EINVAL; | |
2439 | ||
2440 | offset = btf_member_bit_offset(t, access_idx); | |
2441 | if (offset % 8) | |
2442 | return -EINVAL; | |
2443 | spec->offset += offset / 8; | |
2444 | ||
2445 | m = btf_members(t) + access_idx; | |
2446 | if (m->name_off) { | |
2447 | name = btf__name_by_offset(btf, m->name_off); | |
2448 | if (str_is_empty(name)) | |
2449 | return -EINVAL; | |
2450 | ||
2451 | spec->spec[spec->len].type_id = id; | |
2452 | spec->spec[spec->len].idx = access_idx; | |
2453 | spec->spec[spec->len].name = name; | |
2454 | spec->len++; | |
2455 | } | |
2456 | ||
2457 | id = m->type; | |
2458 | } else if (btf_is_array(t)) { | |
2459 | const struct btf_array *a = btf_array(t); | |
2460 | ||
2461 | t = skip_mods_and_typedefs(btf, a->type, &id); | |
2462 | if (!t || access_idx >= a->nelems) | |
2463 | return -EINVAL; | |
2464 | ||
2465 | spec->spec[spec->len].type_id = id; | |
2466 | spec->spec[spec->len].idx = access_idx; | |
2467 | spec->len++; | |
2468 | ||
2469 | sz = btf__resolve_size(btf, id); | |
2470 | if (sz < 0) | |
2471 | return sz; | |
2472 | spec->offset += access_idx * sz; | |
2473 | } else { | |
be18010e KW |
2474 | pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n", |
2475 | type_id, spec_str, i, id, btf_kind(t)); | |
ddc7c304 AN |
2476 | return -EINVAL; |
2477 | } | |
2478 | } | |
2479 | ||
2480 | return 0; | |
2481 | } | |
2482 | ||
2483 | static bool bpf_core_is_flavor_sep(const char *s) | |
2484 | { | |
2485 | /* check X___Y name pattern, where X and Y are not underscores */ | |
2486 | return s[0] != '_' && /* X */ | |
2487 | s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ | |
2488 | s[4] != '_'; /* Y */ | |
2489 | } | |
2490 | ||
2491 | /* Given 'some_struct_name___with_flavor' return the length of a name prefix | |
2492 | * before last triple underscore. Struct name part after last triple | |
2493 | * underscore is ignored by BPF CO-RE relocation during relocation matching. | |
2494 | */ | |
2495 | static size_t bpf_core_essential_name_len(const char *name) | |
2496 | { | |
2497 | size_t n = strlen(name); | |
2498 | int i; | |
2499 | ||
2500 | for (i = n - 5; i >= 0; i--) { | |
2501 | if (bpf_core_is_flavor_sep(name + i)) | |
2502 | return i + 1; | |
2503 | } | |
2504 | return n; | |
2505 | } | |
2506 | ||
2507 | /* dynamically sized list of type IDs */ | |
2508 | struct ids_vec { | |
2509 | __u32 *data; | |
2510 | int len; | |
2511 | }; | |
2512 | ||
2513 | static void bpf_core_free_cands(struct ids_vec *cand_ids) | |
2514 | { | |
2515 | free(cand_ids->data); | |
2516 | free(cand_ids); | |
2517 | } | |
2518 | ||
2519 | static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf, | |
2520 | __u32 local_type_id, | |
2521 | const struct btf *targ_btf) | |
2522 | { | |
2523 | size_t local_essent_len, targ_essent_len; | |
2524 | const char *local_name, *targ_name; | |
2525 | const struct btf_type *t; | |
2526 | struct ids_vec *cand_ids; | |
2527 | __u32 *new_ids; | |
2528 | int i, err, n; | |
2529 | ||
2530 | t = btf__type_by_id(local_btf, local_type_id); | |
2531 | if (!t) | |
2532 | return ERR_PTR(-EINVAL); | |
2533 | ||
2534 | local_name = btf__name_by_offset(local_btf, t->name_off); | |
2535 | if (str_is_empty(local_name)) | |
2536 | return ERR_PTR(-EINVAL); | |
2537 | local_essent_len = bpf_core_essential_name_len(local_name); | |
2538 | ||
2539 | cand_ids = calloc(1, sizeof(*cand_ids)); | |
2540 | if (!cand_ids) | |
2541 | return ERR_PTR(-ENOMEM); | |
2542 | ||
2543 | n = btf__get_nr_types(targ_btf); | |
2544 | for (i = 1; i <= n; i++) { | |
2545 | t = btf__type_by_id(targ_btf, i); | |
2546 | targ_name = btf__name_by_offset(targ_btf, t->name_off); | |
2547 | if (str_is_empty(targ_name)) | |
2548 | continue; | |
2549 | ||
2550 | targ_essent_len = bpf_core_essential_name_len(targ_name); | |
2551 | if (targ_essent_len != local_essent_len) | |
2552 | continue; | |
2553 | ||
2554 | if (strncmp(local_name, targ_name, local_essent_len) == 0) { | |
2555 | pr_debug("[%d] %s: found candidate [%d] %s\n", | |
2556 | local_type_id, local_name, i, targ_name); | |
2557 | new_ids = realloc(cand_ids->data, cand_ids->len + 1); | |
2558 | if (!new_ids) { | |
2559 | err = -ENOMEM; | |
2560 | goto err_out; | |
2561 | } | |
2562 | cand_ids->data = new_ids; | |
2563 | cand_ids->data[cand_ids->len++] = i; | |
2564 | } | |
2565 | } | |
2566 | return cand_ids; | |
2567 | err_out: | |
2568 | bpf_core_free_cands(cand_ids); | |
2569 | return ERR_PTR(err); | |
2570 | } | |
2571 | ||
2572 | /* Check two types for compatibility, skipping const/volatile/restrict and | |
2573 | * typedefs, to ensure we are relocating offset to the compatible entities: | |
2574 | * - any two STRUCTs/UNIONs are compatible and can be mixed; | |
2575 | * - any two FWDs are compatible; | |
2576 | * - any two PTRs are always compatible; | |
2577 | * - for ENUMs, check sizes, names are ignored; | |
2578 | * - for INT, size and bitness should match, signedness is ignored; | |
2579 | * - for ARRAY, dimensionality is ignored, element types are checked for | |
2580 | * compatibility recursively; | |
2581 | * - everything else shouldn't be ever a target of relocation. | |
2582 | * These rules are not set in stone and probably will be adjusted as we get | |
2583 | * more experience with using BPF CO-RE relocations. | |
2584 | */ | |
2585 | static int bpf_core_fields_are_compat(const struct btf *local_btf, | |
2586 | __u32 local_id, | |
2587 | const struct btf *targ_btf, | |
2588 | __u32 targ_id) | |
2589 | { | |
2590 | const struct btf_type *local_type, *targ_type; | |
2591 | ||
2592 | recur: | |
2593 | local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); | |
2594 | targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); | |
2595 | if (!local_type || !targ_type) | |
2596 | return -EINVAL; | |
2597 | ||
2598 | if (btf_is_composite(local_type) && btf_is_composite(targ_type)) | |
2599 | return 1; | |
2600 | if (btf_kind(local_type) != btf_kind(targ_type)) | |
2601 | return 0; | |
2602 | ||
2603 | switch (btf_kind(local_type)) { | |
2604 | case BTF_KIND_FWD: | |
2605 | case BTF_KIND_PTR: | |
2606 | return 1; | |
2607 | case BTF_KIND_ENUM: | |
2608 | return local_type->size == targ_type->size; | |
2609 | case BTF_KIND_INT: | |
2610 | return btf_int_offset(local_type) == 0 && | |
2611 | btf_int_offset(targ_type) == 0 && | |
2612 | local_type->size == targ_type->size && | |
2613 | btf_int_bits(local_type) == btf_int_bits(targ_type); | |
2614 | case BTF_KIND_ARRAY: | |
2615 | local_id = btf_array(local_type)->type; | |
2616 | targ_id = btf_array(targ_type)->type; | |
2617 | goto recur; | |
2618 | default: | |
be18010e KW |
2619 | pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n", |
2620 | btf_kind(local_type), local_id, targ_id); | |
ddc7c304 AN |
2621 | return 0; |
2622 | } | |
2623 | } | |
2624 | ||
2625 | /* | |
2626 | * Given single high-level named field accessor in local type, find | |
2627 | * corresponding high-level accessor for a target type. Along the way, | |
2628 | * maintain low-level spec for target as well. Also keep updating target | |
2629 | * offset. | |
2630 | * | |
2631 | * Searching is performed through recursive exhaustive enumeration of all | |
2632 | * fields of a struct/union. If there are any anonymous (embedded) | |
2633 | * structs/unions, they are recursively searched as well. If field with | |
2634 | * desired name is found, check compatibility between local and target types, | |
2635 | * before returning result. | |
2636 | * | |
2637 | * 1 is returned, if field is found. | |
2638 | * 0 is returned if no compatible field is found. | |
2639 | * <0 is returned on error. | |
2640 | */ | |
2641 | static int bpf_core_match_member(const struct btf *local_btf, | |
2642 | const struct bpf_core_accessor *local_acc, | |
2643 | const struct btf *targ_btf, | |
2644 | __u32 targ_id, | |
2645 | struct bpf_core_spec *spec, | |
2646 | __u32 *next_targ_id) | |
2647 | { | |
2648 | const struct btf_type *local_type, *targ_type; | |
2649 | const struct btf_member *local_member, *m; | |
2650 | const char *local_name, *targ_name; | |
2651 | __u32 local_id; | |
2652 | int i, n, found; | |
2653 | ||
2654 | targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); | |
2655 | if (!targ_type) | |
2656 | return -EINVAL; | |
2657 | if (!btf_is_composite(targ_type)) | |
2658 | return 0; | |
2659 | ||
2660 | local_id = local_acc->type_id; | |
2661 | local_type = btf__type_by_id(local_btf, local_id); | |
2662 | local_member = btf_members(local_type) + local_acc->idx; | |
2663 | local_name = btf__name_by_offset(local_btf, local_member->name_off); | |
2664 | ||
2665 | n = btf_vlen(targ_type); | |
2666 | m = btf_members(targ_type); | |
2667 | for (i = 0; i < n; i++, m++) { | |
2668 | __u32 offset; | |
2669 | ||
2670 | /* bitfield relocations not supported */ | |
2671 | if (btf_member_bitfield_size(targ_type, i)) | |
2672 | continue; | |
2673 | offset = btf_member_bit_offset(targ_type, i); | |
2674 | if (offset % 8) | |
2675 | continue; | |
2676 | ||
2677 | /* too deep struct/union/array nesting */ | |
2678 | if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) | |
2679 | return -E2BIG; | |
2680 | ||
2681 | /* speculate this member will be the good one */ | |
2682 | spec->offset += offset / 8; | |
2683 | spec->raw_spec[spec->raw_len++] = i; | |
2684 | ||
2685 | targ_name = btf__name_by_offset(targ_btf, m->name_off); | |
2686 | if (str_is_empty(targ_name)) { | |
2687 | /* embedded struct/union, we need to go deeper */ | |
2688 | found = bpf_core_match_member(local_btf, local_acc, | |
2689 | targ_btf, m->type, | |
2690 | spec, next_targ_id); | |
2691 | if (found) /* either found or error */ | |
2692 | return found; | |
2693 | } else if (strcmp(local_name, targ_name) == 0) { | |
2694 | /* matching named field */ | |
2695 | struct bpf_core_accessor *targ_acc; | |
2696 | ||
2697 | targ_acc = &spec->spec[spec->len++]; | |
2698 | targ_acc->type_id = targ_id; | |
2699 | targ_acc->idx = i; | |
2700 | targ_acc->name = targ_name; | |
2701 | ||
2702 | *next_targ_id = m->type; | |
2703 | found = bpf_core_fields_are_compat(local_btf, | |
2704 | local_member->type, | |
2705 | targ_btf, m->type); | |
2706 | if (!found) | |
2707 | spec->len--; /* pop accessor */ | |
2708 | return found; | |
2709 | } | |
2710 | /* member turned out not to be what we looked for */ | |
2711 | spec->offset -= offset / 8; | |
2712 | spec->raw_len--; | |
2713 | } | |
2714 | ||
2715 | return 0; | |
2716 | } | |
2717 | ||
2718 | /* | |
2719 | * Try to match local spec to a target type and, if successful, produce full | |
2720 | * target spec (high-level, low-level + offset). | |
2721 | */ | |
2722 | static int bpf_core_spec_match(struct bpf_core_spec *local_spec, | |
2723 | const struct btf *targ_btf, __u32 targ_id, | |
2724 | struct bpf_core_spec *targ_spec) | |
2725 | { | |
2726 | const struct btf_type *targ_type; | |
2727 | const struct bpf_core_accessor *local_acc; | |
2728 | struct bpf_core_accessor *targ_acc; | |
2729 | int i, sz, matched; | |
2730 | ||
2731 | memset(targ_spec, 0, sizeof(*targ_spec)); | |
2732 | targ_spec->btf = targ_btf; | |
2733 | ||
2734 | local_acc = &local_spec->spec[0]; | |
2735 | targ_acc = &targ_spec->spec[0]; | |
2736 | ||
2737 | for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) { | |
2738 | targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, | |
2739 | &targ_id); | |
2740 | if (!targ_type) | |
2741 | return -EINVAL; | |
2742 | ||
2743 | if (local_acc->name) { | |
2744 | matched = bpf_core_match_member(local_spec->btf, | |
2745 | local_acc, | |
2746 | targ_btf, targ_id, | |
2747 | targ_spec, &targ_id); | |
2748 | if (matched <= 0) | |
2749 | return matched; | |
2750 | } else { | |
2751 | /* for i=0, targ_id is already treated as array element | |
2752 | * type (because it's the original struct), for others | |
2753 | * we should find array element type first | |
2754 | */ | |
2755 | if (i > 0) { | |
2756 | const struct btf_array *a; | |
2757 | ||
2758 | if (!btf_is_array(targ_type)) | |
2759 | return 0; | |
2760 | ||
2761 | a = btf_array(targ_type); | |
2762 | if (local_acc->idx >= a->nelems) | |
2763 | return 0; | |
2764 | if (!skip_mods_and_typedefs(targ_btf, a->type, | |
2765 | &targ_id)) | |
2766 | return -EINVAL; | |
2767 | } | |
2768 | ||
2769 | /* too deep struct/union/array nesting */ | |
2770 | if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN) | |
2771 | return -E2BIG; | |
2772 | ||
2773 | targ_acc->type_id = targ_id; | |
2774 | targ_acc->idx = local_acc->idx; | |
2775 | targ_acc->name = NULL; | |
2776 | targ_spec->len++; | |
2777 | targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; | |
2778 | targ_spec->raw_len++; | |
2779 | ||
2780 | sz = btf__resolve_size(targ_btf, targ_id); | |
2781 | if (sz < 0) | |
2782 | return sz; | |
2783 | targ_spec->offset += local_acc->idx * sz; | |
2784 | } | |
2785 | } | |
2786 | ||
2787 | return 1; | |
2788 | } | |
2789 | ||
2790 | /* | |
2791 | * Patch relocatable BPF instruction. | |
62561eb4 AN |
2792 | * |
2793 | * Patched value is determined by relocation kind and target specification. | |
2794 | * For field existence relocation target spec will be NULL if field is not | |
2795 | * found. | |
2796 | * Expected insn->imm value is determined using relocation kind and local | |
2797 | * spec, and is checked before patching instruction. If actual insn->imm value | |
2798 | * is wrong, bail out with error. | |
ddc7c304 AN |
2799 | * |
2800 | * Currently three kinds of BPF instructions are supported: | |
2801 | * 1. rX = <imm> (assignment with immediate operand); | |
2802 | * 2. rX += <imm> (arithmetic operations with immediate operand); | |
ddc7c304 | 2803 | */ |
62561eb4 AN |
2804 | static int bpf_core_reloc_insn(struct bpf_program *prog, |
2805 | const struct bpf_field_reloc *relo, | |
2806 | const struct bpf_core_spec *local_spec, | |
2807 | const struct bpf_core_spec *targ_spec) | |
ddc7c304 | 2808 | { |
62561eb4 | 2809 | __u32 orig_val, new_val; |
ddc7c304 AN |
2810 | struct bpf_insn *insn; |
2811 | int insn_idx; | |
2812 | __u8 class; | |
2813 | ||
62561eb4 AN |
2814 | if (relo->insn_off % sizeof(struct bpf_insn)) |
2815 | return -EINVAL; | |
2816 | insn_idx = relo->insn_off / sizeof(struct bpf_insn); | |
2817 | ||
2818 | switch (relo->kind) { | |
2819 | case BPF_FIELD_BYTE_OFFSET: | |
2820 | orig_val = local_spec->offset; | |
2821 | if (targ_spec) { | |
2822 | new_val = targ_spec->offset; | |
2823 | } else { | |
be18010e KW |
2824 | pr_warn("prog '%s': patching insn #%d w/ failed reloc, imm %d -> %d\n", |
2825 | bpf_program__title(prog, false), insn_idx, | |
2826 | orig_val, -1); | |
62561eb4 AN |
2827 | new_val = (__u32)-1; |
2828 | } | |
2829 | break; | |
2830 | case BPF_FIELD_EXISTS: | |
2831 | orig_val = 1; /* can't generate EXISTS relo w/o local field */ | |
2832 | new_val = targ_spec ? 1 : 0; | |
2833 | break; | |
2834 | default: | |
be18010e KW |
2835 | pr_warn("prog '%s': unknown relo %d at insn #%d'\n", |
2836 | bpf_program__title(prog, false), | |
2837 | relo->kind, insn_idx); | |
ddc7c304 | 2838 | return -EINVAL; |
62561eb4 | 2839 | } |
ddc7c304 AN |
2840 | |
2841 | insn = &prog->insns[insn_idx]; | |
2842 | class = BPF_CLASS(insn->code); | |
2843 | ||
2844 | if (class == BPF_ALU || class == BPF_ALU64) { | |
2845 | if (BPF_SRC(insn->code) != BPF_K) | |
2846 | return -EINVAL; | |
62561eb4 | 2847 | if (insn->imm != orig_val) |
ddc7c304 | 2848 | return -EINVAL; |
62561eb4 | 2849 | insn->imm = new_val; |
ddc7c304 AN |
2850 | pr_debug("prog '%s': patched insn #%d (ALU/ALU64) imm %d -> %d\n", |
2851 | bpf_program__title(prog, false), | |
62561eb4 | 2852 | insn_idx, orig_val, new_val); |
ddc7c304 | 2853 | } else { |
be18010e KW |
2854 | pr_warn("prog '%s': trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n", |
2855 | bpf_program__title(prog, false), | |
2856 | insn_idx, insn->code, insn->src_reg, insn->dst_reg, | |
2857 | insn->off, insn->imm); | |
ddc7c304 AN |
2858 | return -EINVAL; |
2859 | } | |
62561eb4 | 2860 | |
ddc7c304 AN |
2861 | return 0; |
2862 | } | |
2863 | ||
a1916a15 AN |
2864 | static struct btf *btf_load_raw(const char *path) |
2865 | { | |
2866 | struct btf *btf; | |
2867 | size_t read_cnt; | |
2868 | struct stat st; | |
2869 | void *data; | |
2870 | FILE *f; | |
2871 | ||
2872 | if (stat(path, &st)) | |
2873 | return ERR_PTR(-errno); | |
2874 | ||
2875 | data = malloc(st.st_size); | |
2876 | if (!data) | |
2877 | return ERR_PTR(-ENOMEM); | |
2878 | ||
2879 | f = fopen(path, "rb"); | |
2880 | if (!f) { | |
2881 | btf = ERR_PTR(-errno); | |
2882 | goto cleanup; | |
2883 | } | |
2884 | ||
2885 | read_cnt = fread(data, 1, st.st_size, f); | |
2886 | fclose(f); | |
2887 | if (read_cnt < st.st_size) { | |
2888 | btf = ERR_PTR(-EBADF); | |
2889 | goto cleanup; | |
2890 | } | |
2891 | ||
2892 | btf = btf__new(data, read_cnt); | |
2893 | ||
2894 | cleanup: | |
2895 | free(data); | |
2896 | return btf; | |
2897 | } | |
2898 | ||
ddc7c304 AN |
2899 | /* |
2900 | * Probe few well-known locations for vmlinux kernel image and try to load BTF | |
2901 | * data out of it to use for target BTF. | |
2902 | */ | |
2903 | static struct btf *bpf_core_find_kernel_btf(void) | |
2904 | { | |
a1916a15 AN |
2905 | struct { |
2906 | const char *path_fmt; | |
2907 | bool raw_btf; | |
2908 | } locations[] = { | |
2909 | /* try canonical vmlinux BTF through sysfs first */ | |
2910 | { "/sys/kernel/btf/vmlinux", true /* raw BTF */ }, | |
2911 | /* fall back to trying to find vmlinux ELF on disk otherwise */ | |
2912 | { "/boot/vmlinux-%1$s" }, | |
2913 | { "/lib/modules/%1$s/vmlinux-%1$s" }, | |
2914 | { "/lib/modules/%1$s/build/vmlinux" }, | |
2915 | { "/usr/lib/modules/%1$s/kernel/vmlinux" }, | |
2916 | { "/usr/lib/debug/boot/vmlinux-%1$s" }, | |
2917 | { "/usr/lib/debug/boot/vmlinux-%1$s.debug" }, | |
2918 | { "/usr/lib/debug/lib/modules/%1$s/vmlinux" }, | |
ddc7c304 AN |
2919 | }; |
2920 | char path[PATH_MAX + 1]; | |
2921 | struct utsname buf; | |
2922 | struct btf *btf; | |
2923 | int i; | |
2924 | ||
2925 | uname(&buf); | |
2926 | ||
2927 | for (i = 0; i < ARRAY_SIZE(locations); i++) { | |
a1916a15 | 2928 | snprintf(path, PATH_MAX, locations[i].path_fmt, buf.release); |
ddc7c304 AN |
2929 | |
2930 | if (access(path, R_OK)) | |
2931 | continue; | |
2932 | ||
a1916a15 AN |
2933 | if (locations[i].raw_btf) |
2934 | btf = btf_load_raw(path); | |
2935 | else | |
2936 | btf = btf__parse_elf(path, NULL); | |
2937 | ||
2938 | pr_debug("loading kernel BTF '%s': %ld\n", | |
2939 | path, IS_ERR(btf) ? PTR_ERR(btf) : 0); | |
ddc7c304 AN |
2940 | if (IS_ERR(btf)) |
2941 | continue; | |
2942 | ||
2943 | return btf; | |
2944 | } | |
2945 | ||
be18010e | 2946 | pr_warn("failed to find valid kernel BTF\n"); |
ddc7c304 AN |
2947 | return ERR_PTR(-ESRCH); |
2948 | } | |
2949 | ||
2950 | /* Output spec definition in the format: | |
2951 | * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>, | |
2952 | * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b | |
2953 | */ | |
2954 | static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) | |
2955 | { | |
2956 | const struct btf_type *t; | |
2957 | const char *s; | |
2958 | __u32 type_id; | |
2959 | int i; | |
2960 | ||
2961 | type_id = spec->spec[0].type_id; | |
2962 | t = btf__type_by_id(spec->btf, type_id); | |
2963 | s = btf__name_by_offset(spec->btf, t->name_off); | |
2964 | libbpf_print(level, "[%u] %s + ", type_id, s); | |
2965 | ||
2966 | for (i = 0; i < spec->raw_len; i++) | |
2967 | libbpf_print(level, "%d%s", spec->raw_spec[i], | |
2968 | i == spec->raw_len - 1 ? " => " : ":"); | |
2969 | ||
2970 | libbpf_print(level, "%u @ &x", spec->offset); | |
2971 | ||
2972 | for (i = 0; i < spec->len; i++) { | |
2973 | if (spec->spec[i].name) | |
2974 | libbpf_print(level, ".%s", spec->spec[i].name); | |
2975 | else | |
2976 | libbpf_print(level, "[%u]", spec->spec[i].idx); | |
2977 | } | |
2978 | ||
2979 | } | |
2980 | ||
2981 | static size_t bpf_core_hash_fn(const void *key, void *ctx) | |
2982 | { | |
2983 | return (size_t)key; | |
2984 | } | |
2985 | ||
2986 | static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx) | |
2987 | { | |
2988 | return k1 == k2; | |
2989 | } | |
2990 | ||
2991 | static void *u32_as_hash_key(__u32 x) | |
2992 | { | |
2993 | return (void *)(uintptr_t)x; | |
2994 | } | |
2995 | ||
2996 | /* | |
2997 | * CO-RE relocate single instruction. | |
2998 | * | |
2999 | * The outline and important points of the algorithm: | |
3000 | * 1. For given local type, find corresponding candidate target types. | |
3001 | * Candidate type is a type with the same "essential" name, ignoring | |
3002 | * everything after last triple underscore (___). E.g., `sample`, | |
3003 | * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates | |
3004 | * for each other. Names with triple underscore are referred to as | |
3005 | * "flavors" and are useful, among other things, to allow to | |
3006 | * specify/support incompatible variations of the same kernel struct, which | |
3007 | * might differ between different kernel versions and/or build | |
3008 | * configurations. | |
3009 | * | |
3010 | * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C | |
3011 | * converter, when deduplicated BTF of a kernel still contains more than | |
3012 | * one different types with the same name. In that case, ___2, ___3, etc | |
3013 | * are appended starting from second name conflict. But start flavors are | |
3014 | * also useful to be defined "locally", in BPF program, to extract same | |
3015 | * data from incompatible changes between different kernel | |
3016 | * versions/configurations. For instance, to handle field renames between | |
3017 | * kernel versions, one can use two flavors of the struct name with the | |
3018 | * same common name and use conditional relocations to extract that field, | |
3019 | * depending on target kernel version. | |
3020 | * 2. For each candidate type, try to match local specification to this | |
3021 | * candidate target type. Matching involves finding corresponding | |
3022 | * high-level spec accessors, meaning that all named fields should match, | |
3023 | * as well as all array accesses should be within the actual bounds. Also, | |
3024 | * types should be compatible (see bpf_core_fields_are_compat for details). | |
3025 | * 3. It is supported and expected that there might be multiple flavors | |
3026 | * matching the spec. As long as all the specs resolve to the same set of | |
511bb008 | 3027 | * offsets across all candidates, there is no error. If there is any |
ddc7c304 AN |
3028 | * ambiguity, CO-RE relocation will fail. This is necessary to accomodate |
3029 | * imprefection of BTF deduplication, which can cause slight duplication of | |
3030 | * the same BTF type, if some directly or indirectly referenced (by | |
3031 | * pointer) type gets resolved to different actual types in different | |
3032 | * object files. If such situation occurs, deduplicated BTF will end up | |
3033 | * with two (or more) structurally identical types, which differ only in | |
3034 | * types they refer to through pointer. This should be OK in most cases and | |
3035 | * is not an error. | |
3036 | * 4. Candidate types search is performed by linearly scanning through all | |
3037 | * types in target BTF. It is anticipated that this is overall more | |
3038 | * efficient memory-wise and not significantly worse (if not better) | |
3039 | * CPU-wise compared to prebuilding a map from all local type names to | |
3040 | * a list of candidate type names. It's also sped up by caching resolved | |
3041 | * list of matching candidates per each local "root" type ID, that has at | |
511bb008 | 3042 | * least one bpf_field_reloc associated with it. This list is shared |
ddc7c304 AN |
3043 | * between multiple relocations for the same type ID and is updated as some |
3044 | * of the candidates are pruned due to structural incompatibility. | |
3045 | */ | |
511bb008 AN |
3046 | static int bpf_core_reloc_field(struct bpf_program *prog, |
3047 | const struct bpf_field_reloc *relo, | |
ddc7c304 AN |
3048 | int relo_idx, |
3049 | const struct btf *local_btf, | |
3050 | const struct btf *targ_btf, | |
3051 | struct hashmap *cand_cache) | |
3052 | { | |
3053 | const char *prog_name = bpf_program__title(prog, false); | |
3054 | struct bpf_core_spec local_spec, cand_spec, targ_spec; | |
3055 | const void *type_key = u32_as_hash_key(relo->type_id); | |
3056 | const struct btf_type *local_type, *cand_type; | |
3057 | const char *local_name, *cand_name; | |
3058 | struct ids_vec *cand_ids; | |
3059 | __u32 local_id, cand_id; | |
3060 | const char *spec_str; | |
3061 | int i, j, err; | |
3062 | ||
3063 | local_id = relo->type_id; | |
3064 | local_type = btf__type_by_id(local_btf, local_id); | |
3065 | if (!local_type) | |
3066 | return -EINVAL; | |
3067 | ||
3068 | local_name = btf__name_by_offset(local_btf, local_type->name_off); | |
3069 | if (str_is_empty(local_name)) | |
3070 | return -EINVAL; | |
3071 | ||
3072 | spec_str = btf__name_by_offset(local_btf, relo->access_str_off); | |
3073 | if (str_is_empty(spec_str)) | |
3074 | return -EINVAL; | |
3075 | ||
3076 | err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec); | |
3077 | if (err) { | |
be18010e KW |
3078 | pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n", |
3079 | prog_name, relo_idx, local_id, local_name, spec_str, | |
3080 | err); | |
ddc7c304 AN |
3081 | return -EINVAL; |
3082 | } | |
3083 | ||
3084 | pr_debug("prog '%s': relo #%d: spec is ", prog_name, relo_idx); | |
3085 | bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec); | |
3086 | libbpf_print(LIBBPF_DEBUG, "\n"); | |
3087 | ||
3088 | if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) { | |
3089 | cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf); | |
3090 | if (IS_ERR(cand_ids)) { | |
be18010e KW |
3091 | pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld", |
3092 | prog_name, relo_idx, local_id, local_name, | |
3093 | PTR_ERR(cand_ids)); | |
ddc7c304 AN |
3094 | return PTR_ERR(cand_ids); |
3095 | } | |
3096 | err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL); | |
3097 | if (err) { | |
3098 | bpf_core_free_cands(cand_ids); | |
3099 | return err; | |
3100 | } | |
3101 | } | |
3102 | ||
3103 | for (i = 0, j = 0; i < cand_ids->len; i++) { | |
3104 | cand_id = cand_ids->data[i]; | |
3105 | cand_type = btf__type_by_id(targ_btf, cand_id); | |
3106 | cand_name = btf__name_by_offset(targ_btf, cand_type->name_off); | |
3107 | ||
3108 | err = bpf_core_spec_match(&local_spec, targ_btf, | |
3109 | cand_id, &cand_spec); | |
3110 | pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ", | |
3111 | prog_name, relo_idx, i, cand_name); | |
3112 | bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec); | |
3113 | libbpf_print(LIBBPF_DEBUG, ": %d\n", err); | |
3114 | if (err < 0) { | |
be18010e KW |
3115 | pr_warn("prog '%s': relo #%d: matching error: %d\n", |
3116 | prog_name, relo_idx, err); | |
ddc7c304 AN |
3117 | return err; |
3118 | } | |
3119 | if (err == 0) | |
3120 | continue; | |
3121 | ||
3122 | if (j == 0) { | |
3123 | targ_spec = cand_spec; | |
3124 | } else if (cand_spec.offset != targ_spec.offset) { | |
3125 | /* if there are many candidates, they should all | |
3126 | * resolve to the same offset | |
3127 | */ | |
be18010e KW |
3128 | pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n", |
3129 | prog_name, relo_idx, cand_spec.offset, | |
3130 | targ_spec.offset); | |
ddc7c304 AN |
3131 | return -EINVAL; |
3132 | } | |
3133 | ||
3134 | cand_ids->data[j++] = cand_spec.spec[0].type_id; | |
3135 | } | |
3136 | ||
62561eb4 AN |
3137 | /* |
3138 | * For BPF_FIELD_EXISTS relo or when relaxed CO-RE reloc mode is | |
3139 | * requested, it's expected that we might not find any candidates. | |
3140 | * In this case, if field wasn't found in any candidate, the list of | |
3141 | * candidates shouldn't change at all, we'll just handle relocating | |
3142 | * appropriately, depending on relo's kind. | |
3143 | */ | |
3144 | if (j > 0) | |
3145 | cand_ids->len = j; | |
3146 | ||
3147 | if (j == 0 && !prog->obj->relaxed_core_relocs && | |
3148 | relo->kind != BPF_FIELD_EXISTS) { | |
be18010e KW |
3149 | pr_warn("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n", |
3150 | prog_name, relo_idx, local_id, local_name, spec_str); | |
ddc7c304 AN |
3151 | return -ESRCH; |
3152 | } | |
3153 | ||
62561eb4 AN |
3154 | /* bpf_core_reloc_insn should know how to handle missing targ_spec */ |
3155 | err = bpf_core_reloc_insn(prog, relo, &local_spec, | |
3156 | j ? &targ_spec : NULL); | |
ddc7c304 | 3157 | if (err) { |
be18010e KW |
3158 | pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n", |
3159 | prog_name, relo_idx, relo->insn_off, err); | |
ddc7c304 AN |
3160 | return -EINVAL; |
3161 | } | |
3162 | ||
3163 | return 0; | |
3164 | } | |
3165 | ||
3166 | static int | |
511bb008 | 3167 | bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path) |
ddc7c304 AN |
3168 | { |
3169 | const struct btf_ext_info_sec *sec; | |
511bb008 | 3170 | const struct bpf_field_reloc *rec; |
ddc7c304 AN |
3171 | const struct btf_ext_info *seg; |
3172 | struct hashmap_entry *entry; | |
3173 | struct hashmap *cand_cache = NULL; | |
3174 | struct bpf_program *prog; | |
3175 | struct btf *targ_btf; | |
3176 | const char *sec_name; | |
3177 | int i, err = 0; | |
3178 | ||
3179 | if (targ_btf_path) | |
3180 | targ_btf = btf__parse_elf(targ_btf_path, NULL); | |
3181 | else | |
3182 | targ_btf = bpf_core_find_kernel_btf(); | |
3183 | if (IS_ERR(targ_btf)) { | |
be18010e | 3184 | pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf)); |
ddc7c304 AN |
3185 | return PTR_ERR(targ_btf); |
3186 | } | |
3187 | ||
3188 | cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); | |
3189 | if (IS_ERR(cand_cache)) { | |
3190 | err = PTR_ERR(cand_cache); | |
3191 | goto out; | |
3192 | } | |
3193 | ||
511bb008 | 3194 | seg = &obj->btf_ext->field_reloc_info; |
ddc7c304 AN |
3195 | for_each_btf_ext_sec(seg, sec) { |
3196 | sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); | |
3197 | if (str_is_empty(sec_name)) { | |
3198 | err = -EINVAL; | |
3199 | goto out; | |
3200 | } | |
3201 | prog = bpf_object__find_program_by_title(obj, sec_name); | |
3202 | if (!prog) { | |
be18010e KW |
3203 | pr_warn("failed to find program '%s' for CO-RE offset relocation\n", |
3204 | sec_name); | |
ddc7c304 AN |
3205 | err = -EINVAL; |
3206 | goto out; | |
3207 | } | |
3208 | ||
3209 | pr_debug("prog '%s': performing %d CO-RE offset relocs\n", | |
3210 | sec_name, sec->num_info); | |
3211 | ||
3212 | for_each_btf_ext_rec(seg, sec, i, rec) { | |
511bb008 AN |
3213 | err = bpf_core_reloc_field(prog, rec, i, obj->btf, |
3214 | targ_btf, cand_cache); | |
ddc7c304 | 3215 | if (err) { |
be18010e KW |
3216 | pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", |
3217 | sec_name, i, err); | |
ddc7c304 AN |
3218 | goto out; |
3219 | } | |
3220 | } | |
3221 | } | |
3222 | ||
3223 | out: | |
3224 | btf__free(targ_btf); | |
3225 | if (!IS_ERR_OR_NULL(cand_cache)) { | |
3226 | hashmap__for_each_entry(cand_cache, entry, i) { | |
3227 | bpf_core_free_cands(entry->value); | |
3228 | } | |
3229 | hashmap__free(cand_cache); | |
3230 | } | |
3231 | return err; | |
3232 | } | |
3233 | ||
3234 | static int | |
3235 | bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) | |
3236 | { | |
3237 | int err = 0; | |
3238 | ||
511bb008 AN |
3239 | if (obj->btf_ext->field_reloc_info.len) |
3240 | err = bpf_core_reloc_fields(obj, targ_btf_path); | |
ddc7c304 AN |
3241 | |
3242 | return err; | |
3243 | } | |
3244 | ||
48cca7e4 AS |
3245 | static int |
3246 | bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, | |
3247 | struct reloc_desc *relo) | |
3248 | { | |
3249 | struct bpf_insn *insn, *new_insn; | |
3250 | struct bpf_program *text; | |
3251 | size_t new_cnt; | |
2993e051 | 3252 | int err; |
48cca7e4 AS |
3253 | |
3254 | if (relo->type != RELO_CALL) | |
3255 | return -LIBBPF_ERRNO__RELOC; | |
3256 | ||
3257 | if (prog->idx == obj->efile.text_shndx) { | |
be18010e KW |
3258 | pr_warn("relo in .text insn %d into off %d\n", |
3259 | relo->insn_idx, relo->text_off); | |
48cca7e4 AS |
3260 | return -LIBBPF_ERRNO__RELOC; |
3261 | } | |
3262 | ||
3263 | if (prog->main_prog_cnt == 0) { | |
3264 | text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); | |
3265 | if (!text) { | |
be18010e | 3266 | pr_warn("no .text section found yet relo into text exist\n"); |
48cca7e4 AS |
3267 | return -LIBBPF_ERRNO__RELOC; |
3268 | } | |
3269 | new_cnt = prog->insns_cnt + text->insns_cnt; | |
531b014e | 3270 | new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); |
48cca7e4 | 3271 | if (!new_insn) { |
be18010e | 3272 | pr_warn("oom in prog realloc\n"); |
48cca7e4 AS |
3273 | return -ENOMEM; |
3274 | } | |
2993e051 | 3275 | |
f0187f0b MKL |
3276 | if (obj->btf_ext) { |
3277 | err = bpf_program_reloc_btf_ext(prog, obj, | |
3278 | text->section_name, | |
3279 | prog->insns_cnt); | |
3280 | if (err) | |
2993e051 | 3281 | return err; |
2993e051 YS |
3282 | } |
3283 | ||
48cca7e4 AS |
3284 | memcpy(new_insn + prog->insns_cnt, text->insns, |
3285 | text->insns_cnt * sizeof(*insn)); | |
3286 | prog->insns = new_insn; | |
3287 | prog->main_prog_cnt = prog->insns_cnt; | |
3288 | prog->insns_cnt = new_cnt; | |
b1a2ce82 JC |
3289 | pr_debug("added %zd insn from %s to prog %s\n", |
3290 | text->insns_cnt, text->section_name, | |
3291 | prog->section_name); | |
48cca7e4 AS |
3292 | } |
3293 | insn = &prog->insns[relo->insn_idx]; | |
3294 | insn->imm += prog->main_prog_cnt - relo->insn_idx; | |
48cca7e4 AS |
3295 | return 0; |
3296 | } | |
3297 | ||
8a47a6c5 | 3298 | static int |
9d759a9b | 3299 | bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) |
8a47a6c5 | 3300 | { |
48cca7e4 | 3301 | int i, err; |
8a47a6c5 | 3302 | |
2993e051 YS |
3303 | if (!prog) |
3304 | return 0; | |
3305 | ||
f0187f0b MKL |
3306 | if (obj->btf_ext) { |
3307 | err = bpf_program_reloc_btf_ext(prog, obj, | |
3308 | prog->section_name, 0); | |
3309 | if (err) | |
2993e051 | 3310 | return err; |
2993e051 YS |
3311 | } |
3312 | ||
3313 | if (!prog->reloc_desc) | |
8a47a6c5 WN |
3314 | return 0; |
3315 | ||
3316 | for (i = 0; i < prog->nr_reloc; i++) { | |
d859900c DB |
3317 | if (prog->reloc_desc[i].type == RELO_LD64 || |
3318 | prog->reloc_desc[i].type == RELO_DATA) { | |
3319 | bool relo_data = prog->reloc_desc[i].type == RELO_DATA; | |
48cca7e4 AS |
3320 | struct bpf_insn *insns = prog->insns; |
3321 | int insn_idx, map_idx; | |
8a47a6c5 | 3322 | |
48cca7e4 AS |
3323 | insn_idx = prog->reloc_desc[i].insn_idx; |
3324 | map_idx = prog->reloc_desc[i].map_idx; | |
8a47a6c5 | 3325 | |
d859900c | 3326 | if (insn_idx + 1 >= (int)prog->insns_cnt) { |
be18010e KW |
3327 | pr_warn("relocation out of range: '%s'\n", |
3328 | prog->section_name); | |
48cca7e4 AS |
3329 | return -LIBBPF_ERRNO__RELOC; |
3330 | } | |
d859900c DB |
3331 | |
3332 | if (!relo_data) { | |
3333 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
3334 | } else { | |
3335 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_VALUE; | |
3336 | insns[insn_idx + 1].imm = insns[insn_idx].imm; | |
3337 | } | |
48cca7e4 | 3338 | insns[insn_idx].imm = obj->maps[map_idx].fd; |
f8c7a4d4 | 3339 | } else if (prog->reloc_desc[i].type == RELO_CALL) { |
48cca7e4 AS |
3340 | err = bpf_program__reloc_text(prog, obj, |
3341 | &prog->reloc_desc[i]); | |
3342 | if (err) | |
3343 | return err; | |
8a47a6c5 | 3344 | } |
8a47a6c5 WN |
3345 | } |
3346 | ||
3347 | zfree(&prog->reloc_desc); | |
3348 | prog->nr_reloc = 0; | |
3349 | return 0; | |
3350 | } | |
3351 | ||
8a47a6c5 | 3352 | static int |
ddc7c304 | 3353 | bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) |
8a47a6c5 WN |
3354 | { |
3355 | struct bpf_program *prog; | |
3356 | size_t i; | |
3357 | int err; | |
3358 | ||
ddc7c304 AN |
3359 | if (obj->btf_ext) { |
3360 | err = bpf_object__relocate_core(obj, targ_btf_path); | |
3361 | if (err) { | |
be18010e KW |
3362 | pr_warn("failed to perform CO-RE relocations: %d\n", |
3363 | err); | |
ddc7c304 AN |
3364 | return err; |
3365 | } | |
3366 | } | |
8a47a6c5 WN |
3367 | for (i = 0; i < obj->nr_programs; i++) { |
3368 | prog = &obj->programs[i]; | |
3369 | ||
9d759a9b | 3370 | err = bpf_program__relocate(prog, obj); |
8a47a6c5 | 3371 | if (err) { |
be18010e | 3372 | pr_warn("failed to relocate '%s'\n", prog->section_name); |
8a47a6c5 WN |
3373 | return err; |
3374 | } | |
3375 | } | |
3376 | return 0; | |
3377 | } | |
3378 | ||
34090915 WN |
3379 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
3380 | { | |
3381 | int i, err; | |
3382 | ||
3383 | if (!obj_elf_valid(obj)) { | |
be18010e | 3384 | pr_warn("Internal error: elf object is closed\n"); |
6371ca3b | 3385 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
3386 | } |
3387 | ||
3388 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
3389 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
3390 | Elf_Data *data = obj->efile.reloc[i].data; | |
3391 | int idx = shdr->sh_info; | |
3392 | struct bpf_program *prog; | |
34090915 WN |
3393 | |
3394 | if (shdr->sh_type != SHT_REL) { | |
be18010e | 3395 | pr_warn("internal error at %d\n", __LINE__); |
6371ca3b | 3396 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
3397 | } |
3398 | ||
3399 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
3400 | if (!prog) { | |
be18010e | 3401 | pr_warn("relocation failed: no section(%d)\n", idx); |
6371ca3b | 3402 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
3403 | } |
3404 | ||
399dc65e | 3405 | err = bpf_program__collect_reloc(prog, shdr, data, obj); |
34090915 | 3406 | if (err) |
6371ca3b | 3407 | return err; |
34090915 WN |
3408 | } |
3409 | return 0; | |
3410 | } | |
3411 | ||
55cffde2 | 3412 | static int |
2993e051 | 3413 | load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, |
f0187f0b | 3414 | char *license, __u32 kern_version, int *pfd) |
55cffde2 | 3415 | { |
d7be143b | 3416 | struct bpf_load_program_attr load_attr; |
1ce6a9fc | 3417 | char *cp, errmsg[STRERR_BUFSIZE]; |
da11b417 | 3418 | int log_buf_size = BPF_LOG_BUF_SIZE; |
55cffde2 | 3419 | char *log_buf; |
5d01ab7b | 3420 | int btf_fd, ret; |
55cffde2 | 3421 | |
fba01a06 AN |
3422 | if (!insns || !insns_cnt) |
3423 | return -EINVAL; | |
3424 | ||
d7be143b | 3425 | memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); |
2993e051 YS |
3426 | load_attr.prog_type = prog->type; |
3427 | load_attr.expected_attach_type = prog->expected_attach_type; | |
5b32a23e SF |
3428 | if (prog->caps->name) |
3429 | load_attr.name = prog->name; | |
d7be143b AI |
3430 | load_attr.insns = insns; |
3431 | load_attr.insns_cnt = insns_cnt; | |
3432 | load_attr.license = license; | |
3433 | load_attr.kern_version = kern_version; | |
2993e051 | 3434 | load_attr.prog_ifindex = prog->prog_ifindex; |
3415ec64 AN |
3435 | /* if .BTF.ext was loaded, kernel supports associated BTF for prog */ |
3436 | if (prog->obj->btf_ext) | |
3437 | btf_fd = bpf_object__btf_fd(prog->obj); | |
3438 | else | |
3439 | btf_fd = -1; | |
5d01ab7b | 3440 | load_attr.prog_btf_fd = btf_fd >= 0 ? btf_fd : 0; |
2993e051 YS |
3441 | load_attr.func_info = prog->func_info; |
3442 | load_attr.func_info_rec_size = prog->func_info_rec_size; | |
f0187f0b | 3443 | load_attr.func_info_cnt = prog->func_info_cnt; |
3d650141 MKL |
3444 | load_attr.line_info = prog->line_info; |
3445 | load_attr.line_info_rec_size = prog->line_info_rec_size; | |
3446 | load_attr.line_info_cnt = prog->line_info_cnt; | |
da11b417 | 3447 | load_attr.log_level = prog->log_level; |
04656198 | 3448 | load_attr.prog_flags = prog->prog_flags; |
55cffde2 | 3449 | |
da11b417 AS |
3450 | retry_load: |
3451 | log_buf = malloc(log_buf_size); | |
55cffde2 | 3452 | if (!log_buf) |
be18010e | 3453 | pr_warn("Alloc log buffer for bpf loader error, continue without log\n"); |
55cffde2 | 3454 | |
da11b417 | 3455 | ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); |
55cffde2 WN |
3456 | |
3457 | if (ret >= 0) { | |
da11b417 AS |
3458 | if (load_attr.log_level) |
3459 | pr_debug("verifier log:\n%s", log_buf); | |
55cffde2 WN |
3460 | *pfd = ret; |
3461 | ret = 0; | |
3462 | goto out; | |
3463 | } | |
3464 | ||
da11b417 AS |
3465 | if (errno == ENOSPC) { |
3466 | log_buf_size <<= 1; | |
3467 | free(log_buf); | |
3468 | goto retry_load; | |
3469 | } | |
6371ca3b | 3470 | ret = -LIBBPF_ERRNO__LOAD; |
24d6a808 | 3471 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
be18010e | 3472 | pr_warn("load bpf program failed: %s\n", cp); |
55cffde2 | 3473 | |
6371ca3b WN |
3474 | if (log_buf && log_buf[0] != '\0') { |
3475 | ret = -LIBBPF_ERRNO__VERIFY; | |
be18010e KW |
3476 | pr_warn("-- BEGIN DUMP LOG ---\n"); |
3477 | pr_warn("\n%s\n", log_buf); | |
3478 | pr_warn("-- END LOG --\n"); | |
d7be143b | 3479 | } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { |
be18010e KW |
3480 | pr_warn("Program too large (%zu insns), at most %d insns\n", |
3481 | load_attr.insns_cnt, BPF_MAXINSNS); | |
705fa219 | 3482 | ret = -LIBBPF_ERRNO__PROG2BIG; |
6371ca3b | 3483 | } else { |
705fa219 | 3484 | /* Wrong program type? */ |
d7be143b | 3485 | if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { |
705fa219 WN |
3486 | int fd; |
3487 | ||
d7be143b AI |
3488 | load_attr.prog_type = BPF_PROG_TYPE_KPROBE; |
3489 | load_attr.expected_attach_type = 0; | |
3490 | fd = bpf_load_program_xattr(&load_attr, NULL, 0); | |
705fa219 WN |
3491 | if (fd >= 0) { |
3492 | close(fd); | |
3493 | ret = -LIBBPF_ERRNO__PROGTYPE; | |
3494 | goto out; | |
3495 | } | |
6371ca3b | 3496 | } |
705fa219 WN |
3497 | |
3498 | if (log_buf) | |
3499 | ret = -LIBBPF_ERRNO__KVER; | |
55cffde2 WN |
3500 | } |
3501 | ||
3502 | out: | |
3503 | free(log_buf); | |
3504 | return ret; | |
3505 | } | |
3506 | ||
29cd77f4 | 3507 | int |
55cffde2 | 3508 | bpf_program__load(struct bpf_program *prog, |
e5b0863c | 3509 | char *license, __u32 kern_version) |
55cffde2 | 3510 | { |
b580563e | 3511 | int err = 0, fd, i; |
55cffde2 | 3512 | |
b580563e WN |
3513 | if (prog->instances.nr < 0 || !prog->instances.fds) { |
3514 | if (prog->preprocessor) { | |
be18010e KW |
3515 | pr_warn("Internal error: can't load program '%s'\n", |
3516 | prog->section_name); | |
b580563e WN |
3517 | return -LIBBPF_ERRNO__INTERNAL; |
3518 | } | |
55cffde2 | 3519 | |
b580563e WN |
3520 | prog->instances.fds = malloc(sizeof(int)); |
3521 | if (!prog->instances.fds) { | |
be18010e | 3522 | pr_warn("Not enough memory for BPF fds\n"); |
b580563e WN |
3523 | return -ENOMEM; |
3524 | } | |
3525 | prog->instances.nr = 1; | |
3526 | prog->instances.fds[0] = -1; | |
3527 | } | |
3528 | ||
3529 | if (!prog->preprocessor) { | |
3530 | if (prog->instances.nr != 1) { | |
be18010e KW |
3531 | pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n", |
3532 | prog->section_name, prog->instances.nr); | |
b580563e | 3533 | } |
2993e051 | 3534 | err = load_program(prog, prog->insns, prog->insns_cnt, |
f0187f0b | 3535 | license, kern_version, &fd); |
b580563e WN |
3536 | if (!err) |
3537 | prog->instances.fds[0] = fd; | |
3538 | goto out; | |
3539 | } | |
3540 | ||
3541 | for (i = 0; i < prog->instances.nr; i++) { | |
3542 | struct bpf_prog_prep_result result; | |
3543 | bpf_program_prep_t preprocessor = prog->preprocessor; | |
3544 | ||
1ad9cbb8 | 3545 | memset(&result, 0, sizeof(result)); |
b580563e WN |
3546 | err = preprocessor(prog, i, prog->insns, |
3547 | prog->insns_cnt, &result); | |
3548 | if (err) { | |
be18010e KW |
3549 | pr_warn("Preprocessing the %dth instance of program '%s' failed\n", |
3550 | i, prog->section_name); | |
b580563e WN |
3551 | goto out; |
3552 | } | |
3553 | ||
3554 | if (!result.new_insn_ptr || !result.new_insn_cnt) { | |
3555 | pr_debug("Skip loading the %dth instance of program '%s'\n", | |
3556 | i, prog->section_name); | |
3557 | prog->instances.fds[i] = -1; | |
3558 | if (result.pfd) | |
3559 | *result.pfd = -1; | |
3560 | continue; | |
3561 | } | |
3562 | ||
2993e051 | 3563 | err = load_program(prog, result.new_insn_ptr, |
b580563e | 3564 | result.new_insn_cnt, |
f0187f0b | 3565 | license, kern_version, &fd); |
b580563e WN |
3566 | |
3567 | if (err) { | |
be18010e KW |
3568 | pr_warn("Loading the %dth instance of program '%s' failed\n", |
3569 | i, prog->section_name); | |
b580563e WN |
3570 | goto out; |
3571 | } | |
3572 | ||
3573 | if (result.pfd) | |
3574 | *result.pfd = fd; | |
3575 | prog->instances.fds[i] = fd; | |
3576 | } | |
3577 | out: | |
55cffde2 | 3578 | if (err) |
be18010e | 3579 | pr_warn("failed to load program '%s'\n", prog->section_name); |
55cffde2 WN |
3580 | zfree(&prog->insns); |
3581 | prog->insns_cnt = 0; | |
3582 | return err; | |
3583 | } | |
3584 | ||
a324aae3 AN |
3585 | static bool bpf_program__is_function_storage(const struct bpf_program *prog, |
3586 | const struct bpf_object *obj) | |
9a94f277 JK |
3587 | { |
3588 | return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; | |
3589 | } | |
3590 | ||
55cffde2 | 3591 | static int |
60276f98 | 3592 | bpf_object__load_progs(struct bpf_object *obj, int log_level) |
55cffde2 WN |
3593 | { |
3594 | size_t i; | |
3595 | int err; | |
3596 | ||
3597 | for (i = 0; i < obj->nr_programs; i++) { | |
9a94f277 | 3598 | if (bpf_program__is_function_storage(&obj->programs[i], obj)) |
48cca7e4 | 3599 | continue; |
501b125a | 3600 | obj->programs[i].log_level |= log_level; |
55cffde2 WN |
3601 | err = bpf_program__load(&obj->programs[i], |
3602 | obj->license, | |
3603 | obj->kern_version); | |
3604 | if (err) | |
3605 | return err; | |
3606 | } | |
3607 | return 0; | |
3608 | } | |
3609 | ||
1a5e3fb1 | 3610 | static struct bpf_object * |
5e61f270 | 3611 | __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, |
291ee02b | 3612 | struct bpf_object_open_opts *opts) |
1a5e3fb1 WN |
3613 | { |
3614 | struct bpf_object *obj; | |
291ee02b AN |
3615 | const char *obj_name; |
3616 | char tmp_name[64]; | |
3617 | bool relaxed_maps; | |
6371ca3b | 3618 | int err; |
1a5e3fb1 WN |
3619 | |
3620 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
be18010e KW |
3621 | pr_warn("failed to init libelf for %s\n", |
3622 | path ? : "(mem buf)"); | |
6371ca3b | 3623 | return ERR_PTR(-LIBBPF_ERRNO__LIBELF); |
1a5e3fb1 WN |
3624 | } |
3625 | ||
291ee02b AN |
3626 | if (!OPTS_VALID(opts, bpf_object_open_opts)) |
3627 | return ERR_PTR(-EINVAL); | |
3628 | ||
3629 | obj_name = OPTS_GET(opts, object_name, path); | |
3630 | if (obj_buf) { | |
3631 | if (!obj_name) { | |
3632 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
3633 | (unsigned long)obj_buf, | |
3634 | (unsigned long)obj_buf_sz); | |
3635 | obj_name = tmp_name; | |
3636 | } | |
3637 | path = obj_name; | |
3638 | pr_debug("loading object '%s' from buffer\n", obj_name); | |
3639 | } | |
3640 | ||
2ce8450e | 3641 | obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); |
6371ca3b WN |
3642 | if (IS_ERR(obj)) |
3643 | return obj; | |
1a5e3fb1 | 3644 | |
62561eb4 | 3645 | obj->relaxed_core_relocs = OPTS_GET(opts, relaxed_core_relocs, false); |
291ee02b AN |
3646 | relaxed_maps = OPTS_GET(opts, relaxed_maps, false); |
3647 | ||
6371ca3b WN |
3648 | CHECK_ERR(bpf_object__elf_init(obj), err, out); |
3649 | CHECK_ERR(bpf_object__check_endianness(obj), err, out); | |
8837fe5d | 3650 | CHECK_ERR(bpf_object__probe_caps(obj), err, out); |
291ee02b | 3651 | CHECK_ERR(bpf_object__elf_collect(obj, relaxed_maps), err, out); |
6371ca3b | 3652 | CHECK_ERR(bpf_object__collect_reloc(obj), err, out); |
1a5e3fb1 WN |
3653 | |
3654 | bpf_object__elf_finish(obj); | |
3655 | return obj; | |
3656 | out: | |
3657 | bpf_object__close(obj); | |
6371ca3b | 3658 | return ERR_PTR(err); |
1a5e3fb1 WN |
3659 | } |
3660 | ||
5e61f270 AN |
3661 | static struct bpf_object * |
3662 | __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) | |
1a5e3fb1 | 3663 | { |
291ee02b AN |
3664 | LIBBPF_OPTS(bpf_object_open_opts, opts, |
3665 | .relaxed_maps = flags & MAPS_RELAX_COMPAT, | |
3666 | ); | |
3667 | ||
1a5e3fb1 | 3668 | /* param validation */ |
07f2d4ea | 3669 | if (!attr->file) |
1a5e3fb1 WN |
3670 | return NULL; |
3671 | ||
07f2d4ea | 3672 | pr_debug("loading %s\n", attr->file); |
291ee02b | 3673 | return __bpf_object__open(attr->file, NULL, 0, &opts); |
c034a177 JF |
3674 | } |
3675 | ||
3676 | struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) | |
3677 | { | |
3678 | return __bpf_object__open_xattr(attr, 0); | |
07f2d4ea JK |
3679 | } |
3680 | ||
3681 | struct bpf_object *bpf_object__open(const char *path) | |
3682 | { | |
3683 | struct bpf_object_open_attr attr = { | |
3684 | .file = path, | |
3685 | .prog_type = BPF_PROG_TYPE_UNSPEC, | |
3686 | }; | |
1a5e3fb1 | 3687 | |
07f2d4ea | 3688 | return bpf_object__open_xattr(&attr); |
6c956392 WN |
3689 | } |
3690 | ||
2ce8450e AN |
3691 | struct bpf_object * |
3692 | bpf_object__open_file(const char *path, struct bpf_object_open_opts *opts) | |
3693 | { | |
2ce8450e AN |
3694 | if (!path) |
3695 | return ERR_PTR(-EINVAL); | |
3696 | ||
3697 | pr_debug("loading %s\n", path); | |
3698 | ||
291ee02b | 3699 | return __bpf_object__open(path, NULL, 0, opts); |
2ce8450e AN |
3700 | } |
3701 | ||
3702 | struct bpf_object * | |
3703 | bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, | |
3704 | struct bpf_object_open_opts *opts) | |
6c956392 | 3705 | { |
2ce8450e AN |
3706 | if (!obj_buf || obj_buf_sz == 0) |
3707 | return ERR_PTR(-EINVAL); | |
6c956392 | 3708 | |
291ee02b | 3709 | return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts); |
2ce8450e AN |
3710 | } |
3711 | ||
3712 | struct bpf_object * | |
3713 | bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, | |
3714 | const char *name) | |
3715 | { | |
3716 | LIBBPF_OPTS(bpf_object_open_opts, opts, | |
3717 | .object_name = name, | |
3718 | /* wrong default, but backwards-compatible */ | |
3719 | .relaxed_maps = true, | |
3720 | ); | |
3721 | ||
3722 | /* returning NULL is wrong, but backwards-compatible */ | |
3723 | if (!obj_buf || obj_buf_sz == 0) | |
3724 | return NULL; | |
6c956392 | 3725 | |
2ce8450e | 3726 | return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts); |
1a5e3fb1 WN |
3727 | } |
3728 | ||
52d3352e WN |
3729 | int bpf_object__unload(struct bpf_object *obj) |
3730 | { | |
3731 | size_t i; | |
3732 | ||
3733 | if (!obj) | |
3734 | return -EINVAL; | |
3735 | ||
9d759a9b WN |
3736 | for (i = 0; i < obj->nr_maps; i++) |
3737 | zclose(obj->maps[i].fd); | |
52d3352e | 3738 | |
55cffde2 WN |
3739 | for (i = 0; i < obj->nr_programs; i++) |
3740 | bpf_program__unload(&obj->programs[i]); | |
3741 | ||
52d3352e WN |
3742 | return 0; |
3743 | } | |
3744 | ||
60276f98 | 3745 | int bpf_object__load_xattr(struct bpf_object_load_attr *attr) |
52d3352e | 3746 | { |
60276f98 | 3747 | struct bpf_object *obj; |
6371ca3b WN |
3748 | int err; |
3749 | ||
60276f98 QM |
3750 | if (!attr) |
3751 | return -EINVAL; | |
3752 | obj = attr->obj; | |
52d3352e WN |
3753 | if (!obj) |
3754 | return -EINVAL; | |
3755 | ||
3756 | if (obj->loaded) { | |
be18010e | 3757 | pr_warn("object should not be loaded twice\n"); |
52d3352e WN |
3758 | return -EINVAL; |
3759 | } | |
3760 | ||
3761 | obj->loaded = true; | |
6371ca3b WN |
3762 | |
3763 | CHECK_ERR(bpf_object__create_maps(obj), err, out); | |
ddc7c304 | 3764 | CHECK_ERR(bpf_object__relocate(obj, attr->target_btf_path), err, out); |
60276f98 | 3765 | CHECK_ERR(bpf_object__load_progs(obj, attr->log_level), err, out); |
52d3352e WN |
3766 | |
3767 | return 0; | |
3768 | out: | |
3769 | bpf_object__unload(obj); | |
be18010e | 3770 | pr_warn("failed to load object '%s'\n", obj->path); |
6371ca3b | 3771 | return err; |
52d3352e WN |
3772 | } |
3773 | ||
60276f98 QM |
3774 | int bpf_object__load(struct bpf_object *obj) |
3775 | { | |
3776 | struct bpf_object_load_attr attr = { | |
3777 | .obj = obj, | |
3778 | }; | |
3779 | ||
3780 | return bpf_object__load_xattr(&attr); | |
3781 | } | |
3782 | ||
f367540c JS |
3783 | static int check_path(const char *path) |
3784 | { | |
1ce6a9fc | 3785 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
3786 | struct statfs st_fs; |
3787 | char *dname, *dir; | |
3788 | int err = 0; | |
3789 | ||
3790 | if (path == NULL) | |
3791 | return -EINVAL; | |
3792 | ||
3793 | dname = strdup(path); | |
3794 | if (dname == NULL) | |
3795 | return -ENOMEM; | |
3796 | ||
3797 | dir = dirname(dname); | |
3798 | if (statfs(dir, &st_fs)) { | |
24d6a808 | 3799 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
be18010e | 3800 | pr_warn("failed to statfs %s: %s\n", dir, cp); |
f367540c JS |
3801 | err = -errno; |
3802 | } | |
3803 | free(dname); | |
3804 | ||
3805 | if (!err && st_fs.f_type != BPF_FS_MAGIC) { | |
be18010e | 3806 | pr_warn("specified path %s is not on BPF FS\n", path); |
f367540c JS |
3807 | err = -EINVAL; |
3808 | } | |
3809 | ||
3810 | return err; | |
3811 | } | |
3812 | ||
3813 | int bpf_program__pin_instance(struct bpf_program *prog, const char *path, | |
3814 | int instance) | |
3815 | { | |
1ce6a9fc | 3816 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
3817 | int err; |
3818 | ||
3819 | err = check_path(path); | |
3820 | if (err) | |
3821 | return err; | |
3822 | ||
3823 | if (prog == NULL) { | |
be18010e | 3824 | pr_warn("invalid program pointer\n"); |
f367540c JS |
3825 | return -EINVAL; |
3826 | } | |
3827 | ||
3828 | if (instance < 0 || instance >= prog->instances.nr) { | |
be18010e KW |
3829 | pr_warn("invalid prog instance %d of prog %s (max %d)\n", |
3830 | instance, prog->section_name, prog->instances.nr); | |
f367540c JS |
3831 | return -EINVAL; |
3832 | } | |
3833 | ||
3834 | if (bpf_obj_pin(prog->instances.fds[instance], path)) { | |
24d6a808 | 3835 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
be18010e | 3836 | pr_warn("failed to pin program: %s\n", cp); |
f367540c JS |
3837 | return -errno; |
3838 | } | |
3839 | pr_debug("pinned program '%s'\n", path); | |
3840 | ||
3841 | return 0; | |
3842 | } | |
3843 | ||
0c19a9fb SF |
3844 | int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, |
3845 | int instance) | |
3846 | { | |
3847 | int err; | |
3848 | ||
3849 | err = check_path(path); | |
3850 | if (err) | |
3851 | return err; | |
3852 | ||
3853 | if (prog == NULL) { | |
be18010e | 3854 | pr_warn("invalid program pointer\n"); |
0c19a9fb SF |
3855 | return -EINVAL; |
3856 | } | |
3857 | ||
3858 | if (instance < 0 || instance >= prog->instances.nr) { | |
be18010e KW |
3859 | pr_warn("invalid prog instance %d of prog %s (max %d)\n", |
3860 | instance, prog->section_name, prog->instances.nr); | |
0c19a9fb SF |
3861 | return -EINVAL; |
3862 | } | |
3863 | ||
3864 | err = unlink(path); | |
3865 | if (err != 0) | |
3866 | return -errno; | |
3867 | pr_debug("unpinned program '%s'\n", path); | |
3868 | ||
3869 | return 0; | |
3870 | } | |
3871 | ||
f367540c JS |
3872 | static int make_dir(const char *path) |
3873 | { | |
1ce6a9fc | 3874 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
3875 | int err = 0; |
3876 | ||
3877 | if (mkdir(path, 0700) && errno != EEXIST) | |
3878 | err = -errno; | |
3879 | ||
1ce6a9fc | 3880 | if (err) { |
24d6a808 | 3881 | cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); |
be18010e | 3882 | pr_warn("failed to mkdir %s: %s\n", path, cp); |
1ce6a9fc | 3883 | } |
f367540c JS |
3884 | return err; |
3885 | } | |
3886 | ||
3887 | int bpf_program__pin(struct bpf_program *prog, const char *path) | |
3888 | { | |
3889 | int i, err; | |
3890 | ||
3891 | err = check_path(path); | |
3892 | if (err) | |
3893 | return err; | |
3894 | ||
3895 | if (prog == NULL) { | |
be18010e | 3896 | pr_warn("invalid program pointer\n"); |
f367540c JS |
3897 | return -EINVAL; |
3898 | } | |
3899 | ||
3900 | if (prog->instances.nr <= 0) { | |
be18010e | 3901 | pr_warn("no instances of prog %s to pin\n", |
f367540c JS |
3902 | prog->section_name); |
3903 | return -EINVAL; | |
3904 | } | |
3905 | ||
fd734c5c SF |
3906 | if (prog->instances.nr == 1) { |
3907 | /* don't create subdirs when pinning single instance */ | |
3908 | return bpf_program__pin_instance(prog, path, 0); | |
3909 | } | |
3910 | ||
f367540c JS |
3911 | err = make_dir(path); |
3912 | if (err) | |
3913 | return err; | |
3914 | ||
0c19a9fb SF |
3915 | for (i = 0; i < prog->instances.nr; i++) { |
3916 | char buf[PATH_MAX]; | |
3917 | int len; | |
3918 | ||
3919 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
3920 | if (len < 0) { | |
3921 | err = -EINVAL; | |
3922 | goto err_unpin; | |
3923 | } else if (len >= PATH_MAX) { | |
3924 | err = -ENAMETOOLONG; | |
3925 | goto err_unpin; | |
3926 | } | |
3927 | ||
3928 | err = bpf_program__pin_instance(prog, buf, i); | |
3929 | if (err) | |
3930 | goto err_unpin; | |
3931 | } | |
3932 | ||
3933 | return 0; | |
3934 | ||
3935 | err_unpin: | |
3936 | for (i = i - 1; i >= 0; i--) { | |
3937 | char buf[PATH_MAX]; | |
3938 | int len; | |
3939 | ||
3940 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
3941 | if (len < 0) | |
3942 | continue; | |
3943 | else if (len >= PATH_MAX) | |
3944 | continue; | |
3945 | ||
3946 | bpf_program__unpin_instance(prog, buf, i); | |
3947 | } | |
3948 | ||
3949 | rmdir(path); | |
3950 | ||
3951 | return err; | |
3952 | } | |
3953 | ||
3954 | int bpf_program__unpin(struct bpf_program *prog, const char *path) | |
3955 | { | |
3956 | int i, err; | |
3957 | ||
3958 | err = check_path(path); | |
3959 | if (err) | |
3960 | return err; | |
3961 | ||
3962 | if (prog == NULL) { | |
be18010e | 3963 | pr_warn("invalid program pointer\n"); |
0c19a9fb SF |
3964 | return -EINVAL; |
3965 | } | |
3966 | ||
3967 | if (prog->instances.nr <= 0) { | |
be18010e | 3968 | pr_warn("no instances of prog %s to pin\n", |
0c19a9fb SF |
3969 | prog->section_name); |
3970 | return -EINVAL; | |
fd734c5c SF |
3971 | } |
3972 | ||
3973 | if (prog->instances.nr == 1) { | |
3974 | /* don't create subdirs when pinning single instance */ | |
3975 | return bpf_program__unpin_instance(prog, path, 0); | |
0c19a9fb SF |
3976 | } |
3977 | ||
f367540c JS |
3978 | for (i = 0; i < prog->instances.nr; i++) { |
3979 | char buf[PATH_MAX]; | |
3980 | int len; | |
3981 | ||
3982 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
3983 | if (len < 0) | |
3984 | return -EINVAL; | |
3985 | else if (len >= PATH_MAX) | |
3986 | return -ENAMETOOLONG; | |
3987 | ||
0c19a9fb | 3988 | err = bpf_program__unpin_instance(prog, buf, i); |
f367540c JS |
3989 | if (err) |
3990 | return err; | |
3991 | } | |
3992 | ||
0c19a9fb SF |
3993 | err = rmdir(path); |
3994 | if (err) | |
3995 | return -errno; | |
3996 | ||
f367540c JS |
3997 | return 0; |
3998 | } | |
3999 | ||
b6989f35 JS |
4000 | int bpf_map__pin(struct bpf_map *map, const char *path) |
4001 | { | |
1ce6a9fc | 4002 | char *cp, errmsg[STRERR_BUFSIZE]; |
b6989f35 JS |
4003 | int err; |
4004 | ||
4005 | err = check_path(path); | |
4006 | if (err) | |
4007 | return err; | |
4008 | ||
4009 | if (map == NULL) { | |
be18010e | 4010 | pr_warn("invalid map pointer\n"); |
b6989f35 JS |
4011 | return -EINVAL; |
4012 | } | |
4013 | ||
4014 | if (bpf_obj_pin(map->fd, path)) { | |
24d6a808 | 4015 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
be18010e | 4016 | pr_warn("failed to pin map: %s\n", cp); |
b6989f35 JS |
4017 | return -errno; |
4018 | } | |
4019 | ||
4020 | pr_debug("pinned map '%s'\n", path); | |
0c19a9fb | 4021 | |
b6989f35 JS |
4022 | return 0; |
4023 | } | |
4024 | ||
0c19a9fb SF |
4025 | int bpf_map__unpin(struct bpf_map *map, const char *path) |
4026 | { | |
4027 | int err; | |
4028 | ||
4029 | err = check_path(path); | |
4030 | if (err) | |
4031 | return err; | |
4032 | ||
4033 | if (map == NULL) { | |
be18010e | 4034 | pr_warn("invalid map pointer\n"); |
0c19a9fb SF |
4035 | return -EINVAL; |
4036 | } | |
4037 | ||
4038 | err = unlink(path); | |
4039 | if (err != 0) | |
4040 | return -errno; | |
4041 | pr_debug("unpinned map '%s'\n", path); | |
4042 | ||
4043 | return 0; | |
4044 | } | |
4045 | ||
4046 | int bpf_object__pin_maps(struct bpf_object *obj, const char *path) | |
d5148d85 | 4047 | { |
d5148d85 JS |
4048 | struct bpf_map *map; |
4049 | int err; | |
4050 | ||
4051 | if (!obj) | |
4052 | return -ENOENT; | |
4053 | ||
4054 | if (!obj->loaded) { | |
be18010e | 4055 | pr_warn("object not yet loaded; load it first\n"); |
d5148d85 JS |
4056 | return -ENOENT; |
4057 | } | |
4058 | ||
4059 | err = make_dir(path); | |
4060 | if (err) | |
4061 | return err; | |
4062 | ||
f74a53d9 | 4063 | bpf_object__for_each_map(map, obj) { |
0c19a9fb SF |
4064 | char buf[PATH_MAX]; |
4065 | int len; | |
4066 | ||
4067 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
4068 | bpf_map__name(map)); | |
4069 | if (len < 0) { | |
4070 | err = -EINVAL; | |
4071 | goto err_unpin_maps; | |
4072 | } else if (len >= PATH_MAX) { | |
4073 | err = -ENAMETOOLONG; | |
4074 | goto err_unpin_maps; | |
4075 | } | |
4076 | ||
4077 | err = bpf_map__pin(map, buf); | |
4078 | if (err) | |
4079 | goto err_unpin_maps; | |
4080 | } | |
4081 | ||
4082 | return 0; | |
4083 | ||
4084 | err_unpin_maps: | |
4085 | while ((map = bpf_map__prev(map, obj))) { | |
4086 | char buf[PATH_MAX]; | |
4087 | int len; | |
4088 | ||
4089 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
4090 | bpf_map__name(map)); | |
4091 | if (len < 0) | |
4092 | continue; | |
4093 | else if (len >= PATH_MAX) | |
4094 | continue; | |
4095 | ||
4096 | bpf_map__unpin(map, buf); | |
4097 | } | |
4098 | ||
4099 | return err; | |
4100 | } | |
4101 | ||
4102 | int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) | |
4103 | { | |
4104 | struct bpf_map *map; | |
4105 | int err; | |
4106 | ||
4107 | if (!obj) | |
4108 | return -ENOENT; | |
4109 | ||
f74a53d9 | 4110 | bpf_object__for_each_map(map, obj) { |
d5148d85 JS |
4111 | char buf[PATH_MAX]; |
4112 | int len; | |
4113 | ||
4114 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
4115 | bpf_map__name(map)); | |
4116 | if (len < 0) | |
4117 | return -EINVAL; | |
4118 | else if (len >= PATH_MAX) | |
4119 | return -ENAMETOOLONG; | |
4120 | ||
0c19a9fb | 4121 | err = bpf_map__unpin(map, buf); |
d5148d85 JS |
4122 | if (err) |
4123 | return err; | |
4124 | } | |
4125 | ||
0c19a9fb SF |
4126 | return 0; |
4127 | } | |
4128 | ||
4129 | int bpf_object__pin_programs(struct bpf_object *obj, const char *path) | |
4130 | { | |
4131 | struct bpf_program *prog; | |
4132 | int err; | |
4133 | ||
4134 | if (!obj) | |
4135 | return -ENOENT; | |
4136 | ||
4137 | if (!obj->loaded) { | |
be18010e | 4138 | pr_warn("object not yet loaded; load it first\n"); |
0c19a9fb SF |
4139 | return -ENOENT; |
4140 | } | |
4141 | ||
4142 | err = make_dir(path); | |
4143 | if (err) | |
4144 | return err; | |
4145 | ||
4146 | bpf_object__for_each_program(prog, obj) { | |
4147 | char buf[PATH_MAX]; | |
4148 | int len; | |
4149 | ||
4150 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 4151 | prog->pin_name); |
0c19a9fb SF |
4152 | if (len < 0) { |
4153 | err = -EINVAL; | |
4154 | goto err_unpin_programs; | |
4155 | } else if (len >= PATH_MAX) { | |
4156 | err = -ENAMETOOLONG; | |
4157 | goto err_unpin_programs; | |
4158 | } | |
4159 | ||
4160 | err = bpf_program__pin(prog, buf); | |
4161 | if (err) | |
4162 | goto err_unpin_programs; | |
4163 | } | |
4164 | ||
4165 | return 0; | |
4166 | ||
4167 | err_unpin_programs: | |
4168 | while ((prog = bpf_program__prev(prog, obj))) { | |
4169 | char buf[PATH_MAX]; | |
4170 | int len; | |
4171 | ||
4172 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 4173 | prog->pin_name); |
0c19a9fb SF |
4174 | if (len < 0) |
4175 | continue; | |
4176 | else if (len >= PATH_MAX) | |
4177 | continue; | |
4178 | ||
4179 | bpf_program__unpin(prog, buf); | |
4180 | } | |
4181 | ||
4182 | return err; | |
4183 | } | |
4184 | ||
4185 | int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) | |
4186 | { | |
4187 | struct bpf_program *prog; | |
4188 | int err; | |
4189 | ||
4190 | if (!obj) | |
4191 | return -ENOENT; | |
4192 | ||
d5148d85 JS |
4193 | bpf_object__for_each_program(prog, obj) { |
4194 | char buf[PATH_MAX]; | |
4195 | int len; | |
4196 | ||
4197 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 4198 | prog->pin_name); |
d5148d85 JS |
4199 | if (len < 0) |
4200 | return -EINVAL; | |
4201 | else if (len >= PATH_MAX) | |
4202 | return -ENAMETOOLONG; | |
4203 | ||
0c19a9fb | 4204 | err = bpf_program__unpin(prog, buf); |
d5148d85 JS |
4205 | if (err) |
4206 | return err; | |
4207 | } | |
4208 | ||
4209 | return 0; | |
4210 | } | |
4211 | ||
0c19a9fb SF |
4212 | int bpf_object__pin(struct bpf_object *obj, const char *path) |
4213 | { | |
4214 | int err; | |
4215 | ||
4216 | err = bpf_object__pin_maps(obj, path); | |
4217 | if (err) | |
4218 | return err; | |
4219 | ||
4220 | err = bpf_object__pin_programs(obj, path); | |
4221 | if (err) { | |
4222 | bpf_object__unpin_maps(obj, path); | |
4223 | return err; | |
4224 | } | |
4225 | ||
4226 | return 0; | |
4227 | } | |
4228 | ||
1a5e3fb1 WN |
4229 | void bpf_object__close(struct bpf_object *obj) |
4230 | { | |
a5b8bd47 WN |
4231 | size_t i; |
4232 | ||
1a5e3fb1 WN |
4233 | if (!obj) |
4234 | return; | |
4235 | ||
10931d24 WN |
4236 | if (obj->clear_priv) |
4237 | obj->clear_priv(obj, obj->priv); | |
4238 | ||
1a5e3fb1 | 4239 | bpf_object__elf_finish(obj); |
52d3352e | 4240 | bpf_object__unload(obj); |
8a138aed | 4241 | btf__free(obj->btf); |
2993e051 | 4242 | btf_ext__free(obj->btf_ext); |
1a5e3fb1 | 4243 | |
9d759a9b | 4244 | for (i = 0; i < obj->nr_maps; i++) { |
561bbcca | 4245 | zfree(&obj->maps[i].name); |
9d759a9b WN |
4246 | if (obj->maps[i].clear_priv) |
4247 | obj->maps[i].clear_priv(&obj->maps[i], | |
4248 | obj->maps[i].priv); | |
4249 | obj->maps[i].priv = NULL; | |
4250 | obj->maps[i].clear_priv = NULL; | |
4251 | } | |
d859900c DB |
4252 | |
4253 | zfree(&obj->sections.rodata); | |
4254 | zfree(&obj->sections.data); | |
9d759a9b WN |
4255 | zfree(&obj->maps); |
4256 | obj->nr_maps = 0; | |
a5b8bd47 WN |
4257 | |
4258 | if (obj->programs && obj->nr_programs) { | |
4259 | for (i = 0; i < obj->nr_programs; i++) | |
4260 | bpf_program__exit(&obj->programs[i]); | |
4261 | } | |
4262 | zfree(&obj->programs); | |
4263 | ||
9a208eff | 4264 | list_del(&obj->list); |
1a5e3fb1 WN |
4265 | free(obj); |
4266 | } | |
aa9b1ac3 | 4267 | |
9a208eff WN |
4268 | struct bpf_object * |
4269 | bpf_object__next(struct bpf_object *prev) | |
4270 | { | |
4271 | struct bpf_object *next; | |
4272 | ||
4273 | if (!prev) | |
4274 | next = list_first_entry(&bpf_objects_list, | |
4275 | struct bpf_object, | |
4276 | list); | |
4277 | else | |
4278 | next = list_next_entry(prev, list); | |
4279 | ||
4280 | /* Empty list is noticed here so don't need checking on entry. */ | |
4281 | if (&next->list == &bpf_objects_list) | |
4282 | return NULL; | |
4283 | ||
4284 | return next; | |
4285 | } | |
4286 | ||
a324aae3 | 4287 | const char *bpf_object__name(const struct bpf_object *obj) |
acf860ae | 4288 | { |
c9e4c301 | 4289 | return obj ? obj->name : ERR_PTR(-EINVAL); |
acf860ae WN |
4290 | } |
4291 | ||
a324aae3 | 4292 | unsigned int bpf_object__kversion(const struct bpf_object *obj) |
45825d8a | 4293 | { |
a7fe0450 | 4294 | return obj ? obj->kern_version : 0; |
45825d8a WN |
4295 | } |
4296 | ||
a324aae3 | 4297 | struct btf *bpf_object__btf(const struct bpf_object *obj) |
789f6bab AI |
4298 | { |
4299 | return obj ? obj->btf : NULL; | |
4300 | } | |
4301 | ||
8a138aed MKL |
4302 | int bpf_object__btf_fd(const struct bpf_object *obj) |
4303 | { | |
4304 | return obj->btf ? btf__fd(obj->btf) : -1; | |
4305 | } | |
4306 | ||
10931d24 WN |
4307 | int bpf_object__set_priv(struct bpf_object *obj, void *priv, |
4308 | bpf_object_clear_priv_t clear_priv) | |
4309 | { | |
4310 | if (obj->priv && obj->clear_priv) | |
4311 | obj->clear_priv(obj, obj->priv); | |
4312 | ||
4313 | obj->priv = priv; | |
4314 | obj->clear_priv = clear_priv; | |
4315 | return 0; | |
4316 | } | |
4317 | ||
a324aae3 | 4318 | void *bpf_object__priv(const struct bpf_object *obj) |
10931d24 WN |
4319 | { |
4320 | return obj ? obj->priv : ERR_PTR(-EINVAL); | |
4321 | } | |
4322 | ||
eac7d845 | 4323 | static struct bpf_program * |
a324aae3 AN |
4324 | __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, |
4325 | bool forward) | |
aa9b1ac3 | 4326 | { |
a83d6e76 | 4327 | size_t nr_programs = obj->nr_programs; |
0c19a9fb | 4328 | ssize_t idx; |
aa9b1ac3 | 4329 | |
a83d6e76 | 4330 | if (!nr_programs) |
aa9b1ac3 | 4331 | return NULL; |
aa9b1ac3 | 4332 | |
a83d6e76 MKL |
4333 | if (!p) |
4334 | /* Iter from the beginning */ | |
4335 | return forward ? &obj->programs[0] : | |
4336 | &obj->programs[nr_programs - 1]; | |
4337 | ||
0c19a9fb | 4338 | if (p->obj != obj) { |
be18010e | 4339 | pr_warn("error: program handler doesn't match object\n"); |
aa9b1ac3 WN |
4340 | return NULL; |
4341 | } | |
4342 | ||
a83d6e76 | 4343 | idx = (p - obj->programs) + (forward ? 1 : -1); |
0c19a9fb | 4344 | if (idx >= obj->nr_programs || idx < 0) |
aa9b1ac3 WN |
4345 | return NULL; |
4346 | return &obj->programs[idx]; | |
4347 | } | |
4348 | ||
eac7d845 | 4349 | struct bpf_program * |
a324aae3 | 4350 | bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) |
eac7d845 JK |
4351 | { |
4352 | struct bpf_program *prog = prev; | |
4353 | ||
4354 | do { | |
a83d6e76 | 4355 | prog = __bpf_program__iter(prog, obj, true); |
0c19a9fb SF |
4356 | } while (prog && bpf_program__is_function_storage(prog, obj)); |
4357 | ||
4358 | return prog; | |
4359 | } | |
4360 | ||
4361 | struct bpf_program * | |
a324aae3 | 4362 | bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) |
0c19a9fb SF |
4363 | { |
4364 | struct bpf_program *prog = next; | |
4365 | ||
0c19a9fb | 4366 | do { |
a83d6e76 | 4367 | prog = __bpf_program__iter(prog, obj, false); |
eac7d845 JK |
4368 | } while (prog && bpf_program__is_function_storage(prog, obj)); |
4369 | ||
4370 | return prog; | |
4371 | } | |
4372 | ||
edb13ed4 ACM |
4373 | int bpf_program__set_priv(struct bpf_program *prog, void *priv, |
4374 | bpf_program_clear_priv_t clear_priv) | |
aa9b1ac3 WN |
4375 | { |
4376 | if (prog->priv && prog->clear_priv) | |
4377 | prog->clear_priv(prog, prog->priv); | |
4378 | ||
4379 | prog->priv = priv; | |
4380 | prog->clear_priv = clear_priv; | |
4381 | return 0; | |
4382 | } | |
4383 | ||
a324aae3 | 4384 | void *bpf_program__priv(const struct bpf_program *prog) |
aa9b1ac3 | 4385 | { |
be834ffb | 4386 | return prog ? prog->priv : ERR_PTR(-EINVAL); |
aa9b1ac3 WN |
4387 | } |
4388 | ||
9aba3613 JK |
4389 | void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) |
4390 | { | |
4391 | prog->prog_ifindex = ifindex; | |
4392 | } | |
4393 | ||
a324aae3 | 4394 | const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
4395 | { |
4396 | const char *title; | |
4397 | ||
4398 | title = prog->section_name; | |
715f8db9 | 4399 | if (needs_copy) { |
aa9b1ac3 WN |
4400 | title = strdup(title); |
4401 | if (!title) { | |
be18010e | 4402 | pr_warn("failed to strdup program title\n"); |
6371ca3b | 4403 | return ERR_PTR(-ENOMEM); |
aa9b1ac3 WN |
4404 | } |
4405 | } | |
4406 | ||
4407 | return title; | |
4408 | } | |
4409 | ||
a324aae3 | 4410 | int bpf_program__fd(const struct bpf_program *prog) |
aa9b1ac3 | 4411 | { |
b580563e WN |
4412 | return bpf_program__nth_fd(prog, 0); |
4413 | } | |
4414 | ||
4415 | int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, | |
4416 | bpf_program_prep_t prep) | |
4417 | { | |
4418 | int *instances_fds; | |
4419 | ||
4420 | if (nr_instances <= 0 || !prep) | |
4421 | return -EINVAL; | |
4422 | ||
4423 | if (prog->instances.nr > 0 || prog->instances.fds) { | |
be18010e | 4424 | pr_warn("Can't set pre-processor after loading\n"); |
b580563e WN |
4425 | return -EINVAL; |
4426 | } | |
4427 | ||
4428 | instances_fds = malloc(sizeof(int) * nr_instances); | |
4429 | if (!instances_fds) { | |
be18010e | 4430 | pr_warn("alloc memory failed for fds\n"); |
b580563e WN |
4431 | return -ENOMEM; |
4432 | } | |
4433 | ||
4434 | /* fill all fd with -1 */ | |
4435 | memset(instances_fds, -1, sizeof(int) * nr_instances); | |
4436 | ||
4437 | prog->instances.nr = nr_instances; | |
4438 | prog->instances.fds = instances_fds; | |
4439 | prog->preprocessor = prep; | |
4440 | return 0; | |
4441 | } | |
4442 | ||
a324aae3 | 4443 | int bpf_program__nth_fd(const struct bpf_program *prog, int n) |
b580563e WN |
4444 | { |
4445 | int fd; | |
4446 | ||
1e960043 JK |
4447 | if (!prog) |
4448 | return -EINVAL; | |
4449 | ||
b580563e | 4450 | if (n >= prog->instances.nr || n < 0) { |
be18010e KW |
4451 | pr_warn("Can't get the %dth fd from program %s: only %d instances\n", |
4452 | n, prog->section_name, prog->instances.nr); | |
b580563e WN |
4453 | return -EINVAL; |
4454 | } | |
4455 | ||
4456 | fd = prog->instances.fds[n]; | |
4457 | if (fd < 0) { | |
be18010e KW |
4458 | pr_warn("%dth instance of program '%s' is invalid\n", |
4459 | n, prog->section_name); | |
b580563e WN |
4460 | return -ENOENT; |
4461 | } | |
4462 | ||
4463 | return fd; | |
aa9b1ac3 | 4464 | } |
9d759a9b | 4465 | |
f1eead9e AN |
4466 | enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog) |
4467 | { | |
4468 | return prog->type; | |
4469 | } | |
4470 | ||
dd26b7f5 | 4471 | void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) |
5f44e4c8 WN |
4472 | { |
4473 | prog->type = type; | |
4474 | } | |
4475 | ||
a324aae3 | 4476 | static bool bpf_program__is_type(const struct bpf_program *prog, |
5f44e4c8 WN |
4477 | enum bpf_prog_type type) |
4478 | { | |
4479 | return prog ? (prog->type == type) : false; | |
4480 | } | |
4481 | ||
a324aae3 AN |
4482 | #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ |
4483 | int bpf_program__set_##NAME(struct bpf_program *prog) \ | |
4484 | { \ | |
4485 | if (!prog) \ | |
4486 | return -EINVAL; \ | |
4487 | bpf_program__set_type(prog, TYPE); \ | |
4488 | return 0; \ | |
4489 | } \ | |
4490 | \ | |
4491 | bool bpf_program__is_##NAME(const struct bpf_program *prog) \ | |
4492 | { \ | |
4493 | return bpf_program__is_type(prog, TYPE); \ | |
4494 | } \ | |
ed794073 | 4495 | |
7803ba73 | 4496 | BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); |
ed794073 | 4497 | BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); |
7803ba73 JS |
4498 | BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); |
4499 | BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); | |
ed794073 | 4500 | BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); |
e14c93fd | 4501 | BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); |
7803ba73 JS |
4502 | BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); |
4503 | BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); | |
5f44e4c8 | 4504 | |
f1eead9e AN |
4505 | enum bpf_attach_type |
4506 | bpf_program__get_expected_attach_type(struct bpf_program *prog) | |
4507 | { | |
4508 | return prog->expected_attach_type; | |
4509 | } | |
4510 | ||
16962b24 JF |
4511 | void bpf_program__set_expected_attach_type(struct bpf_program *prog, |
4512 | enum bpf_attach_type type) | |
d7be143b AI |
4513 | { |
4514 | prog->expected_attach_type = type; | |
4515 | } | |
4516 | ||
f75a697e AS |
4517 | #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, btf, atype) \ |
4518 | { string, sizeof(string) - 1, ptype, eatype, is_attachable, btf, atype } | |
d7be143b | 4519 | |
956b620f | 4520 | /* Programs that can NOT be attached. */ |
f75a697e | 4521 | #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0) |
d7be143b | 4522 | |
956b620f AI |
4523 | /* Programs that can be attached. */ |
4524 | #define BPF_APROG_SEC(string, ptype, atype) \ | |
f75a697e | 4525 | BPF_PROG_SEC_IMPL(string, ptype, 0, 1, 0, atype) |
81efee75 | 4526 | |
956b620f AI |
4527 | /* Programs that must specify expected attach type at load time. */ |
4528 | #define BPF_EAPROG_SEC(string, ptype, eatype) \ | |
f75a697e AS |
4529 | BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, 0, eatype) |
4530 | ||
4531 | /* Programs that use BTF to identify attach point */ | |
4532 | #define BPF_PROG_BTF(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 1, 0) | |
956b620f AI |
4533 | |
4534 | /* Programs that can be attached but attach type can't be identified by section | |
4535 | * name. Kept for backward compatibility. | |
4536 | */ | |
4537 | #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) | |
e50b0a6f | 4538 | |
583c9009 RG |
4539 | static const struct { |
4540 | const char *sec; | |
4541 | size_t len; | |
4542 | enum bpf_prog_type prog_type; | |
d7be143b | 4543 | enum bpf_attach_type expected_attach_type; |
f75a697e AS |
4544 | bool is_attachable; |
4545 | bool is_attach_btf; | |
956b620f | 4546 | enum bpf_attach_type attach_type; |
583c9009 | 4547 | } section_names[] = { |
956b620f AI |
4548 | BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), |
4549 | BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), | |
32dff6db | 4550 | BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE), |
956b620f | 4551 | BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), |
32dff6db | 4552 | BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE), |
956b620f AI |
4553 | BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), |
4554 | BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), | |
4555 | BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), | |
32dff6db | 4556 | BPF_PROG_SEC("tp/", BPF_PROG_TYPE_TRACEPOINT), |
956b620f | 4557 | BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), |
32dff6db | 4558 | BPF_PROG_SEC("raw_tp/", BPF_PROG_TYPE_RAW_TRACEPOINT), |
f75a697e | 4559 | BPF_PROG_BTF("tp_btf/", BPF_PROG_TYPE_RAW_TRACEPOINT), |
956b620f AI |
4560 | BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), |
4561 | BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), | |
4562 | BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), | |
4563 | BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), | |
4564 | BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), | |
4565 | BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), | |
bafa7afe AI |
4566 | BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, |
4567 | BPF_CGROUP_INET_INGRESS), | |
4568 | BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, | |
4569 | BPF_CGROUP_INET_EGRESS), | |
956b620f AI |
4570 | BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), |
4571 | BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, | |
4572 | BPF_CGROUP_INET_SOCK_CREATE), | |
4573 | BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, | |
4574 | BPF_CGROUP_INET4_POST_BIND), | |
4575 | BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, | |
4576 | BPF_CGROUP_INET6_POST_BIND), | |
4577 | BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, | |
4578 | BPF_CGROUP_DEVICE), | |
4579 | BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, | |
4580 | BPF_CGROUP_SOCK_OPS), | |
c6f6851b AI |
4581 | BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, |
4582 | BPF_SK_SKB_STREAM_PARSER), | |
4583 | BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, | |
4584 | BPF_SK_SKB_STREAM_VERDICT), | |
956b620f AI |
4585 | BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), |
4586 | BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, | |
4587 | BPF_SK_MSG_VERDICT), | |
4588 | BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, | |
4589 | BPF_LIRC_MODE2), | |
4590 | BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, | |
4591 | BPF_FLOW_DISSECTOR), | |
4592 | BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4593 | BPF_CGROUP_INET4_BIND), | |
4594 | BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4595 | BPF_CGROUP_INET6_BIND), | |
4596 | BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4597 | BPF_CGROUP_INET4_CONNECT), | |
4598 | BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4599 | BPF_CGROUP_INET6_CONNECT), | |
4600 | BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4601 | BPF_CGROUP_UDP4_SENDMSG), | |
4602 | BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4603 | BPF_CGROUP_UDP6_SENDMSG), | |
9bb59ac1 DB |
4604 | BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, |
4605 | BPF_CGROUP_UDP4_RECVMSG), | |
4606 | BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
4607 | BPF_CGROUP_UDP6_RECVMSG), | |
063cc9f0 AI |
4608 | BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, |
4609 | BPF_CGROUP_SYSCTL), | |
4cdbfb59 SF |
4610 | BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, |
4611 | BPF_CGROUP_GETSOCKOPT), | |
4612 | BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, | |
4613 | BPF_CGROUP_SETSOCKOPT), | |
583c9009 | 4614 | }; |
d7be143b | 4615 | |
956b620f | 4616 | #undef BPF_PROG_SEC_IMPL |
583c9009 | 4617 | #undef BPF_PROG_SEC |
956b620f AI |
4618 | #undef BPF_APROG_SEC |
4619 | #undef BPF_EAPROG_SEC | |
4620 | #undef BPF_APROG_COMPAT | |
583c9009 | 4621 | |
c76e4c22 TS |
4622 | #define MAX_TYPE_NAME_SIZE 32 |
4623 | ||
4624 | static char *libbpf_get_type_names(bool attach_type) | |
4625 | { | |
4626 | int i, len = ARRAY_SIZE(section_names) * MAX_TYPE_NAME_SIZE; | |
4627 | char *buf; | |
4628 | ||
4629 | buf = malloc(len); | |
4630 | if (!buf) | |
4631 | return NULL; | |
4632 | ||
4633 | buf[0] = '\0'; | |
4634 | /* Forge string buf with all available names */ | |
4635 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { | |
4636 | if (attach_type && !section_names[i].is_attachable) | |
4637 | continue; | |
4638 | ||
4639 | if (strlen(buf) + strlen(section_names[i].sec) + 2 > len) { | |
4640 | free(buf); | |
4641 | return NULL; | |
4642 | } | |
4643 | strcat(buf, " "); | |
4644 | strcat(buf, section_names[i].sec); | |
4645 | } | |
4646 | ||
4647 | return buf; | |
4648 | } | |
4649 | ||
b60df2a0 JK |
4650 | int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, |
4651 | enum bpf_attach_type *expected_attach_type) | |
583c9009 | 4652 | { |
c76e4c22 | 4653 | char *type_names; |
583c9009 RG |
4654 | int i; |
4655 | ||
b60df2a0 JK |
4656 | if (!name) |
4657 | return -EINVAL; | |
583c9009 | 4658 | |
b60df2a0 JK |
4659 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { |
4660 | if (strncmp(name, section_names[i].sec, section_names[i].len)) | |
4661 | continue; | |
4662 | *prog_type = section_names[i].prog_type; | |
4663 | *expected_attach_type = section_names[i].expected_attach_type; | |
f75a697e AS |
4664 | if (section_names[i].is_attach_btf) { |
4665 | struct btf *btf = bpf_core_find_kernel_btf(); | |
4666 | char raw_tp_btf_name[128] = "btf_trace_"; | |
4667 | char *dst = raw_tp_btf_name + sizeof("btf_trace_") - 1; | |
4668 | int ret; | |
4669 | ||
4670 | if (IS_ERR(btf)) { | |
be18010e | 4671 | pr_warn("vmlinux BTF is not found\n"); |
f75a697e AS |
4672 | return -EINVAL; |
4673 | } | |
4674 | /* prepend "btf_trace_" prefix per kernel convention */ | |
4675 | strncat(dst, name + section_names[i].len, | |
4676 | sizeof(raw_tp_btf_name) - (dst - raw_tp_btf_name)); | |
4677 | ret = btf__find_by_name(btf, raw_tp_btf_name); | |
4678 | btf__free(btf); | |
4679 | if (ret <= 0) { | |
be18010e | 4680 | pr_warn("%s is not found in vmlinux BTF\n", dst); |
f75a697e AS |
4681 | return -EINVAL; |
4682 | } | |
4683 | *expected_attach_type = ret; | |
4684 | } | |
b60df2a0 JK |
4685 | return 0; |
4686 | } | |
be18010e | 4687 | pr_warn("failed to guess program type based on ELF section name '%s'\n", name); |
c76e4c22 TS |
4688 | type_names = libbpf_get_type_names(false); |
4689 | if (type_names != NULL) { | |
4690 | pr_info("supported section(type) names are:%s\n", type_names); | |
4691 | free(type_names); | |
4692 | } | |
4693 | ||
b60df2a0 JK |
4694 | return -EINVAL; |
4695 | } | |
583c9009 | 4696 | |
956b620f AI |
4697 | int libbpf_attach_type_by_name(const char *name, |
4698 | enum bpf_attach_type *attach_type) | |
4699 | { | |
c76e4c22 | 4700 | char *type_names; |
956b620f AI |
4701 | int i; |
4702 | ||
4703 | if (!name) | |
4704 | return -EINVAL; | |
4705 | ||
4706 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { | |
4707 | if (strncmp(name, section_names[i].sec, section_names[i].len)) | |
4708 | continue; | |
36153532 | 4709 | if (!section_names[i].is_attachable) |
956b620f AI |
4710 | return -EINVAL; |
4711 | *attach_type = section_names[i].attach_type; | |
4712 | return 0; | |
4713 | } | |
be18010e | 4714 | pr_warn("failed to guess attach type based on ELF section name '%s'\n", name); |
c76e4c22 TS |
4715 | type_names = libbpf_get_type_names(true); |
4716 | if (type_names != NULL) { | |
4717 | pr_info("attachable section(type) names are:%s\n", type_names); | |
4718 | free(type_names); | |
4719 | } | |
4720 | ||
956b620f AI |
4721 | return -EINVAL; |
4722 | } | |
4723 | ||
b60df2a0 JK |
4724 | static int |
4725 | bpf_program__identify_section(struct bpf_program *prog, | |
4726 | enum bpf_prog_type *prog_type, | |
4727 | enum bpf_attach_type *expected_attach_type) | |
4728 | { | |
4729 | return libbpf_prog_type_by_name(prog->section_name, prog_type, | |
4730 | expected_attach_type); | |
583c9009 RG |
4731 | } |
4732 | ||
a324aae3 | 4733 | int bpf_map__fd(const struct bpf_map *map) |
9d759a9b | 4734 | { |
6e009e65 | 4735 | return map ? map->fd : -EINVAL; |
9d759a9b WN |
4736 | } |
4737 | ||
a324aae3 | 4738 | const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) |
9d759a9b | 4739 | { |
53897a78 | 4740 | return map ? &map->def : ERR_PTR(-EINVAL); |
9d759a9b WN |
4741 | } |
4742 | ||
a324aae3 | 4743 | const char *bpf_map__name(const struct bpf_map *map) |
561bbcca | 4744 | { |
009ad5d5 | 4745 | return map ? map->name : NULL; |
561bbcca WN |
4746 | } |
4747 | ||
5b891af7 | 4748 | __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) |
8a138aed | 4749 | { |
61746dbe | 4750 | return map ? map->btf_key_type_id : 0; |
8a138aed MKL |
4751 | } |
4752 | ||
5b891af7 | 4753 | __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) |
8a138aed | 4754 | { |
61746dbe | 4755 | return map ? map->btf_value_type_id : 0; |
8a138aed MKL |
4756 | } |
4757 | ||
edb13ed4 ACM |
4758 | int bpf_map__set_priv(struct bpf_map *map, void *priv, |
4759 | bpf_map_clear_priv_t clear_priv) | |
9d759a9b WN |
4760 | { |
4761 | if (!map) | |
4762 | return -EINVAL; | |
4763 | ||
4764 | if (map->priv) { | |
4765 | if (map->clear_priv) | |
4766 | map->clear_priv(map, map->priv); | |
4767 | } | |
4768 | ||
4769 | map->priv = priv; | |
4770 | map->clear_priv = clear_priv; | |
4771 | return 0; | |
4772 | } | |
4773 | ||
a324aae3 | 4774 | void *bpf_map__priv(const struct bpf_map *map) |
9d759a9b | 4775 | { |
b4cbfa56 | 4776 | return map ? map->priv : ERR_PTR(-EINVAL); |
9d759a9b WN |
4777 | } |
4778 | ||
a324aae3 | 4779 | bool bpf_map__is_offload_neutral(const struct bpf_map *map) |
f83fb22c JK |
4780 | { |
4781 | return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; | |
4782 | } | |
4783 | ||
a324aae3 | 4784 | bool bpf_map__is_internal(const struct bpf_map *map) |
d859900c DB |
4785 | { |
4786 | return map->libbpf_type != LIBBPF_MAP_UNSPEC; | |
4787 | } | |
4788 | ||
9aba3613 JK |
4789 | void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) |
4790 | { | |
4791 | map->map_ifindex = ifindex; | |
4792 | } | |
4793 | ||
addb9fc9 NS |
4794 | int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) |
4795 | { | |
4796 | if (!bpf_map_type__is_map_in_map(map->def.type)) { | |
be18010e | 4797 | pr_warn("error: unsupported map type\n"); |
addb9fc9 NS |
4798 | return -EINVAL; |
4799 | } | |
4800 | if (map->inner_map_fd != -1) { | |
be18010e | 4801 | pr_warn("error: inner_map_fd already specified\n"); |
addb9fc9 NS |
4802 | return -EINVAL; |
4803 | } | |
4804 | map->inner_map_fd = fd; | |
4805 | return 0; | |
4806 | } | |
4807 | ||
0c19a9fb | 4808 | static struct bpf_map * |
a324aae3 | 4809 | __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) |
9d759a9b | 4810 | { |
0c19a9fb | 4811 | ssize_t idx; |
9d759a9b WN |
4812 | struct bpf_map *s, *e; |
4813 | ||
4814 | if (!obj || !obj->maps) | |
4815 | return NULL; | |
4816 | ||
4817 | s = obj->maps; | |
4818 | e = obj->maps + obj->nr_maps; | |
4819 | ||
0c19a9fb | 4820 | if ((m < s) || (m >= e)) { |
be18010e KW |
4821 | pr_warn("error in %s: map handler doesn't belong to object\n", |
4822 | __func__); | |
9d759a9b WN |
4823 | return NULL; |
4824 | } | |
4825 | ||
0c19a9fb SF |
4826 | idx = (m - obj->maps) + i; |
4827 | if (idx >= obj->nr_maps || idx < 0) | |
9d759a9b WN |
4828 | return NULL; |
4829 | return &obj->maps[idx]; | |
4830 | } | |
561bbcca | 4831 | |
0c19a9fb | 4832 | struct bpf_map * |
a324aae3 | 4833 | bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) |
0c19a9fb SF |
4834 | { |
4835 | if (prev == NULL) | |
4836 | return obj->maps; | |
4837 | ||
4838 | return __bpf_map__iter(prev, obj, 1); | |
4839 | } | |
4840 | ||
4841 | struct bpf_map * | |
a324aae3 | 4842 | bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) |
0c19a9fb SF |
4843 | { |
4844 | if (next == NULL) { | |
4845 | if (!obj->nr_maps) | |
4846 | return NULL; | |
4847 | return obj->maps + obj->nr_maps - 1; | |
4848 | } | |
4849 | ||
4850 | return __bpf_map__iter(next, obj, -1); | |
4851 | } | |
4852 | ||
561bbcca | 4853 | struct bpf_map * |
a324aae3 | 4854 | bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) |
561bbcca WN |
4855 | { |
4856 | struct bpf_map *pos; | |
4857 | ||
f74a53d9 | 4858 | bpf_object__for_each_map(pos, obj) { |
973170e6 | 4859 | if (pos->name && !strcmp(pos->name, name)) |
561bbcca WN |
4860 | return pos; |
4861 | } | |
4862 | return NULL; | |
4863 | } | |
5a6acad1 | 4864 | |
f3cea32d | 4865 | int |
a324aae3 | 4866 | bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) |
f3cea32d MF |
4867 | { |
4868 | return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); | |
4869 | } | |
4870 | ||
5a6acad1 WN |
4871 | struct bpf_map * |
4872 | bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) | |
4873 | { | |
db48814b | 4874 | return ERR_PTR(-ENOTSUP); |
5a6acad1 | 4875 | } |
e28ff1a8 JS |
4876 | |
4877 | long libbpf_get_error(const void *ptr) | |
4878 | { | |
d98363b5 | 4879 | return PTR_ERR_OR_ZERO(ptr); |
e28ff1a8 | 4880 | } |
6f6d33f3 JF |
4881 | |
4882 | int bpf_prog_load(const char *file, enum bpf_prog_type type, | |
4883 | struct bpf_object **pobj, int *prog_fd) | |
d7be143b AI |
4884 | { |
4885 | struct bpf_prog_load_attr attr; | |
4886 | ||
4887 | memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); | |
4888 | attr.file = file; | |
4889 | attr.prog_type = type; | |
4890 | attr.expected_attach_type = 0; | |
4891 | ||
4892 | return bpf_prog_load_xattr(&attr, pobj, prog_fd); | |
4893 | } | |
4894 | ||
4895 | int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, | |
4896 | struct bpf_object **pobj, int *prog_fd) | |
6f6d33f3 | 4897 | { |
33bae185 | 4898 | struct bpf_object_open_attr open_attr = {}; |
48cca7e4 | 4899 | struct bpf_program *prog, *first_prog = NULL; |
d7be143b AI |
4900 | enum bpf_attach_type expected_attach_type; |
4901 | enum bpf_prog_type prog_type; | |
6f6d33f3 | 4902 | struct bpf_object *obj; |
f0307a7e | 4903 | struct bpf_map *map; |
6f6d33f3 JF |
4904 | int err; |
4905 | ||
d7be143b AI |
4906 | if (!attr) |
4907 | return -EINVAL; | |
17387dd5 JK |
4908 | if (!attr->file) |
4909 | return -EINVAL; | |
d7be143b | 4910 | |
33bae185 LY |
4911 | open_attr.file = attr->file; |
4912 | open_attr.prog_type = attr->prog_type; | |
4913 | ||
07f2d4ea | 4914 | obj = bpf_object__open_xattr(&open_attr); |
3597683c | 4915 | if (IS_ERR_OR_NULL(obj)) |
6f6d33f3 JF |
4916 | return -ENOENT; |
4917 | ||
48cca7e4 AS |
4918 | bpf_object__for_each_program(prog, obj) { |
4919 | /* | |
4920 | * If type is not specified, try to guess it based on | |
4921 | * section name. | |
4922 | */ | |
d7be143b | 4923 | prog_type = attr->prog_type; |
f0307a7e | 4924 | prog->prog_ifindex = attr->ifindex; |
d7be143b AI |
4925 | expected_attach_type = attr->expected_attach_type; |
4926 | if (prog_type == BPF_PROG_TYPE_UNSPEC) { | |
b60df2a0 JK |
4927 | err = bpf_program__identify_section(prog, &prog_type, |
4928 | &expected_attach_type); | |
4929 | if (err < 0) { | |
48cca7e4 AS |
4930 | bpf_object__close(obj); |
4931 | return -EINVAL; | |
4932 | } | |
583c9009 | 4933 | } |
48cca7e4 | 4934 | |
d7be143b AI |
4935 | bpf_program__set_type(prog, prog_type); |
4936 | bpf_program__set_expected_attach_type(prog, | |
4937 | expected_attach_type); | |
4938 | ||
da11b417 | 4939 | prog->log_level = attr->log_level; |
04656198 | 4940 | prog->prog_flags = attr->prog_flags; |
69495d2a | 4941 | if (!first_prog) |
48cca7e4 AS |
4942 | first_prog = prog; |
4943 | } | |
4944 | ||
f74a53d9 | 4945 | bpf_object__for_each_map(map, obj) { |
f83fb22c JK |
4946 | if (!bpf_map__is_offload_neutral(map)) |
4947 | map->map_ifindex = attr->ifindex; | |
f0307a7e DB |
4948 | } |
4949 | ||
48cca7e4 | 4950 | if (!first_prog) { |
be18010e | 4951 | pr_warn("object file doesn't contain bpf program\n"); |
48cca7e4 AS |
4952 | bpf_object__close(obj); |
4953 | return -ENOENT; | |
583c9009 RG |
4954 | } |
4955 | ||
6f6d33f3 JF |
4956 | err = bpf_object__load(obj); |
4957 | if (err) { | |
4958 | bpf_object__close(obj); | |
4959 | return -EINVAL; | |
4960 | } | |
4961 | ||
4962 | *pobj = obj; | |
48cca7e4 | 4963 | *prog_fd = bpf_program__fd(first_prog); |
6f6d33f3 JF |
4964 | return 0; |
4965 | } | |
d0cabbb0 | 4966 | |
1c2e9efc AN |
4967 | struct bpf_link { |
4968 | int (*destroy)(struct bpf_link *link); | |
4969 | }; | |
4970 | ||
4971 | int bpf_link__destroy(struct bpf_link *link) | |
4972 | { | |
4973 | int err; | |
4974 | ||
4975 | if (!link) | |
4976 | return 0; | |
4977 | ||
4978 | err = link->destroy(link); | |
4979 | free(link); | |
4980 | ||
4981 | return err; | |
4982 | } | |
4983 | ||
63f2f5ee AN |
4984 | struct bpf_link_fd { |
4985 | struct bpf_link link; /* has to be at the top of struct */ | |
4986 | int fd; /* hook FD */ | |
4987 | }; | |
4988 | ||
4989 | static int bpf_link__destroy_perf_event(struct bpf_link *link) | |
4990 | { | |
4991 | struct bpf_link_fd *l = (void *)link; | |
4992 | int err; | |
4993 | ||
4994 | err = ioctl(l->fd, PERF_EVENT_IOC_DISABLE, 0); | |
4995 | if (err) | |
4996 | err = -errno; | |
4997 | ||
4998 | close(l->fd); | |
4999 | return err; | |
5000 | } | |
5001 | ||
5002 | struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, | |
5003 | int pfd) | |
5004 | { | |
5005 | char errmsg[STRERR_BUFSIZE]; | |
5006 | struct bpf_link_fd *link; | |
5007 | int prog_fd, err; | |
5008 | ||
5009 | if (pfd < 0) { | |
be18010e KW |
5010 | pr_warn("program '%s': invalid perf event FD %d\n", |
5011 | bpf_program__title(prog, false), pfd); | |
63f2f5ee AN |
5012 | return ERR_PTR(-EINVAL); |
5013 | } | |
5014 | prog_fd = bpf_program__fd(prog); | |
5015 | if (prog_fd < 0) { | |
be18010e KW |
5016 | pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n", |
5017 | bpf_program__title(prog, false)); | |
63f2f5ee AN |
5018 | return ERR_PTR(-EINVAL); |
5019 | } | |
5020 | ||
5021 | link = malloc(sizeof(*link)); | |
5022 | if (!link) | |
5023 | return ERR_PTR(-ENOMEM); | |
5024 | link->link.destroy = &bpf_link__destroy_perf_event; | |
5025 | link->fd = pfd; | |
5026 | ||
5027 | if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { | |
5028 | err = -errno; | |
5029 | free(link); | |
be18010e KW |
5030 | pr_warn("program '%s': failed to attach to pfd %d: %s\n", |
5031 | bpf_program__title(prog, false), pfd, | |
63f2f5ee AN |
5032 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); |
5033 | return ERR_PTR(err); | |
5034 | } | |
5035 | if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { | |
5036 | err = -errno; | |
5037 | free(link); | |
be18010e KW |
5038 | pr_warn("program '%s': failed to enable pfd %d: %s\n", |
5039 | bpf_program__title(prog, false), pfd, | |
63f2f5ee AN |
5040 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); |
5041 | return ERR_PTR(err); | |
5042 | } | |
5043 | return (struct bpf_link *)link; | |
5044 | } | |
5045 | ||
b2650027 AN |
5046 | /* |
5047 | * this function is expected to parse integer in the range of [0, 2^31-1] from | |
5048 | * given file using scanf format string fmt. If actual parsed value is | |
5049 | * negative, the result might be indistinguishable from error | |
5050 | */ | |
5051 | static int parse_uint_from_file(const char *file, const char *fmt) | |
5052 | { | |
5053 | char buf[STRERR_BUFSIZE]; | |
5054 | int err, ret; | |
5055 | FILE *f; | |
5056 | ||
5057 | f = fopen(file, "r"); | |
5058 | if (!f) { | |
5059 | err = -errno; | |
5060 | pr_debug("failed to open '%s': %s\n", file, | |
5061 | libbpf_strerror_r(err, buf, sizeof(buf))); | |
5062 | return err; | |
5063 | } | |
5064 | err = fscanf(f, fmt, &ret); | |
5065 | if (err != 1) { | |
5066 | err = err == EOF ? -EIO : -errno; | |
5067 | pr_debug("failed to parse '%s': %s\n", file, | |
5068 | libbpf_strerror_r(err, buf, sizeof(buf))); | |
5069 | fclose(f); | |
5070 | return err; | |
5071 | } | |
5072 | fclose(f); | |
5073 | return ret; | |
5074 | } | |
5075 | ||
5076 | static int determine_kprobe_perf_type(void) | |
5077 | { | |
5078 | const char *file = "/sys/bus/event_source/devices/kprobe/type"; | |
5079 | ||
5080 | return parse_uint_from_file(file, "%d\n"); | |
5081 | } | |
5082 | ||
5083 | static int determine_uprobe_perf_type(void) | |
5084 | { | |
5085 | const char *file = "/sys/bus/event_source/devices/uprobe/type"; | |
5086 | ||
5087 | return parse_uint_from_file(file, "%d\n"); | |
5088 | } | |
5089 | ||
5090 | static int determine_kprobe_retprobe_bit(void) | |
5091 | { | |
5092 | const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; | |
5093 | ||
5094 | return parse_uint_from_file(file, "config:%d\n"); | |
5095 | } | |
5096 | ||
5097 | static int determine_uprobe_retprobe_bit(void) | |
5098 | { | |
5099 | const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; | |
5100 | ||
5101 | return parse_uint_from_file(file, "config:%d\n"); | |
5102 | } | |
5103 | ||
5104 | static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, | |
5105 | uint64_t offset, int pid) | |
5106 | { | |
5107 | struct perf_event_attr attr = {}; | |
5108 | char errmsg[STRERR_BUFSIZE]; | |
5109 | int type, pfd, err; | |
5110 | ||
5111 | type = uprobe ? determine_uprobe_perf_type() | |
5112 | : determine_kprobe_perf_type(); | |
5113 | if (type < 0) { | |
be18010e KW |
5114 | pr_warn("failed to determine %s perf type: %s\n", |
5115 | uprobe ? "uprobe" : "kprobe", | |
5116 | libbpf_strerror_r(type, errmsg, sizeof(errmsg))); | |
b2650027 AN |
5117 | return type; |
5118 | } | |
5119 | if (retprobe) { | |
5120 | int bit = uprobe ? determine_uprobe_retprobe_bit() | |
5121 | : determine_kprobe_retprobe_bit(); | |
5122 | ||
5123 | if (bit < 0) { | |
be18010e KW |
5124 | pr_warn("failed to determine %s retprobe bit: %s\n", |
5125 | uprobe ? "uprobe" : "kprobe", | |
5126 | libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); | |
b2650027 AN |
5127 | return bit; |
5128 | } | |
5129 | attr.config |= 1 << bit; | |
5130 | } | |
5131 | attr.size = sizeof(attr); | |
5132 | attr.type = type; | |
36db2a94 AN |
5133 | attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ |
5134 | attr.config2 = offset; /* kprobe_addr or probe_offset */ | |
b2650027 AN |
5135 | |
5136 | /* pid filter is meaningful only for uprobes */ | |
5137 | pfd = syscall(__NR_perf_event_open, &attr, | |
5138 | pid < 0 ? -1 : pid /* pid */, | |
5139 | pid == -1 ? 0 : -1 /* cpu */, | |
5140 | -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); | |
5141 | if (pfd < 0) { | |
5142 | err = -errno; | |
be18010e KW |
5143 | pr_warn("%s perf_event_open() failed: %s\n", |
5144 | uprobe ? "uprobe" : "kprobe", | |
5145 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
b2650027 AN |
5146 | return err; |
5147 | } | |
5148 | return pfd; | |
5149 | } | |
5150 | ||
5151 | struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, | |
5152 | bool retprobe, | |
5153 | const char *func_name) | |
5154 | { | |
5155 | char errmsg[STRERR_BUFSIZE]; | |
5156 | struct bpf_link *link; | |
5157 | int pfd, err; | |
5158 | ||
5159 | pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, | |
5160 | 0 /* offset */, -1 /* pid */); | |
5161 | if (pfd < 0) { | |
be18010e KW |
5162 | pr_warn("program '%s': failed to create %s '%s' perf event: %s\n", |
5163 | bpf_program__title(prog, false), | |
5164 | retprobe ? "kretprobe" : "kprobe", func_name, | |
5165 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
b2650027 AN |
5166 | return ERR_PTR(pfd); |
5167 | } | |
5168 | link = bpf_program__attach_perf_event(prog, pfd); | |
5169 | if (IS_ERR(link)) { | |
5170 | close(pfd); | |
5171 | err = PTR_ERR(link); | |
be18010e KW |
5172 | pr_warn("program '%s': failed to attach to %s '%s': %s\n", |
5173 | bpf_program__title(prog, false), | |
5174 | retprobe ? "kretprobe" : "kprobe", func_name, | |
5175 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
b2650027 AN |
5176 | return link; |
5177 | } | |
5178 | return link; | |
5179 | } | |
5180 | ||
5181 | struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, | |
5182 | bool retprobe, pid_t pid, | |
5183 | const char *binary_path, | |
5184 | size_t func_offset) | |
5185 | { | |
5186 | char errmsg[STRERR_BUFSIZE]; | |
5187 | struct bpf_link *link; | |
5188 | int pfd, err; | |
5189 | ||
5190 | pfd = perf_event_open_probe(true /* uprobe */, retprobe, | |
5191 | binary_path, func_offset, pid); | |
5192 | if (pfd < 0) { | |
be18010e KW |
5193 | pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", |
5194 | bpf_program__title(prog, false), | |
5195 | retprobe ? "uretprobe" : "uprobe", | |
5196 | binary_path, func_offset, | |
5197 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
b2650027 AN |
5198 | return ERR_PTR(pfd); |
5199 | } | |
5200 | link = bpf_program__attach_perf_event(prog, pfd); | |
5201 | if (IS_ERR(link)) { | |
5202 | close(pfd); | |
5203 | err = PTR_ERR(link); | |
be18010e KW |
5204 | pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n", |
5205 | bpf_program__title(prog, false), | |
5206 | retprobe ? "uretprobe" : "uprobe", | |
5207 | binary_path, func_offset, | |
5208 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
b2650027 AN |
5209 | return link; |
5210 | } | |
5211 | return link; | |
5212 | } | |
5213 | ||
f6de59c1 AN |
5214 | static int determine_tracepoint_id(const char *tp_category, |
5215 | const char *tp_name) | |
5216 | { | |
5217 | char file[PATH_MAX]; | |
5218 | int ret; | |
5219 | ||
5220 | ret = snprintf(file, sizeof(file), | |
5221 | "/sys/kernel/debug/tracing/events/%s/%s/id", | |
5222 | tp_category, tp_name); | |
5223 | if (ret < 0) | |
5224 | return -errno; | |
5225 | if (ret >= sizeof(file)) { | |
5226 | pr_debug("tracepoint %s/%s path is too long\n", | |
5227 | tp_category, tp_name); | |
5228 | return -E2BIG; | |
5229 | } | |
5230 | return parse_uint_from_file(file, "%d\n"); | |
5231 | } | |
5232 | ||
5233 | static int perf_event_open_tracepoint(const char *tp_category, | |
5234 | const char *tp_name) | |
5235 | { | |
5236 | struct perf_event_attr attr = {}; | |
5237 | char errmsg[STRERR_BUFSIZE]; | |
5238 | int tp_id, pfd, err; | |
5239 | ||
5240 | tp_id = determine_tracepoint_id(tp_category, tp_name); | |
5241 | if (tp_id < 0) { | |
be18010e KW |
5242 | pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", |
5243 | tp_category, tp_name, | |
5244 | libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); | |
f6de59c1 AN |
5245 | return tp_id; |
5246 | } | |
5247 | ||
5248 | attr.type = PERF_TYPE_TRACEPOINT; | |
5249 | attr.size = sizeof(attr); | |
5250 | attr.config = tp_id; | |
5251 | ||
5252 | pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, | |
5253 | -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); | |
5254 | if (pfd < 0) { | |
5255 | err = -errno; | |
be18010e KW |
5256 | pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", |
5257 | tp_category, tp_name, | |
5258 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
f6de59c1 AN |
5259 | return err; |
5260 | } | |
5261 | return pfd; | |
5262 | } | |
5263 | ||
5264 | struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, | |
5265 | const char *tp_category, | |
5266 | const char *tp_name) | |
5267 | { | |
5268 | char errmsg[STRERR_BUFSIZE]; | |
5269 | struct bpf_link *link; | |
5270 | int pfd, err; | |
5271 | ||
5272 | pfd = perf_event_open_tracepoint(tp_category, tp_name); | |
5273 | if (pfd < 0) { | |
be18010e KW |
5274 | pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", |
5275 | bpf_program__title(prog, false), | |
5276 | tp_category, tp_name, | |
5277 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
f6de59c1 AN |
5278 | return ERR_PTR(pfd); |
5279 | } | |
5280 | link = bpf_program__attach_perf_event(prog, pfd); | |
5281 | if (IS_ERR(link)) { | |
5282 | close(pfd); | |
5283 | err = PTR_ERR(link); | |
be18010e KW |
5284 | pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n", |
5285 | bpf_program__title(prog, false), | |
5286 | tp_category, tp_name, | |
5287 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
f6de59c1 AN |
5288 | return link; |
5289 | } | |
5290 | return link; | |
5291 | } | |
5292 | ||
84bf5e1f AN |
5293 | static int bpf_link__destroy_fd(struct bpf_link *link) |
5294 | { | |
5295 | struct bpf_link_fd *l = (void *)link; | |
5296 | ||
5297 | return close(l->fd); | |
5298 | } | |
5299 | ||
5300 | struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, | |
5301 | const char *tp_name) | |
5302 | { | |
5303 | char errmsg[STRERR_BUFSIZE]; | |
5304 | struct bpf_link_fd *link; | |
5305 | int prog_fd, pfd; | |
5306 | ||
5307 | prog_fd = bpf_program__fd(prog); | |
5308 | if (prog_fd < 0) { | |
be18010e KW |
5309 | pr_warn("program '%s': can't attach before loaded\n", |
5310 | bpf_program__title(prog, false)); | |
84bf5e1f AN |
5311 | return ERR_PTR(-EINVAL); |
5312 | } | |
5313 | ||
5314 | link = malloc(sizeof(*link)); | |
5315 | if (!link) | |
5316 | return ERR_PTR(-ENOMEM); | |
5317 | link->link.destroy = &bpf_link__destroy_fd; | |
5318 | ||
5319 | pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); | |
5320 | if (pfd < 0) { | |
5321 | pfd = -errno; | |
5322 | free(link); | |
be18010e KW |
5323 | pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n", |
5324 | bpf_program__title(prog, false), tp_name, | |
5325 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
84bf5e1f AN |
5326 | return ERR_PTR(pfd); |
5327 | } | |
5328 | link->fd = pfd; | |
5329 | return (struct bpf_link *)link; | |
5330 | } | |
5331 | ||
d0cabbb0 | 5332 | enum bpf_perf_event_ret |
3dca2115 DB |
5333 | bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, |
5334 | void **copy_mem, size_t *copy_size, | |
5335 | bpf_perf_event_print_t fn, void *private_data) | |
d0cabbb0 | 5336 | { |
3dca2115 | 5337 | struct perf_event_mmap_page *header = mmap_mem; |
a64af0ef | 5338 | __u64 data_head = ring_buffer_read_head(header); |
d0cabbb0 | 5339 | __u64 data_tail = header->data_tail; |
3dca2115 DB |
5340 | void *base = ((__u8 *)header) + page_size; |
5341 | int ret = LIBBPF_PERF_EVENT_CONT; | |
5342 | struct perf_event_header *ehdr; | |
5343 | size_t ehdr_size; | |
5344 | ||
5345 | while (data_head != data_tail) { | |
5346 | ehdr = base + (data_tail & (mmap_size - 1)); | |
5347 | ehdr_size = ehdr->size; | |
5348 | ||
5349 | if (((void *)ehdr) + ehdr_size > base + mmap_size) { | |
5350 | void *copy_start = ehdr; | |
5351 | size_t len_first = base + mmap_size - copy_start; | |
5352 | size_t len_secnd = ehdr_size - len_first; | |
5353 | ||
5354 | if (*copy_size < ehdr_size) { | |
5355 | free(*copy_mem); | |
5356 | *copy_mem = malloc(ehdr_size); | |
5357 | if (!*copy_mem) { | |
5358 | *copy_size = 0; | |
d0cabbb0 JK |
5359 | ret = LIBBPF_PERF_EVENT_ERROR; |
5360 | break; | |
5361 | } | |
3dca2115 | 5362 | *copy_size = ehdr_size; |
d0cabbb0 JK |
5363 | } |
5364 | ||
3dca2115 DB |
5365 | memcpy(*copy_mem, copy_start, len_first); |
5366 | memcpy(*copy_mem + len_first, base, len_secnd); | |
5367 | ehdr = *copy_mem; | |
d0cabbb0 JK |
5368 | } |
5369 | ||
3dca2115 DB |
5370 | ret = fn(ehdr, private_data); |
5371 | data_tail += ehdr_size; | |
d0cabbb0 JK |
5372 | if (ret != LIBBPF_PERF_EVENT_CONT) |
5373 | break; | |
d0cabbb0 JK |
5374 | } |
5375 | ||
a64af0ef | 5376 | ring_buffer_write_tail(header, data_tail); |
d0cabbb0 JK |
5377 | return ret; |
5378 | } | |
34be1646 | 5379 | |
fb84b822 AN |
5380 | struct perf_buffer; |
5381 | ||
5382 | struct perf_buffer_params { | |
5383 | struct perf_event_attr *attr; | |
5384 | /* if event_cb is specified, it takes precendence */ | |
5385 | perf_buffer_event_fn event_cb; | |
5386 | /* sample_cb and lost_cb are higher-level common-case callbacks */ | |
5387 | perf_buffer_sample_fn sample_cb; | |
5388 | perf_buffer_lost_fn lost_cb; | |
5389 | void *ctx; | |
5390 | int cpu_cnt; | |
5391 | int *cpus; | |
5392 | int *map_keys; | |
5393 | }; | |
5394 | ||
5395 | struct perf_cpu_buf { | |
5396 | struct perf_buffer *pb; | |
5397 | void *base; /* mmap()'ed memory */ | |
5398 | void *buf; /* for reconstructing segmented data */ | |
5399 | size_t buf_size; | |
5400 | int fd; | |
5401 | int cpu; | |
5402 | int map_key; | |
5403 | }; | |
5404 | ||
5405 | struct perf_buffer { | |
5406 | perf_buffer_event_fn event_cb; | |
5407 | perf_buffer_sample_fn sample_cb; | |
5408 | perf_buffer_lost_fn lost_cb; | |
5409 | void *ctx; /* passed into callbacks */ | |
5410 | ||
5411 | size_t page_size; | |
5412 | size_t mmap_size; | |
5413 | struct perf_cpu_buf **cpu_bufs; | |
5414 | struct epoll_event *events; | |
5415 | int cpu_cnt; | |
5416 | int epoll_fd; /* perf event FD */ | |
5417 | int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ | |
5418 | }; | |
5419 | ||
5420 | static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, | |
5421 | struct perf_cpu_buf *cpu_buf) | |
5422 | { | |
5423 | if (!cpu_buf) | |
5424 | return; | |
5425 | if (cpu_buf->base && | |
5426 | munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) | |
be18010e | 5427 | pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); |
fb84b822 AN |
5428 | if (cpu_buf->fd >= 0) { |
5429 | ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); | |
5430 | close(cpu_buf->fd); | |
5431 | } | |
5432 | free(cpu_buf->buf); | |
5433 | free(cpu_buf); | |
5434 | } | |
5435 | ||
5436 | void perf_buffer__free(struct perf_buffer *pb) | |
5437 | { | |
5438 | int i; | |
5439 | ||
5440 | if (!pb) | |
5441 | return; | |
5442 | if (pb->cpu_bufs) { | |
5443 | for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) { | |
5444 | struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; | |
5445 | ||
5446 | bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); | |
5447 | perf_buffer__free_cpu_buf(pb, cpu_buf); | |
5448 | } | |
5449 | free(pb->cpu_bufs); | |
5450 | } | |
5451 | if (pb->epoll_fd >= 0) | |
5452 | close(pb->epoll_fd); | |
5453 | free(pb->events); | |
5454 | free(pb); | |
5455 | } | |
5456 | ||
5457 | static struct perf_cpu_buf * | |
5458 | perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, | |
5459 | int cpu, int map_key) | |
5460 | { | |
5461 | struct perf_cpu_buf *cpu_buf; | |
5462 | char msg[STRERR_BUFSIZE]; | |
5463 | int err; | |
5464 | ||
5465 | cpu_buf = calloc(1, sizeof(*cpu_buf)); | |
5466 | if (!cpu_buf) | |
5467 | return ERR_PTR(-ENOMEM); | |
5468 | ||
5469 | cpu_buf->pb = pb; | |
5470 | cpu_buf->cpu = cpu; | |
5471 | cpu_buf->map_key = map_key; | |
5472 | ||
5473 | cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, | |
5474 | -1, PERF_FLAG_FD_CLOEXEC); | |
5475 | if (cpu_buf->fd < 0) { | |
5476 | err = -errno; | |
be18010e KW |
5477 | pr_warn("failed to open perf buffer event on cpu #%d: %s\n", |
5478 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
5479 | goto error; |
5480 | } | |
5481 | ||
5482 | cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, | |
5483 | PROT_READ | PROT_WRITE, MAP_SHARED, | |
5484 | cpu_buf->fd, 0); | |
5485 | if (cpu_buf->base == MAP_FAILED) { | |
5486 | cpu_buf->base = NULL; | |
5487 | err = -errno; | |
be18010e KW |
5488 | pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", |
5489 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
5490 | goto error; |
5491 | } | |
5492 | ||
5493 | if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { | |
5494 | err = -errno; | |
be18010e KW |
5495 | pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", |
5496 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
5497 | goto error; |
5498 | } | |
5499 | ||
5500 | return cpu_buf; | |
5501 | ||
5502 | error: | |
5503 | perf_buffer__free_cpu_buf(pb, cpu_buf); | |
5504 | return (struct perf_cpu_buf *)ERR_PTR(err); | |
5505 | } | |
5506 | ||
5507 | static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, | |
5508 | struct perf_buffer_params *p); | |
5509 | ||
5510 | struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, | |
5511 | const struct perf_buffer_opts *opts) | |
5512 | { | |
5513 | struct perf_buffer_params p = {}; | |
4be6e05c ACM |
5514 | struct perf_event_attr attr = { 0, }; |
5515 | ||
5516 | attr.config = PERF_COUNT_SW_BPF_OUTPUT, | |
5517 | attr.type = PERF_TYPE_SOFTWARE; | |
5518 | attr.sample_type = PERF_SAMPLE_RAW; | |
5519 | attr.sample_period = 1; | |
5520 | attr.wakeup_events = 1; | |
fb84b822 AN |
5521 | |
5522 | p.attr = &attr; | |
5523 | p.sample_cb = opts ? opts->sample_cb : NULL; | |
5524 | p.lost_cb = opts ? opts->lost_cb : NULL; | |
5525 | p.ctx = opts ? opts->ctx : NULL; | |
5526 | ||
5527 | return __perf_buffer__new(map_fd, page_cnt, &p); | |
5528 | } | |
5529 | ||
5530 | struct perf_buffer * | |
5531 | perf_buffer__new_raw(int map_fd, size_t page_cnt, | |
5532 | const struct perf_buffer_raw_opts *opts) | |
5533 | { | |
5534 | struct perf_buffer_params p = {}; | |
5535 | ||
5536 | p.attr = opts->attr; | |
5537 | p.event_cb = opts->event_cb; | |
5538 | p.ctx = opts->ctx; | |
5539 | p.cpu_cnt = opts->cpu_cnt; | |
5540 | p.cpus = opts->cpus; | |
5541 | p.map_keys = opts->map_keys; | |
5542 | ||
5543 | return __perf_buffer__new(map_fd, page_cnt, &p); | |
5544 | } | |
5545 | ||
5546 | static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, | |
5547 | struct perf_buffer_params *p) | |
5548 | { | |
5549 | struct bpf_map_info map = {}; | |
5550 | char msg[STRERR_BUFSIZE]; | |
5551 | struct perf_buffer *pb; | |
5552 | __u32 map_info_len; | |
5553 | int err, i; | |
5554 | ||
5555 | if (page_cnt & (page_cnt - 1)) { | |
be18010e KW |
5556 | pr_warn("page count should be power of two, but is %zu\n", |
5557 | page_cnt); | |
fb84b822 AN |
5558 | return ERR_PTR(-EINVAL); |
5559 | } | |
5560 | ||
5561 | map_info_len = sizeof(map); | |
5562 | err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); | |
5563 | if (err) { | |
5564 | err = -errno; | |
be18010e KW |
5565 | pr_warn("failed to get map info for map FD %d: %s\n", |
5566 | map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
5567 | return ERR_PTR(err); |
5568 | } | |
5569 | ||
5570 | if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { | |
be18010e KW |
5571 | pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", |
5572 | map.name); | |
fb84b822 AN |
5573 | return ERR_PTR(-EINVAL); |
5574 | } | |
5575 | ||
5576 | pb = calloc(1, sizeof(*pb)); | |
5577 | if (!pb) | |
5578 | return ERR_PTR(-ENOMEM); | |
5579 | ||
5580 | pb->event_cb = p->event_cb; | |
5581 | pb->sample_cb = p->sample_cb; | |
5582 | pb->lost_cb = p->lost_cb; | |
5583 | pb->ctx = p->ctx; | |
5584 | ||
5585 | pb->page_size = getpagesize(); | |
5586 | pb->mmap_size = pb->page_size * page_cnt; | |
5587 | pb->map_fd = map_fd; | |
5588 | ||
5589 | pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); | |
5590 | if (pb->epoll_fd < 0) { | |
5591 | err = -errno; | |
be18010e KW |
5592 | pr_warn("failed to create epoll instance: %s\n", |
5593 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
5594 | goto error; |
5595 | } | |
5596 | ||
5597 | if (p->cpu_cnt > 0) { | |
5598 | pb->cpu_cnt = p->cpu_cnt; | |
5599 | } else { | |
5600 | pb->cpu_cnt = libbpf_num_possible_cpus(); | |
5601 | if (pb->cpu_cnt < 0) { | |
5602 | err = pb->cpu_cnt; | |
5603 | goto error; | |
5604 | } | |
5605 | if (map.max_entries < pb->cpu_cnt) | |
5606 | pb->cpu_cnt = map.max_entries; | |
5607 | } | |
5608 | ||
5609 | pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); | |
5610 | if (!pb->events) { | |
5611 | err = -ENOMEM; | |
be18010e | 5612 | pr_warn("failed to allocate events: out of memory\n"); |
fb84b822 AN |
5613 | goto error; |
5614 | } | |
5615 | pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); | |
5616 | if (!pb->cpu_bufs) { | |
5617 | err = -ENOMEM; | |
be18010e | 5618 | pr_warn("failed to allocate buffers: out of memory\n"); |
fb84b822 AN |
5619 | goto error; |
5620 | } | |
5621 | ||
5622 | for (i = 0; i < pb->cpu_cnt; i++) { | |
5623 | struct perf_cpu_buf *cpu_buf; | |
5624 | int cpu, map_key; | |
5625 | ||
5626 | cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; | |
5627 | map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; | |
5628 | ||
5629 | cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); | |
5630 | if (IS_ERR(cpu_buf)) { | |
5631 | err = PTR_ERR(cpu_buf); | |
5632 | goto error; | |
5633 | } | |
5634 | ||
5635 | pb->cpu_bufs[i] = cpu_buf; | |
5636 | ||
5637 | err = bpf_map_update_elem(pb->map_fd, &map_key, | |
5638 | &cpu_buf->fd, 0); | |
5639 | if (err) { | |
5640 | err = -errno; | |
be18010e KW |
5641 | pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", |
5642 | cpu, map_key, cpu_buf->fd, | |
5643 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
5644 | goto error; |
5645 | } | |
5646 | ||
5647 | pb->events[i].events = EPOLLIN; | |
5648 | pb->events[i].data.ptr = cpu_buf; | |
5649 | if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, | |
5650 | &pb->events[i]) < 0) { | |
5651 | err = -errno; | |
be18010e KW |
5652 | pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", |
5653 | cpu, cpu_buf->fd, | |
5654 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
5655 | goto error; |
5656 | } | |
5657 | } | |
5658 | ||
5659 | return pb; | |
5660 | ||
5661 | error: | |
5662 | if (pb) | |
5663 | perf_buffer__free(pb); | |
5664 | return ERR_PTR(err); | |
5665 | } | |
5666 | ||
5667 | struct perf_sample_raw { | |
5668 | struct perf_event_header header; | |
5669 | uint32_t size; | |
5670 | char data[0]; | |
5671 | }; | |
5672 | ||
5673 | struct perf_sample_lost { | |
5674 | struct perf_event_header header; | |
5675 | uint64_t id; | |
5676 | uint64_t lost; | |
5677 | uint64_t sample_id; | |
5678 | }; | |
5679 | ||
5680 | static enum bpf_perf_event_ret | |
5681 | perf_buffer__process_record(struct perf_event_header *e, void *ctx) | |
5682 | { | |
5683 | struct perf_cpu_buf *cpu_buf = ctx; | |
5684 | struct perf_buffer *pb = cpu_buf->pb; | |
5685 | void *data = e; | |
5686 | ||
5687 | /* user wants full control over parsing perf event */ | |
5688 | if (pb->event_cb) | |
5689 | return pb->event_cb(pb->ctx, cpu_buf->cpu, e); | |
5690 | ||
5691 | switch (e->type) { | |
5692 | case PERF_RECORD_SAMPLE: { | |
5693 | struct perf_sample_raw *s = data; | |
5694 | ||
5695 | if (pb->sample_cb) | |
5696 | pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); | |
5697 | break; | |
5698 | } | |
5699 | case PERF_RECORD_LOST: { | |
5700 | struct perf_sample_lost *s = data; | |
5701 | ||
5702 | if (pb->lost_cb) | |
5703 | pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); | |
5704 | break; | |
5705 | } | |
5706 | default: | |
be18010e | 5707 | pr_warn("unknown perf sample type %d\n", e->type); |
fb84b822 AN |
5708 | return LIBBPF_PERF_EVENT_ERROR; |
5709 | } | |
5710 | return LIBBPF_PERF_EVENT_CONT; | |
5711 | } | |
5712 | ||
5713 | static int perf_buffer__process_records(struct perf_buffer *pb, | |
5714 | struct perf_cpu_buf *cpu_buf) | |
5715 | { | |
5716 | enum bpf_perf_event_ret ret; | |
5717 | ||
5718 | ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, | |
5719 | pb->page_size, &cpu_buf->buf, | |
5720 | &cpu_buf->buf_size, | |
5721 | perf_buffer__process_record, cpu_buf); | |
5722 | if (ret != LIBBPF_PERF_EVENT_CONT) | |
5723 | return ret; | |
5724 | return 0; | |
5725 | } | |
5726 | ||
5727 | int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) | |
5728 | { | |
5729 | int i, cnt, err; | |
5730 | ||
5731 | cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); | |
5732 | for (i = 0; i < cnt; i++) { | |
5733 | struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; | |
5734 | ||
5735 | err = perf_buffer__process_records(pb, cpu_buf); | |
5736 | if (err) { | |
be18010e | 5737 | pr_warn("error while processing records: %d\n", err); |
fb84b822 AN |
5738 | return err; |
5739 | } | |
5740 | } | |
5741 | return cnt < 0 ? -errno : cnt; | |
5742 | } | |
5743 | ||
34be1646 SL |
5744 | struct bpf_prog_info_array_desc { |
5745 | int array_offset; /* e.g. offset of jited_prog_insns */ | |
5746 | int count_offset; /* e.g. offset of jited_prog_len */ | |
5747 | int size_offset; /* > 0: offset of rec size, | |
5748 | * < 0: fix size of -size_offset | |
5749 | */ | |
5750 | }; | |
5751 | ||
5752 | static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { | |
5753 | [BPF_PROG_INFO_JITED_INSNS] = { | |
5754 | offsetof(struct bpf_prog_info, jited_prog_insns), | |
5755 | offsetof(struct bpf_prog_info, jited_prog_len), | |
5756 | -1, | |
5757 | }, | |
5758 | [BPF_PROG_INFO_XLATED_INSNS] = { | |
5759 | offsetof(struct bpf_prog_info, xlated_prog_insns), | |
5760 | offsetof(struct bpf_prog_info, xlated_prog_len), | |
5761 | -1, | |
5762 | }, | |
5763 | [BPF_PROG_INFO_MAP_IDS] = { | |
5764 | offsetof(struct bpf_prog_info, map_ids), | |
5765 | offsetof(struct bpf_prog_info, nr_map_ids), | |
5766 | -(int)sizeof(__u32), | |
5767 | }, | |
5768 | [BPF_PROG_INFO_JITED_KSYMS] = { | |
5769 | offsetof(struct bpf_prog_info, jited_ksyms), | |
5770 | offsetof(struct bpf_prog_info, nr_jited_ksyms), | |
5771 | -(int)sizeof(__u64), | |
5772 | }, | |
5773 | [BPF_PROG_INFO_JITED_FUNC_LENS] = { | |
5774 | offsetof(struct bpf_prog_info, jited_func_lens), | |
5775 | offsetof(struct bpf_prog_info, nr_jited_func_lens), | |
5776 | -(int)sizeof(__u32), | |
5777 | }, | |
5778 | [BPF_PROG_INFO_FUNC_INFO] = { | |
5779 | offsetof(struct bpf_prog_info, func_info), | |
5780 | offsetof(struct bpf_prog_info, nr_func_info), | |
5781 | offsetof(struct bpf_prog_info, func_info_rec_size), | |
5782 | }, | |
5783 | [BPF_PROG_INFO_LINE_INFO] = { | |
5784 | offsetof(struct bpf_prog_info, line_info), | |
5785 | offsetof(struct bpf_prog_info, nr_line_info), | |
5786 | offsetof(struct bpf_prog_info, line_info_rec_size), | |
5787 | }, | |
5788 | [BPF_PROG_INFO_JITED_LINE_INFO] = { | |
5789 | offsetof(struct bpf_prog_info, jited_line_info), | |
5790 | offsetof(struct bpf_prog_info, nr_jited_line_info), | |
5791 | offsetof(struct bpf_prog_info, jited_line_info_rec_size), | |
5792 | }, | |
5793 | [BPF_PROG_INFO_PROG_TAGS] = { | |
5794 | offsetof(struct bpf_prog_info, prog_tags), | |
5795 | offsetof(struct bpf_prog_info, nr_prog_tags), | |
5796 | -(int)sizeof(__u8) * BPF_TAG_SIZE, | |
5797 | }, | |
5798 | ||
5799 | }; | |
5800 | ||
5801 | static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, int offset) | |
5802 | { | |
5803 | __u32 *array = (__u32 *)info; | |
5804 | ||
5805 | if (offset >= 0) | |
5806 | return array[offset / sizeof(__u32)]; | |
5807 | return -(int)offset; | |
5808 | } | |
5809 | ||
5810 | static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, int offset) | |
5811 | { | |
5812 | __u64 *array = (__u64 *)info; | |
5813 | ||
5814 | if (offset >= 0) | |
5815 | return array[offset / sizeof(__u64)]; | |
5816 | return -(int)offset; | |
5817 | } | |
5818 | ||
5819 | static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, | |
5820 | __u32 val) | |
5821 | { | |
5822 | __u32 *array = (__u32 *)info; | |
5823 | ||
5824 | if (offset >= 0) | |
5825 | array[offset / sizeof(__u32)] = val; | |
5826 | } | |
5827 | ||
5828 | static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, | |
5829 | __u64 val) | |
5830 | { | |
5831 | __u64 *array = (__u64 *)info; | |
5832 | ||
5833 | if (offset >= 0) | |
5834 | array[offset / sizeof(__u64)] = val; | |
5835 | } | |
5836 | ||
5837 | struct bpf_prog_info_linear * | |
5838 | bpf_program__get_prog_info_linear(int fd, __u64 arrays) | |
5839 | { | |
5840 | struct bpf_prog_info_linear *info_linear; | |
5841 | struct bpf_prog_info info = {}; | |
5842 | __u32 info_len = sizeof(info); | |
5843 | __u32 data_len = 0; | |
5844 | int i, err; | |
5845 | void *ptr; | |
5846 | ||
5847 | if (arrays >> BPF_PROG_INFO_LAST_ARRAY) | |
5848 | return ERR_PTR(-EINVAL); | |
5849 | ||
5850 | /* step 1: get array dimensions */ | |
5851 | err = bpf_obj_get_info_by_fd(fd, &info, &info_len); | |
5852 | if (err) { | |
5853 | pr_debug("can't get prog info: %s", strerror(errno)); | |
5854 | return ERR_PTR(-EFAULT); | |
5855 | } | |
5856 | ||
5857 | /* step 2: calculate total size of all arrays */ | |
5858 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5859 | bool include_array = (arrays & (1UL << i)) > 0; | |
5860 | struct bpf_prog_info_array_desc *desc; | |
5861 | __u32 count, size; | |
5862 | ||
5863 | desc = bpf_prog_info_array_desc + i; | |
5864 | ||
5865 | /* kernel is too old to support this field */ | |
5866 | if (info_len < desc->array_offset + sizeof(__u32) || | |
5867 | info_len < desc->count_offset + sizeof(__u32) || | |
5868 | (desc->size_offset > 0 && info_len < desc->size_offset)) | |
5869 | include_array = false; | |
5870 | ||
5871 | if (!include_array) { | |
5872 | arrays &= ~(1UL << i); /* clear the bit */ | |
5873 | continue; | |
5874 | } | |
5875 | ||
5876 | count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
5877 | size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
5878 | ||
5879 | data_len += count * size; | |
5880 | } | |
5881 | ||
5882 | /* step 3: allocate continuous memory */ | |
5883 | data_len = roundup(data_len, sizeof(__u64)); | |
5884 | info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); | |
5885 | if (!info_linear) | |
5886 | return ERR_PTR(-ENOMEM); | |
5887 | ||
5888 | /* step 4: fill data to info_linear->info */ | |
5889 | info_linear->arrays = arrays; | |
5890 | memset(&info_linear->info, 0, sizeof(info)); | |
5891 | ptr = info_linear->data; | |
5892 | ||
5893 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5894 | struct bpf_prog_info_array_desc *desc; | |
5895 | __u32 count, size; | |
5896 | ||
5897 | if ((arrays & (1UL << i)) == 0) | |
5898 | continue; | |
5899 | ||
5900 | desc = bpf_prog_info_array_desc + i; | |
5901 | count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
5902 | size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
5903 | bpf_prog_info_set_offset_u32(&info_linear->info, | |
5904 | desc->count_offset, count); | |
5905 | bpf_prog_info_set_offset_u32(&info_linear->info, | |
5906 | desc->size_offset, size); | |
5907 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
5908 | desc->array_offset, | |
5909 | ptr_to_u64(ptr)); | |
5910 | ptr += count * size; | |
5911 | } | |
5912 | ||
5913 | /* step 5: call syscall again to get required arrays */ | |
5914 | err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); | |
5915 | if (err) { | |
5916 | pr_debug("can't get prog info: %s", strerror(errno)); | |
5917 | free(info_linear); | |
5918 | return ERR_PTR(-EFAULT); | |
5919 | } | |
5920 | ||
5921 | /* step 6: verify the data */ | |
5922 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5923 | struct bpf_prog_info_array_desc *desc; | |
5924 | __u32 v1, v2; | |
5925 | ||
5926 | if ((arrays & (1UL << i)) == 0) | |
5927 | continue; | |
5928 | ||
5929 | desc = bpf_prog_info_array_desc + i; | |
5930 | v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
5931 | v2 = bpf_prog_info_read_offset_u32(&info_linear->info, | |
5932 | desc->count_offset); | |
5933 | if (v1 != v2) | |
be18010e | 5934 | pr_warn("%s: mismatch in element count\n", __func__); |
34be1646 SL |
5935 | |
5936 | v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
5937 | v2 = bpf_prog_info_read_offset_u32(&info_linear->info, | |
5938 | desc->size_offset); | |
5939 | if (v1 != v2) | |
be18010e | 5940 | pr_warn("%s: mismatch in rec size\n", __func__); |
34be1646 SL |
5941 | } |
5942 | ||
5943 | /* step 7: update info_len and data_len */ | |
5944 | info_linear->info_len = sizeof(struct bpf_prog_info); | |
5945 | info_linear->data_len = data_len; | |
5946 | ||
5947 | return info_linear; | |
5948 | } | |
5949 | ||
5950 | void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) | |
5951 | { | |
5952 | int i; | |
5953 | ||
5954 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5955 | struct bpf_prog_info_array_desc *desc; | |
5956 | __u64 addr, offs; | |
5957 | ||
5958 | if ((info_linear->arrays & (1UL << i)) == 0) | |
5959 | continue; | |
5960 | ||
5961 | desc = bpf_prog_info_array_desc + i; | |
5962 | addr = bpf_prog_info_read_offset_u64(&info_linear->info, | |
5963 | desc->array_offset); | |
5964 | offs = addr - ptr_to_u64(info_linear->data); | |
5965 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
5966 | desc->array_offset, offs); | |
5967 | } | |
5968 | } | |
5969 | ||
5970 | void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) | |
5971 | { | |
5972 | int i; | |
5973 | ||
5974 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
5975 | struct bpf_prog_info_array_desc *desc; | |
5976 | __u64 addr, offs; | |
5977 | ||
5978 | if ((info_linear->arrays & (1UL << i)) == 0) | |
5979 | continue; | |
5980 | ||
5981 | desc = bpf_prog_info_array_desc + i; | |
5982 | offs = bpf_prog_info_read_offset_u64(&info_linear->info, | |
5983 | desc->array_offset); | |
5984 | addr = offs + ptr_to_u64(info_linear->data); | |
5985 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
5986 | desc->array_offset, addr); | |
5987 | } | |
5988 | } | |
6446b315 HL |
5989 | |
5990 | int libbpf_num_possible_cpus(void) | |
5991 | { | |
5992 | static const char *fcpu = "/sys/devices/system/cpu/possible"; | |
5993 | int len = 0, n = 0, il = 0, ir = 0; | |
5994 | unsigned int start = 0, end = 0; | |
56fbc241 | 5995 | int tmp_cpus = 0; |
6446b315 HL |
5996 | static int cpus; |
5997 | char buf[128]; | |
5998 | int error = 0; | |
5999 | int fd = -1; | |
6000 | ||
56fbc241 TC |
6001 | tmp_cpus = READ_ONCE(cpus); |
6002 | if (tmp_cpus > 0) | |
6003 | return tmp_cpus; | |
6446b315 HL |
6004 | |
6005 | fd = open(fcpu, O_RDONLY); | |
6006 | if (fd < 0) { | |
6007 | error = errno; | |
be18010e | 6008 | pr_warn("Failed to open file %s: %s\n", fcpu, strerror(error)); |
6446b315 HL |
6009 | return -error; |
6010 | } | |
6011 | len = read(fd, buf, sizeof(buf)); | |
6012 | close(fd); | |
6013 | if (len <= 0) { | |
6014 | error = len ? errno : EINVAL; | |
be18010e KW |
6015 | pr_warn("Failed to read # of possible cpus from %s: %s\n", |
6016 | fcpu, strerror(error)); | |
6446b315 HL |
6017 | return -error; |
6018 | } | |
6019 | if (len == sizeof(buf)) { | |
be18010e | 6020 | pr_warn("File %s size overflow\n", fcpu); |
6446b315 HL |
6021 | return -EOVERFLOW; |
6022 | } | |
6023 | buf[len] = '\0'; | |
6024 | ||
56fbc241 | 6025 | for (ir = 0, tmp_cpus = 0; ir <= len; ir++) { |
6446b315 HL |
6026 | /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ |
6027 | if (buf[ir] == ',' || buf[ir] == '\0') { | |
6028 | buf[ir] = '\0'; | |
6029 | n = sscanf(&buf[il], "%u-%u", &start, &end); | |
6030 | if (n <= 0) { | |
be18010e KW |
6031 | pr_warn("Failed to get # CPUs from %s\n", |
6032 | &buf[il]); | |
6446b315 HL |
6033 | return -EINVAL; |
6034 | } else if (n == 1) { | |
6035 | end = start; | |
6036 | } | |
56fbc241 | 6037 | tmp_cpus += end - start + 1; |
6446b315 HL |
6038 | il = ir + 1; |
6039 | } | |
6040 | } | |
56fbc241 | 6041 | if (tmp_cpus <= 0) { |
be18010e | 6042 | pr_warn("Invalid #CPUs %d from %s\n", tmp_cpus, fcpu); |
6446b315 HL |
6043 | return -EINVAL; |
6044 | } | |
56fbc241 TC |
6045 | |
6046 | WRITE_ONCE(cpus, tmp_cpus); | |
6047 | return tmp_cpus; | |
6446b315 | 6048 | } |