Commit | Line | Data |
---|---|---|
1bc38b8f | 1 | // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) |
6061a3d6 | 2 | |
1b76c13e WN |
3 | /* |
4 | * Common eBPF ELF object loading operations. | |
5 | * | |
6 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
7 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
8 | * Copyright (C) 2015 Huawei Inc. | |
f367540c | 9 | * Copyright (C) 2017 Nicira, Inc. |
d859900c | 10 | * Copyright (C) 2019 Isovalent, Inc. |
1b76c13e WN |
11 | */ |
12 | ||
b4269954 | 13 | #ifndef _GNU_SOURCE |
531b014e | 14 | #define _GNU_SOURCE |
b4269954 | 15 | #endif |
1b76c13e | 16 | #include <stdlib.h> |
b3f59d66 WN |
17 | #include <stdio.h> |
18 | #include <stdarg.h> | |
f367540c | 19 | #include <libgen.h> |
34090915 | 20 | #include <inttypes.h> |
8ab9da57 | 21 | #include <limits.h> |
b3f59d66 | 22 | #include <string.h> |
1b76c13e | 23 | #include <unistd.h> |
cdb2f920 | 24 | #include <endian.h> |
1a5e3fb1 WN |
25 | #include <fcntl.h> |
26 | #include <errno.h> | |
113e6b7e | 27 | #include <ctype.h> |
1b76c13e | 28 | #include <asm/unistd.h> |
e28ff1a8 | 29 | #include <linux/err.h> |
cb1e5e96 | 30 | #include <linux/kernel.h> |
1b76c13e | 31 | #include <linux/bpf.h> |
38d5d3b3 | 32 | #include <linux/btf.h> |
47eff617 | 33 | #include <linux/filter.h> |
9a208eff | 34 | #include <linux/list.h> |
f367540c | 35 | #include <linux/limits.h> |
438363c0 | 36 | #include <linux/perf_event.h> |
a64af0ef | 37 | #include <linux/ring_buffer.h> |
5e61f270 | 38 | #include <linux/version.h> |
fb84b822 | 39 | #include <sys/epoll.h> |
63f2f5ee | 40 | #include <sys/ioctl.h> |
fb84b822 | 41 | #include <sys/mman.h> |
f367540c JS |
42 | #include <sys/stat.h> |
43 | #include <sys/types.h> | |
44 | #include <sys/vfs.h> | |
ddc7c304 | 45 | #include <sys/utsname.h> |
dc3a2d25 | 46 | #include <sys/resource.h> |
531b014e | 47 | #include <tools/libc_compat.h> |
1a5e3fb1 WN |
48 | #include <libelf.h> |
49 | #include <gelf.h> | |
166750bc | 50 | #include <zlib.h> |
1b76c13e WN |
51 | |
52 | #include "libbpf.h" | |
52d3352e | 53 | #include "bpf.h" |
8a138aed | 54 | #include "btf.h" |
6d41907c | 55 | #include "str_error.h" |
d7c4b398 | 56 | #include "libbpf_internal.h" |
ddc7c304 | 57 | #include "hashmap.h" |
b3f59d66 | 58 | |
1d1a3bcf AN |
59 | /* make sure libbpf doesn't use kernel-only integer typedefs */ |
60 | #pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 | |
61 | ||
9b16137a WN |
62 | #ifndef EM_BPF |
63 | #define EM_BPF 247 | |
64 | #endif | |
65 | ||
f367540c JS |
66 | #ifndef BPF_FS_MAGIC |
67 | #define BPF_FS_MAGIC 0xcafe4a11 | |
68 | #endif | |
69 | ||
ff466b58 AI |
70 | /* vsprintf() in __base_pr() uses nonliteral format string. It may break |
71 | * compilation if user enables corresponding warning. Disable it explicitly. | |
72 | */ | |
73 | #pragma GCC diagnostic ignored "-Wformat-nonliteral" | |
74 | ||
b3f59d66 WN |
75 | #define __printf(a, b) __attribute__((format(printf, a, b))) |
76 | ||
590a0088 MKL |
77 | static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); |
78 | static struct bpf_program *bpf_object__find_prog_by_idx(struct bpf_object *obj, | |
79 | int idx); | |
80 | static const struct btf_type * | |
81 | skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id); | |
82 | ||
a8a1f7d0 SF |
83 | static int __base_pr(enum libbpf_print_level level, const char *format, |
84 | va_list args) | |
b3f59d66 | 85 | { |
6f1ae8b6 YS |
86 | if (level == LIBBPF_DEBUG) |
87 | return 0; | |
88 | ||
a8a1f7d0 | 89 | return vfprintf(stderr, format, args); |
b3f59d66 WN |
90 | } |
91 | ||
a8a1f7d0 | 92 | static libbpf_print_fn_t __libbpf_pr = __base_pr; |
b3f59d66 | 93 | |
e87fd8ba | 94 | libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) |
b3f59d66 | 95 | { |
e87fd8ba AN |
96 | libbpf_print_fn_t old_print_fn = __libbpf_pr; |
97 | ||
6f1ae8b6 | 98 | __libbpf_pr = fn; |
e87fd8ba | 99 | return old_print_fn; |
b3f59d66 | 100 | } |
1a5e3fb1 | 101 | |
8461ef8b YS |
102 | __printf(2, 3) |
103 | void libbpf_print(enum libbpf_print_level level, const char *format, ...) | |
104 | { | |
105 | va_list args; | |
106 | ||
6f1ae8b6 YS |
107 | if (!__libbpf_pr) |
108 | return; | |
109 | ||
8461ef8b | 110 | va_start(args, format); |
6f1ae8b6 | 111 | __libbpf_pr(level, format, args); |
8461ef8b YS |
112 | va_end(args); |
113 | } | |
114 | ||
dc3a2d25 THJ |
115 | static void pr_perm_msg(int err) |
116 | { | |
117 | struct rlimit limit; | |
118 | char buf[100]; | |
119 | ||
120 | if (err != -EPERM || geteuid() != 0) | |
121 | return; | |
122 | ||
123 | err = getrlimit(RLIMIT_MEMLOCK, &limit); | |
124 | if (err) | |
125 | return; | |
126 | ||
127 | if (limit.rlim_cur == RLIM_INFINITY) | |
128 | return; | |
129 | ||
130 | if (limit.rlim_cur < 1024) | |
b5c7d0d0 | 131 | snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); |
dc3a2d25 THJ |
132 | else if (limit.rlim_cur < 1024*1024) |
133 | snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); | |
134 | else | |
135 | snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); | |
136 | ||
137 | pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", | |
138 | buf); | |
139 | } | |
140 | ||
6371ca3b WN |
141 | #define STRERR_BUFSIZE 128 |
142 | ||
1a5e3fb1 WN |
143 | /* Copied from tools/perf/util/util.h */ |
144 | #ifndef zfree | |
145 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
146 | #endif | |
147 | ||
148 | #ifndef zclose | |
149 | # define zclose(fd) ({ \ | |
150 | int ___err = 0; \ | |
151 | if ((fd) >= 0) \ | |
152 | ___err = close((fd)); \ | |
153 | fd = -1; \ | |
154 | ___err; }) | |
155 | #endif | |
156 | ||
157 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
158 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
159 | #else | |
160 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
161 | #endif | |
162 | ||
34be1646 SL |
163 | static inline __u64 ptr_to_u64(const void *ptr) |
164 | { | |
165 | return (__u64) (unsigned long) ptr; | |
166 | } | |
167 | ||
47eff617 SF |
168 | struct bpf_capabilities { |
169 | /* v4.14: kernel support for program & map names. */ | |
170 | __u32 name:1; | |
8837fe5d DB |
171 | /* v5.2: kernel support for global data sections. */ |
172 | __u32 global_data:1; | |
d7c4b398 AN |
173 | /* BTF_KIND_FUNC and BTF_KIND_FUNC_PROTO support */ |
174 | __u32 btf_func:1; | |
175 | /* BTF_KIND_VAR and BTF_KIND_DATASEC support */ | |
176 | __u32 btf_datasec:1; | |
7fe74b43 AN |
177 | /* BPF_F_MMAPABLE is supported for arrays */ |
178 | __u32 array_mmap:1; | |
2d3eb67f AS |
179 | /* BTF_FUNC_GLOBAL is supported */ |
180 | __u32 btf_func_global:1; | |
47eff617 SF |
181 | }; |
182 | ||
166750bc AN |
183 | enum reloc_type { |
184 | RELO_LD64, | |
185 | RELO_CALL, | |
186 | RELO_DATA, | |
187 | RELO_EXTERN, | |
188 | }; | |
189 | ||
190 | struct reloc_desc { | |
191 | enum reloc_type type; | |
192 | int insn_idx; | |
193 | int map_idx; | |
194 | int sym_off; | |
195 | }; | |
196 | ||
a5b8bd47 WN |
197 | /* |
198 | * bpf_prog should be a better name but it has been used in | |
199 | * linux/filter.h. | |
200 | */ | |
201 | struct bpf_program { | |
202 | /* Index in elf obj file, for relocation use. */ | |
203 | int idx; | |
88cda1c9 | 204 | char *name; |
f0307a7e | 205 | int prog_ifindex; |
a5b8bd47 | 206 | char *section_name; |
33a2c75c SF |
207 | /* section_name with / replaced by _; makes recursive pinning |
208 | * in bpf_object__pin_programs easier | |
209 | */ | |
210 | char *pin_name; | |
a5b8bd47 | 211 | struct bpf_insn *insns; |
48cca7e4 | 212 | size_t insns_cnt, main_prog_cnt; |
5f44e4c8 | 213 | enum bpf_prog_type type; |
34090915 | 214 | |
166750bc | 215 | struct reloc_desc *reloc_desc; |
34090915 | 216 | int nr_reloc; |
da11b417 | 217 | int log_level; |
55cffde2 | 218 | |
b580563e WN |
219 | struct { |
220 | int nr; | |
221 | int *fds; | |
222 | } instances; | |
223 | bpf_program_prep_t preprocessor; | |
aa9b1ac3 WN |
224 | |
225 | struct bpf_object *obj; | |
226 | void *priv; | |
227 | bpf_program_clear_priv_t clear_priv; | |
d7be143b AI |
228 | |
229 | enum bpf_attach_type expected_attach_type; | |
12a8654b | 230 | __u32 attach_btf_id; |
e7bf94db | 231 | __u32 attach_prog_fd; |
2993e051 YS |
232 | void *func_info; |
233 | __u32 func_info_rec_size; | |
f0187f0b | 234 | __u32 func_info_cnt; |
47eff617 SF |
235 | |
236 | struct bpf_capabilities *caps; | |
3d650141 MKL |
237 | |
238 | void *line_info; | |
239 | __u32 line_info_rec_size; | |
240 | __u32 line_info_cnt; | |
04656198 | 241 | __u32 prog_flags; |
a5b8bd47 WN |
242 | }; |
243 | ||
590a0088 MKL |
244 | struct bpf_struct_ops { |
245 | const char *tname; | |
246 | const struct btf_type *type; | |
247 | struct bpf_program **progs; | |
248 | __u32 *kern_func_off; | |
249 | /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ | |
250 | void *data; | |
251 | /* e.g. struct bpf_struct_ops_tcp_congestion_ops in | |
252 | * btf_vmlinux's format. | |
253 | * struct bpf_struct_ops_tcp_congestion_ops { | |
254 | * [... some other kernel fields ...] | |
255 | * struct tcp_congestion_ops data; | |
256 | * } | |
257 | * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) | |
258 | * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" | |
259 | * from "data". | |
260 | */ | |
261 | void *kern_vdata; | |
262 | __u32 type_id; | |
263 | }; | |
264 | ||
ac9d1389 AN |
265 | #define DATA_SEC ".data" |
266 | #define BSS_SEC ".bss" | |
267 | #define RODATA_SEC ".rodata" | |
81bfdd08 | 268 | #define KCONFIG_SEC ".kconfig" |
590a0088 | 269 | #define STRUCT_OPS_SEC ".struct_ops" |
ac9d1389 | 270 | |
d859900c DB |
271 | enum libbpf_map_type { |
272 | LIBBPF_MAP_UNSPEC, | |
273 | LIBBPF_MAP_DATA, | |
274 | LIBBPF_MAP_BSS, | |
275 | LIBBPF_MAP_RODATA, | |
81bfdd08 | 276 | LIBBPF_MAP_KCONFIG, |
d859900c DB |
277 | }; |
278 | ||
279 | static const char * const libbpf_type_to_btf_name[] = { | |
ac9d1389 AN |
280 | [LIBBPF_MAP_DATA] = DATA_SEC, |
281 | [LIBBPF_MAP_BSS] = BSS_SEC, | |
282 | [LIBBPF_MAP_RODATA] = RODATA_SEC, | |
81bfdd08 | 283 | [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC, |
d859900c DB |
284 | }; |
285 | ||
9d759a9b | 286 | struct bpf_map { |
561bbcca | 287 | char *name; |
01af3bf0 | 288 | int fd; |
db48814b AN |
289 | int sec_idx; |
290 | size_t sec_offset; | |
f0307a7e | 291 | int map_ifindex; |
addb9fc9 | 292 | int inner_map_fd; |
9d759a9b | 293 | struct bpf_map_def def; |
5b891af7 MKL |
294 | __u32 btf_key_type_id; |
295 | __u32 btf_value_type_id; | |
590a0088 | 296 | __u32 btf_vmlinux_value_type_id; |
9d759a9b WN |
297 | void *priv; |
298 | bpf_map_clear_priv_t clear_priv; | |
d859900c | 299 | enum libbpf_map_type libbpf_type; |
eba9c5f4 | 300 | void *mmaped; |
590a0088 | 301 | struct bpf_struct_ops *st_ops; |
4580b25f THJ |
302 | char *pin_path; |
303 | bool pinned; | |
ec6d5f47 | 304 | bool reused; |
d859900c DB |
305 | }; |
306 | ||
166750bc AN |
307 | enum extern_type { |
308 | EXT_UNKNOWN, | |
309 | EXT_CHAR, | |
310 | EXT_BOOL, | |
311 | EXT_INT, | |
312 | EXT_TRISTATE, | |
313 | EXT_CHAR_ARR, | |
314 | }; | |
315 | ||
316 | struct extern_desc { | |
317 | const char *name; | |
318 | int sym_idx; | |
319 | int btf_id; | |
320 | enum extern_type type; | |
321 | int sz; | |
322 | int align; | |
323 | int data_off; | |
324 | bool is_signed; | |
325 | bool is_weak; | |
326 | bool is_set; | |
327 | }; | |
328 | ||
9a208eff WN |
329 | static LIST_HEAD(bpf_objects_list); |
330 | ||
1a5e3fb1 | 331 | struct bpf_object { |
d859900c | 332 | char name[BPF_OBJ_NAME_LEN]; |
cb1e5e96 | 333 | char license[64]; |
438363c0 | 334 | __u32 kern_version; |
0b3d1efa | 335 | |
a5b8bd47 WN |
336 | struct bpf_program *programs; |
337 | size_t nr_programs; | |
9d759a9b WN |
338 | struct bpf_map *maps; |
339 | size_t nr_maps; | |
bf829271 | 340 | size_t maps_cap; |
9d759a9b | 341 | |
8601fd42 | 342 | char *kconfig; |
166750bc AN |
343 | struct extern_desc *externs; |
344 | int nr_extern; | |
81bfdd08 | 345 | int kconfig_map_idx; |
166750bc | 346 | |
52d3352e | 347 | bool loaded; |
9a94f277 | 348 | bool has_pseudo_calls; |
a5b8bd47 | 349 | |
1a5e3fb1 WN |
350 | /* |
351 | * Information when doing elf related work. Only valid if fd | |
352 | * is valid. | |
353 | */ | |
354 | struct { | |
355 | int fd; | |
5e61f270 | 356 | const void *obj_buf; |
6c956392 | 357 | size_t obj_buf_sz; |
1a5e3fb1 WN |
358 | Elf *elf; |
359 | GElf_Ehdr ehdr; | |
bec7d68c | 360 | Elf_Data *symbols; |
d859900c DB |
361 | Elf_Data *data; |
362 | Elf_Data *rodata; | |
363 | Elf_Data *bss; | |
590a0088 | 364 | Elf_Data *st_ops_data; |
77ba9a5b | 365 | size_t strtabidx; |
b62f06e8 WN |
366 | struct { |
367 | GElf_Shdr shdr; | |
368 | Elf_Data *data; | |
1f8e2bcb AN |
369 | } *reloc_sects; |
370 | int nr_reloc_sects; | |
666810e8 | 371 | int maps_shndx; |
abd29c93 | 372 | int btf_maps_shndx; |
48cca7e4 | 373 | int text_shndx; |
166750bc | 374 | int symbols_shndx; |
d859900c DB |
375 | int data_shndx; |
376 | int rodata_shndx; | |
377 | int bss_shndx; | |
590a0088 | 378 | int st_ops_shndx; |
1a5e3fb1 | 379 | } efile; |
9a208eff WN |
380 | /* |
381 | * All loaded bpf_object is linked in a list, which is | |
382 | * hidden to caller. bpf_objects__<func> handlers deal with | |
383 | * all objects. | |
384 | */ | |
385 | struct list_head list; | |
10931d24 | 386 | |
8a138aed | 387 | struct btf *btf; |
a6ed02ca KS |
388 | /* Parse and load BTF vmlinux if any of the programs in the object need |
389 | * it at load time. | |
390 | */ | |
391 | struct btf *btf_vmlinux; | |
2993e051 | 392 | struct btf_ext *btf_ext; |
8a138aed | 393 | |
10931d24 WN |
394 | void *priv; |
395 | bpf_object_clear_priv_t clear_priv; | |
396 | ||
47eff617 SF |
397 | struct bpf_capabilities caps; |
398 | ||
1a5e3fb1 WN |
399 | char path[]; |
400 | }; | |
401 | #define obj_elf_valid(o) ((o)->efile.elf) | |
402 | ||
29cd77f4 | 403 | void bpf_program__unload(struct bpf_program *prog) |
55cffde2 | 404 | { |
b580563e WN |
405 | int i; |
406 | ||
55cffde2 WN |
407 | if (!prog) |
408 | return; | |
409 | ||
b580563e WN |
410 | /* |
411 | * If the object is opened but the program was never loaded, | |
412 | * it is possible that prog->instances.nr == -1. | |
413 | */ | |
414 | if (prog->instances.nr > 0) { | |
415 | for (i = 0; i < prog->instances.nr; i++) | |
416 | zclose(prog->instances.fds[i]); | |
417 | } else if (prog->instances.nr != -1) { | |
be18010e KW |
418 | pr_warn("Internal error: instances.nr is %d\n", |
419 | prog->instances.nr); | |
b580563e WN |
420 | } |
421 | ||
422 | prog->instances.nr = -1; | |
423 | zfree(&prog->instances.fds); | |
2993e051 | 424 | |
2993e051 | 425 | zfree(&prog->func_info); |
07a09d1b | 426 | zfree(&prog->line_info); |
55cffde2 WN |
427 | } |
428 | ||
a5b8bd47 WN |
429 | static void bpf_program__exit(struct bpf_program *prog) |
430 | { | |
431 | if (!prog) | |
432 | return; | |
433 | ||
aa9b1ac3 WN |
434 | if (prog->clear_priv) |
435 | prog->clear_priv(prog, prog->priv); | |
436 | ||
437 | prog->priv = NULL; | |
438 | prog->clear_priv = NULL; | |
439 | ||
55cffde2 | 440 | bpf_program__unload(prog); |
88cda1c9 | 441 | zfree(&prog->name); |
a5b8bd47 | 442 | zfree(&prog->section_name); |
33a2c75c | 443 | zfree(&prog->pin_name); |
a5b8bd47 | 444 | zfree(&prog->insns); |
34090915 WN |
445 | zfree(&prog->reloc_desc); |
446 | ||
447 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
448 | prog->insns_cnt = 0; |
449 | prog->idx = -1; | |
450 | } | |
451 | ||
33a2c75c SF |
452 | static char *__bpf_program__pin_name(struct bpf_program *prog) |
453 | { | |
454 | char *name, *p; | |
455 | ||
456 | name = p = strdup(prog->section_name); | |
457 | while ((p = strchr(p, '/'))) | |
458 | *p = '_'; | |
459 | ||
460 | return name; | |
461 | } | |
462 | ||
a5b8bd47 | 463 | static int |
88cda1c9 MKL |
464 | bpf_program__init(void *data, size_t size, char *section_name, int idx, |
465 | struct bpf_program *prog) | |
a5b8bd47 | 466 | { |
8ca990ce AN |
467 | const size_t bpf_insn_sz = sizeof(struct bpf_insn); |
468 | ||
469 | if (size == 0 || size % bpf_insn_sz) { | |
be18010e KW |
470 | pr_warn("corrupted section '%s', size: %zu\n", |
471 | section_name, size); | |
a5b8bd47 WN |
472 | return -EINVAL; |
473 | } | |
474 | ||
1ad9cbb8 | 475 | memset(prog, 0, sizeof(*prog)); |
a5b8bd47 | 476 | |
88cda1c9 | 477 | prog->section_name = strdup(section_name); |
a5b8bd47 | 478 | if (!prog->section_name) { |
be18010e KW |
479 | pr_warn("failed to alloc name for prog under section(%d) %s\n", |
480 | idx, section_name); | |
a5b8bd47 WN |
481 | goto errout; |
482 | } | |
483 | ||
33a2c75c SF |
484 | prog->pin_name = __bpf_program__pin_name(prog); |
485 | if (!prog->pin_name) { | |
be18010e KW |
486 | pr_warn("failed to alloc pin name for prog under section(%d) %s\n", |
487 | idx, section_name); | |
33a2c75c SF |
488 | goto errout; |
489 | } | |
490 | ||
a5b8bd47 WN |
491 | prog->insns = malloc(size); |
492 | if (!prog->insns) { | |
be18010e KW |
493 | pr_warn("failed to alloc insns for prog under section %s\n", |
494 | section_name); | |
a5b8bd47 WN |
495 | goto errout; |
496 | } | |
8ca990ce AN |
497 | prog->insns_cnt = size / bpf_insn_sz; |
498 | memcpy(prog->insns, data, size); | |
a5b8bd47 | 499 | prog->idx = idx; |
b580563e WN |
500 | prog->instances.fds = NULL; |
501 | prog->instances.nr = -1; | |
47ae7e3d | 502 | prog->type = BPF_PROG_TYPE_UNSPEC; |
a5b8bd47 WN |
503 | |
504 | return 0; | |
505 | errout: | |
506 | bpf_program__exit(prog); | |
507 | return -ENOMEM; | |
508 | } | |
509 | ||
510 | static int | |
511 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
88cda1c9 | 512 | char *section_name, int idx) |
a5b8bd47 WN |
513 | { |
514 | struct bpf_program prog, *progs; | |
515 | int nr_progs, err; | |
516 | ||
88cda1c9 | 517 | err = bpf_program__init(data, size, section_name, idx, &prog); |
a5b8bd47 WN |
518 | if (err) |
519 | return err; | |
520 | ||
47eff617 | 521 | prog.caps = &obj->caps; |
a5b8bd47 WN |
522 | progs = obj->programs; |
523 | nr_progs = obj->nr_programs; | |
524 | ||
531b014e | 525 | progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); |
a5b8bd47 WN |
526 | if (!progs) { |
527 | /* | |
528 | * In this case the original obj->programs | |
529 | * is still valid, so don't need special treat for | |
530 | * bpf_close_object(). | |
531 | */ | |
be18010e KW |
532 | pr_warn("failed to alloc a new program under section '%s'\n", |
533 | section_name); | |
a5b8bd47 WN |
534 | bpf_program__exit(&prog); |
535 | return -ENOMEM; | |
536 | } | |
537 | ||
538 | pr_debug("found program %s\n", prog.section_name); | |
539 | obj->programs = progs; | |
540 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 541 | prog.obj = obj; |
a5b8bd47 WN |
542 | progs[nr_progs] = prog; |
543 | return 0; | |
544 | } | |
545 | ||
88cda1c9 MKL |
546 | static int |
547 | bpf_object__init_prog_names(struct bpf_object *obj) | |
548 | { | |
549 | Elf_Data *symbols = obj->efile.symbols; | |
550 | struct bpf_program *prog; | |
551 | size_t pi, si; | |
552 | ||
553 | for (pi = 0; pi < obj->nr_programs; pi++) { | |
48cca7e4 | 554 | const char *name = NULL; |
88cda1c9 MKL |
555 | |
556 | prog = &obj->programs[pi]; | |
557 | ||
558 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; | |
559 | si++) { | |
560 | GElf_Sym sym; | |
561 | ||
562 | if (!gelf_getsym(symbols, si, &sym)) | |
563 | continue; | |
564 | if (sym.st_shndx != prog->idx) | |
565 | continue; | |
fe4d44b2 RG |
566 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) |
567 | continue; | |
88cda1c9 MKL |
568 | |
569 | name = elf_strptr(obj->efile.elf, | |
570 | obj->efile.strtabidx, | |
571 | sym.st_name); | |
572 | if (!name) { | |
be18010e KW |
573 | pr_warn("failed to get sym name string for prog %s\n", |
574 | prog->section_name); | |
88cda1c9 MKL |
575 | return -LIBBPF_ERRNO__LIBELF; |
576 | } | |
577 | } | |
578 | ||
9a94f277 JK |
579 | if (!name && prog->idx == obj->efile.text_shndx) |
580 | name = ".text"; | |
581 | ||
88cda1c9 | 582 | if (!name) { |
be18010e KW |
583 | pr_warn("failed to find sym for prog %s\n", |
584 | prog->section_name); | |
88cda1c9 MKL |
585 | return -EINVAL; |
586 | } | |
9a94f277 | 587 | |
88cda1c9 MKL |
588 | prog->name = strdup(name); |
589 | if (!prog->name) { | |
be18010e KW |
590 | pr_warn("failed to allocate memory for prog sym %s\n", |
591 | name); | |
88cda1c9 MKL |
592 | return -ENOMEM; |
593 | } | |
594 | } | |
595 | ||
596 | return 0; | |
597 | } | |
598 | ||
5e61f270 AN |
599 | static __u32 get_kernel_version(void) |
600 | { | |
601 | __u32 major, minor, patch; | |
602 | struct utsname info; | |
603 | ||
604 | uname(&info); | |
605 | if (sscanf(info.release, "%u.%u.%u", &major, &minor, &patch) != 3) | |
606 | return 0; | |
607 | return KERNEL_VERSION(major, minor, patch); | |
608 | } | |
609 | ||
590a0088 MKL |
610 | static const struct btf_member * |
611 | find_member_by_offset(const struct btf_type *t, __u32 bit_offset) | |
612 | { | |
613 | struct btf_member *m; | |
614 | int i; | |
615 | ||
616 | for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { | |
617 | if (btf_member_bit_offset(t, i) == bit_offset) | |
618 | return m; | |
619 | } | |
620 | ||
621 | return NULL; | |
622 | } | |
623 | ||
624 | static const struct btf_member * | |
625 | find_member_by_name(const struct btf *btf, const struct btf_type *t, | |
626 | const char *name) | |
627 | { | |
628 | struct btf_member *m; | |
629 | int i; | |
630 | ||
631 | for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { | |
632 | if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) | |
633 | return m; | |
634 | } | |
635 | ||
636 | return NULL; | |
637 | } | |
638 | ||
639 | #define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" | |
a6ed02ca KS |
640 | static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, |
641 | const char *name, __u32 kind); | |
590a0088 MKL |
642 | |
643 | static int | |
644 | find_struct_ops_kern_types(const struct btf *btf, const char *tname, | |
645 | const struct btf_type **type, __u32 *type_id, | |
646 | const struct btf_type **vtype, __u32 *vtype_id, | |
647 | const struct btf_member **data_member) | |
648 | { | |
649 | const struct btf_type *kern_type, *kern_vtype; | |
650 | const struct btf_member *kern_data_member; | |
651 | __s32 kern_vtype_id, kern_type_id; | |
590a0088 MKL |
652 | __u32 i; |
653 | ||
654 | kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); | |
655 | if (kern_type_id < 0) { | |
656 | pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", | |
657 | tname); | |
658 | return kern_type_id; | |
659 | } | |
660 | kern_type = btf__type_by_id(btf, kern_type_id); | |
661 | ||
662 | /* Find the corresponding "map_value" type that will be used | |
663 | * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, | |
664 | * find "struct bpf_struct_ops_tcp_congestion_ops" from the | |
665 | * btf_vmlinux. | |
666 | */ | |
a6ed02ca KS |
667 | kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, |
668 | tname, BTF_KIND_STRUCT); | |
590a0088 | 669 | if (kern_vtype_id < 0) { |
a6ed02ca KS |
670 | pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", |
671 | STRUCT_OPS_VALUE_PREFIX, tname); | |
590a0088 MKL |
672 | return kern_vtype_id; |
673 | } | |
674 | kern_vtype = btf__type_by_id(btf, kern_vtype_id); | |
675 | ||
676 | /* Find "struct tcp_congestion_ops" from | |
677 | * struct bpf_struct_ops_tcp_congestion_ops { | |
678 | * [ ... ] | |
679 | * struct tcp_congestion_ops data; | |
680 | * } | |
681 | */ | |
682 | kern_data_member = btf_members(kern_vtype); | |
683 | for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { | |
684 | if (kern_data_member->type == kern_type_id) | |
685 | break; | |
686 | } | |
687 | if (i == btf_vlen(kern_vtype)) { | |
a6ed02ca KS |
688 | pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", |
689 | tname, STRUCT_OPS_VALUE_PREFIX, tname); | |
590a0088 MKL |
690 | return -EINVAL; |
691 | } | |
692 | ||
693 | *type = kern_type; | |
694 | *type_id = kern_type_id; | |
695 | *vtype = kern_vtype; | |
696 | *vtype_id = kern_vtype_id; | |
697 | *data_member = kern_data_member; | |
698 | ||
699 | return 0; | |
700 | } | |
701 | ||
702 | static bool bpf_map__is_struct_ops(const struct bpf_map *map) | |
703 | { | |
704 | return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; | |
705 | } | |
706 | ||
707 | /* Init the map's fields that depend on kern_btf */ | |
708 | static int bpf_map__init_kern_struct_ops(struct bpf_map *map, | |
709 | const struct btf *btf, | |
710 | const struct btf *kern_btf) | |
711 | { | |
712 | const struct btf_member *member, *kern_member, *kern_data_member; | |
713 | const struct btf_type *type, *kern_type, *kern_vtype; | |
714 | __u32 i, kern_type_id, kern_vtype_id, kern_data_off; | |
715 | struct bpf_struct_ops *st_ops; | |
716 | void *data, *kern_data; | |
717 | const char *tname; | |
718 | int err; | |
719 | ||
720 | st_ops = map->st_ops; | |
721 | type = st_ops->type; | |
722 | tname = st_ops->tname; | |
723 | err = find_struct_ops_kern_types(kern_btf, tname, | |
724 | &kern_type, &kern_type_id, | |
725 | &kern_vtype, &kern_vtype_id, | |
726 | &kern_data_member); | |
727 | if (err) | |
728 | return err; | |
729 | ||
730 | pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", | |
731 | map->name, st_ops->type_id, kern_type_id, kern_vtype_id); | |
732 | ||
733 | map->def.value_size = kern_vtype->size; | |
734 | map->btf_vmlinux_value_type_id = kern_vtype_id; | |
735 | ||
736 | st_ops->kern_vdata = calloc(1, kern_vtype->size); | |
737 | if (!st_ops->kern_vdata) | |
738 | return -ENOMEM; | |
739 | ||
740 | data = st_ops->data; | |
741 | kern_data_off = kern_data_member->offset / 8; | |
742 | kern_data = st_ops->kern_vdata + kern_data_off; | |
743 | ||
744 | member = btf_members(type); | |
745 | for (i = 0; i < btf_vlen(type); i++, member++) { | |
746 | const struct btf_type *mtype, *kern_mtype; | |
747 | __u32 mtype_id, kern_mtype_id; | |
748 | void *mdata, *kern_mdata; | |
749 | __s64 msize, kern_msize; | |
750 | __u32 moff, kern_moff; | |
751 | __u32 kern_member_idx; | |
752 | const char *mname; | |
753 | ||
754 | mname = btf__name_by_offset(btf, member->name_off); | |
755 | kern_member = find_member_by_name(kern_btf, kern_type, mname); | |
756 | if (!kern_member) { | |
757 | pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", | |
758 | map->name, mname); | |
759 | return -ENOTSUP; | |
760 | } | |
761 | ||
762 | kern_member_idx = kern_member - btf_members(kern_type); | |
763 | if (btf_member_bitfield_size(type, i) || | |
764 | btf_member_bitfield_size(kern_type, kern_member_idx)) { | |
765 | pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", | |
766 | map->name, mname); | |
767 | return -ENOTSUP; | |
768 | } | |
769 | ||
770 | moff = member->offset / 8; | |
771 | kern_moff = kern_member->offset / 8; | |
772 | ||
773 | mdata = data + moff; | |
774 | kern_mdata = kern_data + kern_moff; | |
775 | ||
776 | mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); | |
777 | kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, | |
778 | &kern_mtype_id); | |
779 | if (BTF_INFO_KIND(mtype->info) != | |
780 | BTF_INFO_KIND(kern_mtype->info)) { | |
781 | pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", | |
782 | map->name, mname, BTF_INFO_KIND(mtype->info), | |
783 | BTF_INFO_KIND(kern_mtype->info)); | |
784 | return -ENOTSUP; | |
785 | } | |
786 | ||
787 | if (btf_is_ptr(mtype)) { | |
788 | struct bpf_program *prog; | |
789 | ||
790 | mtype = skip_mods_and_typedefs(btf, mtype->type, &mtype_id); | |
791 | kern_mtype = skip_mods_and_typedefs(kern_btf, | |
792 | kern_mtype->type, | |
793 | &kern_mtype_id); | |
794 | if (!btf_is_func_proto(mtype) || | |
795 | !btf_is_func_proto(kern_mtype)) { | |
796 | pr_warn("struct_ops init_kern %s: non func ptr %s is not supported\n", | |
797 | map->name, mname); | |
798 | return -ENOTSUP; | |
799 | } | |
800 | ||
801 | prog = st_ops->progs[i]; | |
802 | if (!prog) { | |
803 | pr_debug("struct_ops init_kern %s: func ptr %s is not set\n", | |
804 | map->name, mname); | |
805 | continue; | |
806 | } | |
807 | ||
808 | prog->attach_btf_id = kern_type_id; | |
809 | prog->expected_attach_type = kern_member_idx; | |
810 | ||
811 | st_ops->kern_func_off[i] = kern_data_off + kern_moff; | |
812 | ||
813 | pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", | |
814 | map->name, mname, prog->name, moff, | |
815 | kern_moff); | |
816 | ||
817 | continue; | |
818 | } | |
819 | ||
820 | msize = btf__resolve_size(btf, mtype_id); | |
821 | kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); | |
822 | if (msize < 0 || kern_msize < 0 || msize != kern_msize) { | |
823 | pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", | |
824 | map->name, mname, (ssize_t)msize, | |
825 | (ssize_t)kern_msize); | |
826 | return -ENOTSUP; | |
827 | } | |
828 | ||
829 | pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", | |
830 | map->name, mname, (unsigned int)msize, | |
831 | moff, kern_moff); | |
832 | memcpy(kern_mdata, mdata, msize); | |
833 | } | |
834 | ||
835 | return 0; | |
836 | } | |
837 | ||
838 | static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) | |
839 | { | |
590a0088 MKL |
840 | struct bpf_map *map; |
841 | size_t i; | |
842 | int err; | |
843 | ||
844 | for (i = 0; i < obj->nr_maps; i++) { | |
845 | map = &obj->maps[i]; | |
846 | ||
847 | if (!bpf_map__is_struct_ops(map)) | |
848 | continue; | |
849 | ||
a6ed02ca KS |
850 | err = bpf_map__init_kern_struct_ops(map, obj->btf, |
851 | obj->btf_vmlinux); | |
852 | if (err) | |
590a0088 | 853 | return err; |
590a0088 MKL |
854 | } |
855 | ||
590a0088 MKL |
856 | return 0; |
857 | } | |
858 | ||
859 | static int bpf_object__init_struct_ops_maps(struct bpf_object *obj) | |
860 | { | |
861 | const struct btf_type *type, *datasec; | |
862 | const struct btf_var_secinfo *vsi; | |
863 | struct bpf_struct_ops *st_ops; | |
864 | const char *tname, *var_name; | |
865 | __s32 type_id, datasec_id; | |
866 | const struct btf *btf; | |
867 | struct bpf_map *map; | |
868 | __u32 i; | |
869 | ||
870 | if (obj->efile.st_ops_shndx == -1) | |
871 | return 0; | |
872 | ||
873 | btf = obj->btf; | |
874 | datasec_id = btf__find_by_name_kind(btf, STRUCT_OPS_SEC, | |
875 | BTF_KIND_DATASEC); | |
876 | if (datasec_id < 0) { | |
877 | pr_warn("struct_ops init: DATASEC %s not found\n", | |
878 | STRUCT_OPS_SEC); | |
879 | return -EINVAL; | |
880 | } | |
881 | ||
882 | datasec = btf__type_by_id(btf, datasec_id); | |
883 | vsi = btf_var_secinfos(datasec); | |
884 | for (i = 0; i < btf_vlen(datasec); i++, vsi++) { | |
885 | type = btf__type_by_id(obj->btf, vsi->type); | |
886 | var_name = btf__name_by_offset(obj->btf, type->name_off); | |
887 | ||
888 | type_id = btf__resolve_type(obj->btf, vsi->type); | |
889 | if (type_id < 0) { | |
890 | pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", | |
891 | vsi->type, STRUCT_OPS_SEC); | |
892 | return -EINVAL; | |
893 | } | |
894 | ||
895 | type = btf__type_by_id(obj->btf, type_id); | |
896 | tname = btf__name_by_offset(obj->btf, type->name_off); | |
897 | if (!tname[0]) { | |
898 | pr_warn("struct_ops init: anonymous type is not supported\n"); | |
899 | return -ENOTSUP; | |
900 | } | |
901 | if (!btf_is_struct(type)) { | |
902 | pr_warn("struct_ops init: %s is not a struct\n", tname); | |
903 | return -EINVAL; | |
904 | } | |
905 | ||
906 | map = bpf_object__add_map(obj); | |
907 | if (IS_ERR(map)) | |
908 | return PTR_ERR(map); | |
909 | ||
910 | map->sec_idx = obj->efile.st_ops_shndx; | |
911 | map->sec_offset = vsi->offset; | |
912 | map->name = strdup(var_name); | |
913 | if (!map->name) | |
914 | return -ENOMEM; | |
915 | ||
916 | map->def.type = BPF_MAP_TYPE_STRUCT_OPS; | |
917 | map->def.key_size = sizeof(int); | |
918 | map->def.value_size = type->size; | |
919 | map->def.max_entries = 1; | |
920 | ||
921 | map->st_ops = calloc(1, sizeof(*map->st_ops)); | |
922 | if (!map->st_ops) | |
923 | return -ENOMEM; | |
924 | st_ops = map->st_ops; | |
925 | st_ops->data = malloc(type->size); | |
926 | st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); | |
927 | st_ops->kern_func_off = malloc(btf_vlen(type) * | |
928 | sizeof(*st_ops->kern_func_off)); | |
929 | if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) | |
930 | return -ENOMEM; | |
931 | ||
932 | if (vsi->offset + type->size > obj->efile.st_ops_data->d_size) { | |
933 | pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", | |
934 | var_name, STRUCT_OPS_SEC); | |
935 | return -EINVAL; | |
936 | } | |
937 | ||
938 | memcpy(st_ops->data, | |
939 | obj->efile.st_ops_data->d_buf + vsi->offset, | |
940 | type->size); | |
941 | st_ops->tname = tname; | |
942 | st_ops->type = type; | |
943 | st_ops->type_id = type_id; | |
944 | ||
945 | pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", | |
946 | tname, type_id, var_name, vsi->offset); | |
947 | } | |
948 | ||
949 | return 0; | |
950 | } | |
951 | ||
6c956392 | 952 | static struct bpf_object *bpf_object__new(const char *path, |
5e61f270 | 953 | const void *obj_buf, |
2ce8450e AN |
954 | size_t obj_buf_sz, |
955 | const char *obj_name) | |
1a5e3fb1 WN |
956 | { |
957 | struct bpf_object *obj; | |
d859900c | 958 | char *end; |
1a5e3fb1 WN |
959 | |
960 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
961 | if (!obj) { | |
be18010e | 962 | pr_warn("alloc memory failed for %s\n", path); |
6371ca3b | 963 | return ERR_PTR(-ENOMEM); |
1a5e3fb1 WN |
964 | } |
965 | ||
966 | strcpy(obj->path, path); | |
2ce8450e AN |
967 | if (obj_name) { |
968 | strncpy(obj->name, obj_name, sizeof(obj->name) - 1); | |
969 | obj->name[sizeof(obj->name) - 1] = 0; | |
970 | } else { | |
971 | /* Using basename() GNU version which doesn't modify arg. */ | |
972 | strncpy(obj->name, basename((void *)path), | |
973 | sizeof(obj->name) - 1); | |
974 | end = strchr(obj->name, '.'); | |
975 | if (end) | |
976 | *end = 0; | |
977 | } | |
6c956392 | 978 | |
d859900c | 979 | obj->efile.fd = -1; |
6c956392 | 980 | /* |
76e1022b | 981 | * Caller of this function should also call |
6c956392 WN |
982 | * bpf_object__elf_finish() after data collection to return |
983 | * obj_buf to user. If not, we should duplicate the buffer to | |
984 | * avoid user freeing them before elf finish. | |
985 | */ | |
986 | obj->efile.obj_buf = obj_buf; | |
987 | obj->efile.obj_buf_sz = obj_buf_sz; | |
666810e8 | 988 | obj->efile.maps_shndx = -1; |
abd29c93 | 989 | obj->efile.btf_maps_shndx = -1; |
d859900c DB |
990 | obj->efile.data_shndx = -1; |
991 | obj->efile.rodata_shndx = -1; | |
992 | obj->efile.bss_shndx = -1; | |
590a0088 | 993 | obj->efile.st_ops_shndx = -1; |
81bfdd08 | 994 | obj->kconfig_map_idx = -1; |
6c956392 | 995 | |
5e61f270 | 996 | obj->kern_version = get_kernel_version(); |
52d3352e | 997 | obj->loaded = false; |
9a208eff WN |
998 | |
999 | INIT_LIST_HEAD(&obj->list); | |
1000 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
1001 | return obj; |
1002 | } | |
1003 | ||
1004 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
1005 | { | |
1006 | if (!obj_elf_valid(obj)) | |
1007 | return; | |
1008 | ||
1009 | if (obj->efile.elf) { | |
1010 | elf_end(obj->efile.elf); | |
1011 | obj->efile.elf = NULL; | |
1012 | } | |
bec7d68c | 1013 | obj->efile.symbols = NULL; |
d859900c DB |
1014 | obj->efile.data = NULL; |
1015 | obj->efile.rodata = NULL; | |
1016 | obj->efile.bss = NULL; | |
590a0088 | 1017 | obj->efile.st_ops_data = NULL; |
b62f06e8 | 1018 | |
1f8e2bcb AN |
1019 | zfree(&obj->efile.reloc_sects); |
1020 | obj->efile.nr_reloc_sects = 0; | |
1a5e3fb1 | 1021 | zclose(obj->efile.fd); |
6c956392 WN |
1022 | obj->efile.obj_buf = NULL; |
1023 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
1024 | } |
1025 | ||
1026 | static int bpf_object__elf_init(struct bpf_object *obj) | |
1027 | { | |
1028 | int err = 0; | |
1029 | GElf_Ehdr *ep; | |
1030 | ||
1031 | if (obj_elf_valid(obj)) { | |
be18010e | 1032 | pr_warn("elf init: internal error\n"); |
6371ca3b | 1033 | return -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
1034 | } |
1035 | ||
6c956392 WN |
1036 | if (obj->efile.obj_buf_sz > 0) { |
1037 | /* | |
1038 | * obj_buf should have been validated by | |
1039 | * bpf_object__open_buffer(). | |
1040 | */ | |
5e61f270 | 1041 | obj->efile.elf = elf_memory((char *)obj->efile.obj_buf, |
6c956392 WN |
1042 | obj->efile.obj_buf_sz); |
1043 | } else { | |
1044 | obj->efile.fd = open(obj->path, O_RDONLY); | |
1045 | if (obj->efile.fd < 0) { | |
be5c5d4e | 1046 | char errmsg[STRERR_BUFSIZE], *cp; |
1ce6a9fc | 1047 | |
be5c5d4e AN |
1048 | err = -errno; |
1049 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
be18010e | 1050 | pr_warn("failed to open %s: %s\n", obj->path, cp); |
be5c5d4e | 1051 | return err; |
6c956392 WN |
1052 | } |
1053 | ||
1054 | obj->efile.elf = elf_begin(obj->efile.fd, | |
76e1022b | 1055 | LIBBPF_ELF_C_READ_MMAP, NULL); |
1a5e3fb1 WN |
1056 | } |
1057 | ||
1a5e3fb1 | 1058 | if (!obj->efile.elf) { |
be18010e | 1059 | pr_warn("failed to open %s as ELF file\n", obj->path); |
6371ca3b | 1060 | err = -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
1061 | goto errout; |
1062 | } | |
1063 | ||
1064 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
be18010e | 1065 | pr_warn("failed to get EHDR from %s\n", obj->path); |
6371ca3b | 1066 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
1067 | goto errout; |
1068 | } | |
1069 | ep = &obj->efile.ehdr; | |
1070 | ||
9b16137a | 1071 | /* Old LLVM set e_machine to EM_NONE */ |
76e1022b AN |
1072 | if (ep->e_type != ET_REL || |
1073 | (ep->e_machine && ep->e_machine != EM_BPF)) { | |
be18010e | 1074 | pr_warn("%s is not an eBPF object file\n", obj->path); |
6371ca3b | 1075 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
1076 | goto errout; |
1077 | } | |
1078 | ||
1079 | return 0; | |
1080 | errout: | |
1081 | bpf_object__elf_finish(obj); | |
1082 | return err; | |
1083 | } | |
1084 | ||
12ef5634 | 1085 | static int bpf_object__check_endianness(struct bpf_object *obj) |
cc4228d5 | 1086 | { |
cdb2f920 | 1087 | #if __BYTE_ORDER == __LITTLE_ENDIAN |
12ef5634 AN |
1088 | if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2LSB) |
1089 | return 0; | |
cdb2f920 | 1090 | #elif __BYTE_ORDER == __BIG_ENDIAN |
12ef5634 AN |
1091 | if (obj->efile.ehdr.e_ident[EI_DATA] == ELFDATA2MSB) |
1092 | return 0; | |
1093 | #else | |
1094 | # error "Unrecognized __BYTE_ORDER__" | |
1095 | #endif | |
be18010e | 1096 | pr_warn("endianness mismatch.\n"); |
6371ca3b | 1097 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
1098 | } |
1099 | ||
cb1e5e96 | 1100 | static int |
399dc65e | 1101 | bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) |
cb1e5e96 | 1102 | { |
399dc65e | 1103 | memcpy(obj->license, data, min(size, sizeof(obj->license) - 1)); |
cb1e5e96 WN |
1104 | pr_debug("license of %s is %s\n", obj->path, obj->license); |
1105 | return 0; | |
1106 | } | |
1107 | ||
54b8625c JF |
1108 | static int |
1109 | bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) | |
1110 | { | |
1111 | __u32 kver; | |
1112 | ||
1113 | if (size != sizeof(kver)) { | |
be18010e | 1114 | pr_warn("invalid kver section in %s\n", obj->path); |
54b8625c JF |
1115 | return -LIBBPF_ERRNO__FORMAT; |
1116 | } | |
1117 | memcpy(&kver, data, sizeof(kver)); | |
1118 | obj->kern_version = kver; | |
1119 | pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); | |
1120 | return 0; | |
1121 | } | |
1122 | ||
addb9fc9 NS |
1123 | static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) |
1124 | { | |
1125 | if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || | |
1126 | type == BPF_MAP_TYPE_HASH_OF_MAPS) | |
1127 | return true; | |
1128 | return false; | |
1129 | } | |
1130 | ||
1713d68b DB |
1131 | static int bpf_object_search_section_size(const struct bpf_object *obj, |
1132 | const char *name, size_t *d_size) | |
1133 | { | |
1134 | const GElf_Ehdr *ep = &obj->efile.ehdr; | |
1135 | Elf *elf = obj->efile.elf; | |
1136 | Elf_Scn *scn = NULL; | |
1137 | int idx = 0; | |
1138 | ||
1139 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
1140 | const char *sec_name; | |
1141 | Elf_Data *data; | |
1142 | GElf_Shdr sh; | |
1143 | ||
1144 | idx++; | |
1145 | if (gelf_getshdr(scn, &sh) != &sh) { | |
be18010e KW |
1146 | pr_warn("failed to get section(%d) header from %s\n", |
1147 | idx, obj->path); | |
1713d68b DB |
1148 | return -EIO; |
1149 | } | |
1150 | ||
1151 | sec_name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
1152 | if (!sec_name) { | |
be18010e KW |
1153 | pr_warn("failed to get section(%d) name from %s\n", |
1154 | idx, obj->path); | |
1713d68b DB |
1155 | return -EIO; |
1156 | } | |
1157 | ||
1158 | if (strcmp(name, sec_name)) | |
1159 | continue; | |
1160 | ||
1161 | data = elf_getdata(scn, 0); | |
1162 | if (!data) { | |
be18010e KW |
1163 | pr_warn("failed to get section(%d) data from %s(%s)\n", |
1164 | idx, name, obj->path); | |
1713d68b DB |
1165 | return -EIO; |
1166 | } | |
1167 | ||
1168 | *d_size = data->d_size; | |
1169 | return 0; | |
1170 | } | |
1171 | ||
1172 | return -ENOENT; | |
1173 | } | |
1174 | ||
1175 | int bpf_object__section_size(const struct bpf_object *obj, const char *name, | |
1176 | __u32 *size) | |
1177 | { | |
1178 | int ret = -ENOENT; | |
1179 | size_t d_size; | |
1180 | ||
1181 | *size = 0; | |
1182 | if (!name) { | |
1183 | return -EINVAL; | |
ac9d1389 | 1184 | } else if (!strcmp(name, DATA_SEC)) { |
1713d68b DB |
1185 | if (obj->efile.data) |
1186 | *size = obj->efile.data->d_size; | |
ac9d1389 | 1187 | } else if (!strcmp(name, BSS_SEC)) { |
1713d68b DB |
1188 | if (obj->efile.bss) |
1189 | *size = obj->efile.bss->d_size; | |
ac9d1389 | 1190 | } else if (!strcmp(name, RODATA_SEC)) { |
1713d68b DB |
1191 | if (obj->efile.rodata) |
1192 | *size = obj->efile.rodata->d_size; | |
590a0088 MKL |
1193 | } else if (!strcmp(name, STRUCT_OPS_SEC)) { |
1194 | if (obj->efile.st_ops_data) | |
1195 | *size = obj->efile.st_ops_data->d_size; | |
1713d68b DB |
1196 | } else { |
1197 | ret = bpf_object_search_section_size(obj, name, &d_size); | |
1198 | if (!ret) | |
1199 | *size = d_size; | |
1200 | } | |
1201 | ||
1202 | return *size ? 0 : ret; | |
1203 | } | |
1204 | ||
1205 | int bpf_object__variable_offset(const struct bpf_object *obj, const char *name, | |
1206 | __u32 *off) | |
1207 | { | |
1208 | Elf_Data *symbols = obj->efile.symbols; | |
1209 | const char *sname; | |
1210 | size_t si; | |
1211 | ||
1212 | if (!name || !off) | |
1213 | return -EINVAL; | |
1214 | ||
1215 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym); si++) { | |
1216 | GElf_Sym sym; | |
1217 | ||
1218 | if (!gelf_getsym(symbols, si, &sym)) | |
1219 | continue; | |
1220 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL || | |
1221 | GELF_ST_TYPE(sym.st_info) != STT_OBJECT) | |
1222 | continue; | |
1223 | ||
1224 | sname = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
1225 | sym.st_name); | |
1226 | if (!sname) { | |
be18010e KW |
1227 | pr_warn("failed to get sym name string for var %s\n", |
1228 | name); | |
1713d68b DB |
1229 | return -EIO; |
1230 | } | |
1231 | if (strcmp(name, sname) == 0) { | |
1232 | *off = sym.st_value; | |
1233 | return 0; | |
1234 | } | |
1235 | } | |
1236 | ||
1237 | return -ENOENT; | |
1238 | } | |
1239 | ||
bf829271 | 1240 | static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) |
d859900c | 1241 | { |
bf829271 AN |
1242 | struct bpf_map *new_maps; |
1243 | size_t new_cap; | |
1244 | int i; | |
1245 | ||
1246 | if (obj->nr_maps < obj->maps_cap) | |
1247 | return &obj->maps[obj->nr_maps++]; | |
1248 | ||
95064979 | 1249 | new_cap = max((size_t)4, obj->maps_cap * 3 / 2); |
bf829271 AN |
1250 | new_maps = realloc(obj->maps, new_cap * sizeof(*obj->maps)); |
1251 | if (!new_maps) { | |
be18010e | 1252 | pr_warn("alloc maps for object failed\n"); |
bf829271 AN |
1253 | return ERR_PTR(-ENOMEM); |
1254 | } | |
1255 | ||
1256 | obj->maps_cap = new_cap; | |
1257 | obj->maps = new_maps; | |
1258 | ||
1259 | /* zero out new maps */ | |
1260 | memset(obj->maps + obj->nr_maps, 0, | |
1261 | (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); | |
1262 | /* | |
1263 | * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) | |
1264 | * when failure (zclose won't close negative fd)). | |
1265 | */ | |
1266 | for (i = obj->nr_maps; i < obj->maps_cap; i++) { | |
1267 | obj->maps[i].fd = -1; | |
1268 | obj->maps[i].inner_map_fd = -1; | |
1269 | } | |
1270 | ||
1271 | return &obj->maps[obj->nr_maps++]; | |
d859900c DB |
1272 | } |
1273 | ||
eba9c5f4 AN |
1274 | static size_t bpf_map_mmap_sz(const struct bpf_map *map) |
1275 | { | |
1276 | long page_sz = sysconf(_SC_PAGE_SIZE); | |
1277 | size_t map_sz; | |
1278 | ||
c701917e | 1279 | map_sz = (size_t)roundup(map->def.value_size, 8) * map->def.max_entries; |
eba9c5f4 AN |
1280 | map_sz = roundup(map_sz, page_sz); |
1281 | return map_sz; | |
1282 | } | |
1283 | ||
81bfdd08 AN |
1284 | static char *internal_map_name(struct bpf_object *obj, |
1285 | enum libbpf_map_type type) | |
1286 | { | |
113e6b7e | 1287 | char map_name[BPF_OBJ_NAME_LEN], *p; |
81bfdd08 AN |
1288 | const char *sfx = libbpf_type_to_btf_name[type]; |
1289 | int sfx_len = max((size_t)7, strlen(sfx)); | |
1290 | int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, | |
1291 | strlen(obj->name)); | |
1292 | ||
1293 | snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, | |
1294 | sfx_len, libbpf_type_to_btf_name[type]); | |
1295 | ||
113e6b7e THJ |
1296 | /* sanitise map name to characters allowed by kernel */ |
1297 | for (p = map_name; *p && p < map_name + sizeof(map_name); p++) | |
1298 | if (!isalnum(*p) && *p != '_' && *p != '.') | |
1299 | *p = '_'; | |
1300 | ||
81bfdd08 AN |
1301 | return strdup(map_name); |
1302 | } | |
1303 | ||
d859900c | 1304 | static int |
bf829271 | 1305 | bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, |
eba9c5f4 | 1306 | int sec_idx, void *data, size_t data_sz) |
d859900c | 1307 | { |
bf829271 AN |
1308 | struct bpf_map_def *def; |
1309 | struct bpf_map *map; | |
eba9c5f4 | 1310 | int err; |
bf829271 AN |
1311 | |
1312 | map = bpf_object__add_map(obj); | |
1313 | if (IS_ERR(map)) | |
1314 | return PTR_ERR(map); | |
d859900c DB |
1315 | |
1316 | map->libbpf_type = type; | |
db48814b AN |
1317 | map->sec_idx = sec_idx; |
1318 | map->sec_offset = 0; | |
81bfdd08 | 1319 | map->name = internal_map_name(obj, type); |
d859900c | 1320 | if (!map->name) { |
be18010e | 1321 | pr_warn("failed to alloc map name\n"); |
d859900c DB |
1322 | return -ENOMEM; |
1323 | } | |
1324 | ||
bf829271 | 1325 | def = &map->def; |
d859900c DB |
1326 | def->type = BPF_MAP_TYPE_ARRAY; |
1327 | def->key_size = sizeof(int); | |
eba9c5f4 | 1328 | def->value_size = data_sz; |
d859900c | 1329 | def->max_entries = 1; |
81bfdd08 | 1330 | def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG |
166750bc | 1331 | ? BPF_F_RDONLY_PROG : 0; |
0d13bfce | 1332 | def->map_flags |= BPF_F_MMAPABLE; |
7fe74b43 AN |
1333 | |
1334 | pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", | |
81bfdd08 | 1335 | map->name, map->sec_idx, map->sec_offset, def->map_flags); |
7fe74b43 | 1336 | |
eba9c5f4 AN |
1337 | map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE, |
1338 | MAP_SHARED | MAP_ANONYMOUS, -1, 0); | |
1339 | if (map->mmaped == MAP_FAILED) { | |
1340 | err = -errno; | |
1341 | map->mmaped = NULL; | |
1342 | pr_warn("failed to alloc map '%s' content buffer: %d\n", | |
1343 | map->name, err); | |
1344 | zfree(&map->name); | |
1345 | return err; | |
d859900c DB |
1346 | } |
1347 | ||
166750bc | 1348 | if (data) |
eba9c5f4 AN |
1349 | memcpy(map->mmaped, data, data_sz); |
1350 | ||
e1d1dc46 | 1351 | pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); |
d859900c DB |
1352 | return 0; |
1353 | } | |
1354 | ||
bf829271 AN |
1355 | static int bpf_object__init_global_data_maps(struct bpf_object *obj) |
1356 | { | |
1357 | int err; | |
1358 | ||
bf829271 AN |
1359 | /* |
1360 | * Populate obj->maps with libbpf internal maps. | |
1361 | */ | |
1362 | if (obj->efile.data_shndx >= 0) { | |
1363 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, | |
db48814b | 1364 | obj->efile.data_shndx, |
eba9c5f4 AN |
1365 | obj->efile.data->d_buf, |
1366 | obj->efile.data->d_size); | |
bf829271 AN |
1367 | if (err) |
1368 | return err; | |
1369 | } | |
1370 | if (obj->efile.rodata_shndx >= 0) { | |
1371 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, | |
db48814b | 1372 | obj->efile.rodata_shndx, |
eba9c5f4 AN |
1373 | obj->efile.rodata->d_buf, |
1374 | obj->efile.rodata->d_size); | |
bf829271 AN |
1375 | if (err) |
1376 | return err; | |
1377 | } | |
1378 | if (obj->efile.bss_shndx >= 0) { | |
1379 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, | |
db48814b | 1380 | obj->efile.bss_shndx, |
eba9c5f4 AN |
1381 | NULL, |
1382 | obj->efile.bss->d_size); | |
bf829271 AN |
1383 | if (err) |
1384 | return err; | |
1385 | } | |
1386 | return 0; | |
1387 | } | |
1388 | ||
166750bc AN |
1389 | |
1390 | static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, | |
1391 | const void *name) | |
1392 | { | |
1393 | int i; | |
1394 | ||
1395 | for (i = 0; i < obj->nr_extern; i++) { | |
1396 | if (strcmp(obj->externs[i].name, name) == 0) | |
1397 | return &obj->externs[i]; | |
1398 | } | |
1399 | return NULL; | |
1400 | } | |
1401 | ||
1402 | static int set_ext_value_tri(struct extern_desc *ext, void *ext_val, | |
1403 | char value) | |
1404 | { | |
1405 | switch (ext->type) { | |
1406 | case EXT_BOOL: | |
1407 | if (value == 'm') { | |
1408 | pr_warn("extern %s=%c should be tristate or char\n", | |
1409 | ext->name, value); | |
1410 | return -EINVAL; | |
1411 | } | |
1412 | *(bool *)ext_val = value == 'y' ? true : false; | |
1413 | break; | |
1414 | case EXT_TRISTATE: | |
1415 | if (value == 'y') | |
1416 | *(enum libbpf_tristate *)ext_val = TRI_YES; | |
1417 | else if (value == 'm') | |
1418 | *(enum libbpf_tristate *)ext_val = TRI_MODULE; | |
1419 | else /* value == 'n' */ | |
1420 | *(enum libbpf_tristate *)ext_val = TRI_NO; | |
1421 | break; | |
1422 | case EXT_CHAR: | |
1423 | *(char *)ext_val = value; | |
1424 | break; | |
1425 | case EXT_UNKNOWN: | |
1426 | case EXT_INT: | |
1427 | case EXT_CHAR_ARR: | |
1428 | default: | |
1429 | pr_warn("extern %s=%c should be bool, tristate, or char\n", | |
1430 | ext->name, value); | |
1431 | return -EINVAL; | |
1432 | } | |
1433 | ext->is_set = true; | |
1434 | return 0; | |
1435 | } | |
1436 | ||
1437 | static int set_ext_value_str(struct extern_desc *ext, char *ext_val, | |
1438 | const char *value) | |
1439 | { | |
1440 | size_t len; | |
1441 | ||
1442 | if (ext->type != EXT_CHAR_ARR) { | |
1443 | pr_warn("extern %s=%s should char array\n", ext->name, value); | |
1444 | return -EINVAL; | |
1445 | } | |
1446 | ||
1447 | len = strlen(value); | |
1448 | if (value[len - 1] != '"') { | |
1449 | pr_warn("extern '%s': invalid string config '%s'\n", | |
1450 | ext->name, value); | |
1451 | return -EINVAL; | |
1452 | } | |
1453 | ||
1454 | /* strip quotes */ | |
1455 | len -= 2; | |
1456 | if (len >= ext->sz) { | |
1457 | pr_warn("extern '%s': long string config %s of (%zu bytes) truncated to %d bytes\n", | |
1458 | ext->name, value, len, ext->sz - 1); | |
1459 | len = ext->sz - 1; | |
1460 | } | |
1461 | memcpy(ext_val, value + 1, len); | |
1462 | ext_val[len] = '\0'; | |
1463 | ext->is_set = true; | |
1464 | return 0; | |
1465 | } | |
1466 | ||
1467 | static int parse_u64(const char *value, __u64 *res) | |
1468 | { | |
1469 | char *value_end; | |
1470 | int err; | |
1471 | ||
1472 | errno = 0; | |
1473 | *res = strtoull(value, &value_end, 0); | |
1474 | if (errno) { | |
1475 | err = -errno; | |
1476 | pr_warn("failed to parse '%s' as integer: %d\n", value, err); | |
1477 | return err; | |
1478 | } | |
1479 | if (*value_end) { | |
1480 | pr_warn("failed to parse '%s' as integer completely\n", value); | |
1481 | return -EINVAL; | |
1482 | } | |
1483 | return 0; | |
1484 | } | |
1485 | ||
1486 | static bool is_ext_value_in_range(const struct extern_desc *ext, __u64 v) | |
1487 | { | |
1488 | int bit_sz = ext->sz * 8; | |
1489 | ||
1490 | if (ext->sz == 8) | |
1491 | return true; | |
1492 | ||
1493 | /* Validate that value stored in u64 fits in integer of `ext->sz` | |
1494 | * bytes size without any loss of information. If the target integer | |
1495 | * is signed, we rely on the following limits of integer type of | |
1496 | * Y bits and subsequent transformation: | |
1497 | * | |
1498 | * -2^(Y-1) <= X <= 2^(Y-1) - 1 | |
1499 | * 0 <= X + 2^(Y-1) <= 2^Y - 1 | |
1500 | * 0 <= X + 2^(Y-1) < 2^Y | |
1501 | * | |
1502 | * For unsigned target integer, check that all the (64 - Y) bits are | |
1503 | * zero. | |
1504 | */ | |
1505 | if (ext->is_signed) | |
1506 | return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); | |
1507 | else | |
1508 | return (v >> bit_sz) == 0; | |
1509 | } | |
1510 | ||
1511 | static int set_ext_value_num(struct extern_desc *ext, void *ext_val, | |
1512 | __u64 value) | |
1513 | { | |
1514 | if (ext->type != EXT_INT && ext->type != EXT_CHAR) { | |
1515 | pr_warn("extern %s=%llu should be integer\n", | |
7745ff98 | 1516 | ext->name, (unsigned long long)value); |
166750bc AN |
1517 | return -EINVAL; |
1518 | } | |
1519 | if (!is_ext_value_in_range(ext, value)) { | |
1520 | pr_warn("extern %s=%llu value doesn't fit in %d bytes\n", | |
7745ff98 | 1521 | ext->name, (unsigned long long)value, ext->sz); |
166750bc AN |
1522 | return -ERANGE; |
1523 | } | |
1524 | switch (ext->sz) { | |
1525 | case 1: *(__u8 *)ext_val = value; break; | |
1526 | case 2: *(__u16 *)ext_val = value; break; | |
1527 | case 4: *(__u32 *)ext_val = value; break; | |
1528 | case 8: *(__u64 *)ext_val = value; break; | |
1529 | default: | |
1530 | return -EINVAL; | |
1531 | } | |
1532 | ext->is_set = true; | |
1533 | return 0; | |
1534 | } | |
1535 | ||
8601fd42 AN |
1536 | static int bpf_object__process_kconfig_line(struct bpf_object *obj, |
1537 | char *buf, void *data) | |
166750bc | 1538 | { |
166750bc | 1539 | struct extern_desc *ext; |
8601fd42 | 1540 | char *sep, *value; |
166750bc AN |
1541 | int len, err = 0; |
1542 | void *ext_val; | |
1543 | __u64 num; | |
166750bc | 1544 | |
8601fd42 AN |
1545 | if (strncmp(buf, "CONFIG_", 7)) |
1546 | return 0; | |
166750bc | 1547 | |
8601fd42 AN |
1548 | sep = strchr(buf, '='); |
1549 | if (!sep) { | |
1550 | pr_warn("failed to parse '%s': no separator\n", buf); | |
1551 | return -EINVAL; | |
1552 | } | |
1553 | ||
1554 | /* Trim ending '\n' */ | |
1555 | len = strlen(buf); | |
1556 | if (buf[len - 1] == '\n') | |
1557 | buf[len - 1] = '\0'; | |
1558 | /* Split on '=' and ensure that a value is present. */ | |
1559 | *sep = '\0'; | |
1560 | if (!sep[1]) { | |
1561 | *sep = '='; | |
1562 | pr_warn("failed to parse '%s': no value\n", buf); | |
1563 | return -EINVAL; | |
1564 | } | |
1565 | ||
1566 | ext = find_extern_by_name(obj, buf); | |
1567 | if (!ext || ext->is_set) | |
1568 | return 0; | |
1569 | ||
1570 | ext_val = data + ext->data_off; | |
1571 | value = sep + 1; | |
1572 | ||
1573 | switch (*value) { | |
1574 | case 'y': case 'n': case 'm': | |
1575 | err = set_ext_value_tri(ext, ext_val, *value); | |
1576 | break; | |
1577 | case '"': | |
1578 | err = set_ext_value_str(ext, ext_val, value); | |
1579 | break; | |
1580 | default: | |
1581 | /* assume integer */ | |
1582 | err = parse_u64(value, &num); | |
1583 | if (err) { | |
1584 | pr_warn("extern %s=%s should be integer\n", | |
1585 | ext->name, value); | |
1586 | return err; | |
1587 | } | |
1588 | err = set_ext_value_num(ext, ext_val, num); | |
1589 | break; | |
166750bc | 1590 | } |
8601fd42 AN |
1591 | if (err) |
1592 | return err; | |
1593 | pr_debug("extern %s=%s\n", ext->name, value); | |
1594 | return 0; | |
1595 | } | |
1596 | ||
1597 | static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) | |
1598 | { | |
1599 | char buf[PATH_MAX]; | |
1600 | struct utsname uts; | |
1601 | int len, err = 0; | |
1602 | gzFile file; | |
1603 | ||
1604 | uname(&uts); | |
1605 | len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); | |
1606 | if (len < 0) | |
1607 | return -EINVAL; | |
1608 | else if (len >= PATH_MAX) | |
1609 | return -ENAMETOOLONG; | |
1610 | ||
1611 | /* gzopen also accepts uncompressed files. */ | |
1612 | file = gzopen(buf, "r"); | |
1613 | if (!file) | |
1614 | file = gzopen("/proc/config.gz", "r"); | |
1615 | ||
166750bc | 1616 | if (!file) { |
8601fd42 | 1617 | pr_warn("failed to open system Kconfig\n"); |
166750bc AN |
1618 | return -ENOENT; |
1619 | } | |
1620 | ||
1621 | while (gzgets(file, buf, sizeof(buf))) { | |
8601fd42 AN |
1622 | err = bpf_object__process_kconfig_line(obj, buf, data); |
1623 | if (err) { | |
1624 | pr_warn("error parsing system Kconfig line '%s': %d\n", | |
1625 | buf, err); | |
166750bc AN |
1626 | goto out; |
1627 | } | |
8601fd42 | 1628 | } |
166750bc | 1629 | |
8601fd42 AN |
1630 | out: |
1631 | gzclose(file); | |
1632 | return err; | |
1633 | } | |
166750bc | 1634 | |
8601fd42 AN |
1635 | static int bpf_object__read_kconfig_mem(struct bpf_object *obj, |
1636 | const char *config, void *data) | |
1637 | { | |
1638 | char buf[PATH_MAX]; | |
1639 | int err = 0; | |
1640 | FILE *file; | |
166750bc | 1641 | |
8601fd42 AN |
1642 | file = fmemopen((void *)config, strlen(config), "r"); |
1643 | if (!file) { | |
1644 | err = -errno; | |
1645 | pr_warn("failed to open in-memory Kconfig: %d\n", err); | |
1646 | return err; | |
1647 | } | |
1648 | ||
1649 | while (fgets(buf, sizeof(buf), file)) { | |
1650 | err = bpf_object__process_kconfig_line(obj, buf, data); | |
1651 | if (err) { | |
1652 | pr_warn("error parsing in-memory Kconfig line '%s': %d\n", | |
1653 | buf, err); | |
166750bc AN |
1654 | break; |
1655 | } | |
166750bc AN |
1656 | } |
1657 | ||
8601fd42 | 1658 | fclose(file); |
166750bc AN |
1659 | return err; |
1660 | } | |
1661 | ||
81bfdd08 | 1662 | static int bpf_object__init_kconfig_map(struct bpf_object *obj) |
166750bc AN |
1663 | { |
1664 | struct extern_desc *last_ext; | |
1665 | size_t map_sz; | |
1666 | int err; | |
1667 | ||
1668 | if (obj->nr_extern == 0) | |
1669 | return 0; | |
1670 | ||
1671 | last_ext = &obj->externs[obj->nr_extern - 1]; | |
1672 | map_sz = last_ext->data_off + last_ext->sz; | |
1673 | ||
81bfdd08 | 1674 | err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, |
166750bc AN |
1675 | obj->efile.symbols_shndx, |
1676 | NULL, map_sz); | |
1677 | if (err) | |
1678 | return err; | |
1679 | ||
81bfdd08 | 1680 | obj->kconfig_map_idx = obj->nr_maps - 1; |
166750bc AN |
1681 | |
1682 | return 0; | |
1683 | } | |
1684 | ||
bf829271 | 1685 | static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) |
561bbcca | 1686 | { |
561bbcca | 1687 | Elf_Data *symbols = obj->efile.symbols; |
bf829271 | 1688 | int i, map_def_sz = 0, nr_maps = 0, nr_syms; |
d859900c | 1689 | Elf_Data *data = NULL; |
bf829271 AN |
1690 | Elf_Scn *scn; |
1691 | ||
1692 | if (obj->efile.maps_shndx < 0) | |
1693 | return 0; | |
561bbcca | 1694 | |
4708bbda EL |
1695 | if (!symbols) |
1696 | return -EINVAL; | |
1697 | ||
bf829271 AN |
1698 | scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); |
1699 | if (scn) | |
1700 | data = elf_getdata(scn, NULL); | |
1701 | if (!scn || !data) { | |
be18010e KW |
1702 | pr_warn("failed to get Elf_Data from map section %d\n", |
1703 | obj->efile.maps_shndx); | |
bf829271 | 1704 | return -EINVAL; |
4708bbda | 1705 | } |
561bbcca | 1706 | |
4708bbda EL |
1707 | /* |
1708 | * Count number of maps. Each map has a name. | |
1709 | * Array of maps is not supported: only the first element is | |
1710 | * considered. | |
1711 | * | |
1712 | * TODO: Detect array of map and report error. | |
1713 | */ | |
bf829271 AN |
1714 | nr_syms = symbols->d_size / sizeof(GElf_Sym); |
1715 | for (i = 0; i < nr_syms; i++) { | |
561bbcca | 1716 | GElf_Sym sym; |
4708bbda EL |
1717 | |
1718 | if (!gelf_getsym(symbols, i, &sym)) | |
1719 | continue; | |
1720 | if (sym.st_shndx != obj->efile.maps_shndx) | |
1721 | continue; | |
1722 | nr_maps++; | |
1723 | } | |
b13c5c14 | 1724 | /* Assume equally sized map definitions */ |
bf829271 AN |
1725 | pr_debug("maps in %s: %d maps in %zd bytes\n", |
1726 | obj->path, nr_maps, data->d_size); | |
1727 | ||
98e527af | 1728 | if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) { |
8983b731 | 1729 | pr_warn("unable to determine map definition size section %s, %d maps in %zd bytes\n", |
be18010e | 1730 | obj->path, nr_maps, data->d_size); |
bf829271 | 1731 | return -EINVAL; |
addb9fc9 | 1732 | } |
98e527af | 1733 | map_def_sz = data->d_size / nr_maps; |
4708bbda | 1734 | |
bf829271 AN |
1735 | /* Fill obj->maps using data in "maps" section. */ |
1736 | for (i = 0; i < nr_syms; i++) { | |
4708bbda | 1737 | GElf_Sym sym; |
561bbcca | 1738 | const char *map_name; |
4708bbda | 1739 | struct bpf_map_def *def; |
bf829271 | 1740 | struct bpf_map *map; |
561bbcca WN |
1741 | |
1742 | if (!gelf_getsym(symbols, i, &sym)) | |
1743 | continue; | |
666810e8 | 1744 | if (sym.st_shndx != obj->efile.maps_shndx) |
561bbcca WN |
1745 | continue; |
1746 | ||
bf829271 AN |
1747 | map = bpf_object__add_map(obj); |
1748 | if (IS_ERR(map)) | |
1749 | return PTR_ERR(map); | |
1750 | ||
1751 | map_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
561bbcca | 1752 | sym.st_name); |
c51829bb | 1753 | if (!map_name) { |
be18010e KW |
1754 | pr_warn("failed to get map #%d name sym string for obj %s\n", |
1755 | i, obj->path); | |
c51829bb AN |
1756 | return -LIBBPF_ERRNO__FORMAT; |
1757 | } | |
d859900c | 1758 | |
bf829271 | 1759 | map->libbpf_type = LIBBPF_MAP_UNSPEC; |
db48814b AN |
1760 | map->sec_idx = sym.st_shndx; |
1761 | map->sec_offset = sym.st_value; | |
1762 | pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", | |
1763 | map_name, map->sec_idx, map->sec_offset); | |
b13c5c14 | 1764 | if (sym.st_value + map_def_sz > data->d_size) { |
be18010e KW |
1765 | pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", |
1766 | obj->path, map_name); | |
4708bbda | 1767 | return -EINVAL; |
561bbcca | 1768 | } |
4708bbda | 1769 | |
bf829271 AN |
1770 | map->name = strdup(map_name); |
1771 | if (!map->name) { | |
be18010e | 1772 | pr_warn("failed to alloc map name\n"); |
973170e6 WN |
1773 | return -ENOMEM; |
1774 | } | |
bf829271 | 1775 | pr_debug("map %d is \"%s\"\n", i, map->name); |
4708bbda | 1776 | def = (struct bpf_map_def *)(data->d_buf + sym.st_value); |
b13c5c14 CG |
1777 | /* |
1778 | * If the definition of the map in the object file fits in | |
1779 | * bpf_map_def, copy it. Any extra fields in our version | |
1780 | * of bpf_map_def will default to zero as a result of the | |
1781 | * calloc above. | |
1782 | */ | |
1783 | if (map_def_sz <= sizeof(struct bpf_map_def)) { | |
bf829271 | 1784 | memcpy(&map->def, def, map_def_sz); |
b13c5c14 CG |
1785 | } else { |
1786 | /* | |
1787 | * Here the map structure being read is bigger than what | |
1788 | * we expect, truncate if the excess bits are all zero. | |
1789 | * If they are not zero, reject this map as | |
1790 | * incompatible. | |
1791 | */ | |
1792 | char *b; | |
8983b731 | 1793 | |
b13c5c14 CG |
1794 | for (b = ((char *)def) + sizeof(struct bpf_map_def); |
1795 | b < ((char *)def) + map_def_sz; b++) { | |
1796 | if (*b != 0) { | |
8983b731 | 1797 | pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n", |
be18010e | 1798 | obj->path, map_name); |
c034a177 JF |
1799 | if (strict) |
1800 | return -EINVAL; | |
b13c5c14 CG |
1801 | } |
1802 | } | |
bf829271 | 1803 | memcpy(&map->def, def, sizeof(struct bpf_map_def)); |
b13c5c14 | 1804 | } |
561bbcca | 1805 | } |
bf829271 AN |
1806 | return 0; |
1807 | } | |
4708bbda | 1808 | |
ddc7c304 AN |
1809 | static const struct btf_type * |
1810 | skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) | |
abd29c93 AN |
1811 | { |
1812 | const struct btf_type *t = btf__type_by_id(btf, id); | |
8837fe5d | 1813 | |
ddc7c304 AN |
1814 | if (res_id) |
1815 | *res_id = id; | |
1816 | ||
1817 | while (btf_is_mod(t) || btf_is_typedef(t)) { | |
1818 | if (res_id) | |
1819 | *res_id = t->type; | |
1820 | t = btf__type_by_id(btf, t->type); | |
abd29c93 | 1821 | } |
ddc7c304 AN |
1822 | |
1823 | return t; | |
abd29c93 AN |
1824 | } |
1825 | ||
590a0088 MKL |
1826 | static const struct btf_type * |
1827 | resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) | |
1828 | { | |
1829 | const struct btf_type *t; | |
1830 | ||
1831 | t = skip_mods_and_typedefs(btf, id, NULL); | |
1832 | if (!btf_is_ptr(t)) | |
1833 | return NULL; | |
1834 | ||
1835 | t = skip_mods_and_typedefs(btf, t->type, res_id); | |
1836 | ||
1837 | return btf_is_func_proto(t) ? t : NULL; | |
1838 | } | |
1839 | ||
ef99b02b AN |
1840 | /* |
1841 | * Fetch integer attribute of BTF map definition. Such attributes are | |
1842 | * represented using a pointer to an array, in which dimensionality of array | |
1843 | * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; | |
1844 | * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF | |
1845 | * type definition, while using only sizeof(void *) space in ELF data section. | |
1846 | */ | |
1847 | static bool get_map_field_int(const char *map_name, const struct btf *btf, | |
abd29c93 | 1848 | const struct btf_type *def, |
8983b731 AN |
1849 | const struct btf_member *m, __u32 *res) |
1850 | { | |
ddc7c304 | 1851 | const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); |
abd29c93 | 1852 | const char *name = btf__name_by_offset(btf, m->name_off); |
ef99b02b AN |
1853 | const struct btf_array *arr_info; |
1854 | const struct btf_type *arr_t; | |
abd29c93 | 1855 | |
b03bc685 | 1856 | if (!btf_is_ptr(t)) { |
be18010e KW |
1857 | pr_warn("map '%s': attr '%s': expected PTR, got %u.\n", |
1858 | map_name, name, btf_kind(t)); | |
abd29c93 AN |
1859 | return false; |
1860 | } | |
ef99b02b AN |
1861 | |
1862 | arr_t = btf__type_by_id(btf, t->type); | |
1863 | if (!arr_t) { | |
be18010e KW |
1864 | pr_warn("map '%s': attr '%s': type [%u] not found.\n", |
1865 | map_name, name, t->type); | |
abd29c93 AN |
1866 | return false; |
1867 | } | |
b03bc685 | 1868 | if (!btf_is_array(arr_t)) { |
be18010e KW |
1869 | pr_warn("map '%s': attr '%s': expected ARRAY, got %u.\n", |
1870 | map_name, name, btf_kind(arr_t)); | |
abd29c93 AN |
1871 | return false; |
1872 | } | |
b03bc685 | 1873 | arr_info = btf_array(arr_t); |
ef99b02b | 1874 | *res = arr_info->nelems; |
abd29c93 AN |
1875 | return true; |
1876 | } | |
1877 | ||
57a00f41 THJ |
1878 | static int build_map_pin_path(struct bpf_map *map, const char *path) |
1879 | { | |
1880 | char buf[PATH_MAX]; | |
1881 | int err, len; | |
1882 | ||
1883 | if (!path) | |
1884 | path = "/sys/fs/bpf"; | |
1885 | ||
1886 | len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); | |
1887 | if (len < 0) | |
1888 | return -EINVAL; | |
1889 | else if (len >= PATH_MAX) | |
1890 | return -ENAMETOOLONG; | |
1891 | ||
1892 | err = bpf_map__set_pin_path(map, buf); | |
1893 | if (err) | |
1894 | return err; | |
1895 | ||
1896 | return 0; | |
1897 | } | |
1898 | ||
abd29c93 AN |
1899 | static int bpf_object__init_user_btf_map(struct bpf_object *obj, |
1900 | const struct btf_type *sec, | |
1901 | int var_idx, int sec_idx, | |
57a00f41 THJ |
1902 | const Elf_Data *data, bool strict, |
1903 | const char *pin_root_path) | |
abd29c93 AN |
1904 | { |
1905 | const struct btf_type *var, *def, *t; | |
1906 | const struct btf_var_secinfo *vi; | |
1907 | const struct btf_var *var_extra; | |
1908 | const struct btf_member *m; | |
abd29c93 AN |
1909 | const char *map_name; |
1910 | struct bpf_map *map; | |
1911 | int vlen, i; | |
1912 | ||
b03bc685 | 1913 | vi = btf_var_secinfos(sec) + var_idx; |
abd29c93 | 1914 | var = btf__type_by_id(obj->btf, vi->type); |
b03bc685 | 1915 | var_extra = btf_var(var); |
abd29c93 | 1916 | map_name = btf__name_by_offset(obj->btf, var->name_off); |
b03bc685 | 1917 | vlen = btf_vlen(var); |
abd29c93 AN |
1918 | |
1919 | if (map_name == NULL || map_name[0] == '\0') { | |
be18010e | 1920 | pr_warn("map #%d: empty name.\n", var_idx); |
abd29c93 AN |
1921 | return -EINVAL; |
1922 | } | |
1923 | if ((__u64)vi->offset + vi->size > data->d_size) { | |
be18010e | 1924 | pr_warn("map '%s' BTF data is corrupted.\n", map_name); |
abd29c93 AN |
1925 | return -EINVAL; |
1926 | } | |
b03bc685 | 1927 | if (!btf_is_var(var)) { |
be18010e KW |
1928 | pr_warn("map '%s': unexpected var kind %u.\n", |
1929 | map_name, btf_kind(var)); | |
abd29c93 AN |
1930 | return -EINVAL; |
1931 | } | |
1932 | if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED && | |
1933 | var_extra->linkage != BTF_VAR_STATIC) { | |
be18010e KW |
1934 | pr_warn("map '%s': unsupported var linkage %u.\n", |
1935 | map_name, var_extra->linkage); | |
abd29c93 AN |
1936 | return -EOPNOTSUPP; |
1937 | } | |
1938 | ||
ddc7c304 | 1939 | def = skip_mods_and_typedefs(obj->btf, var->type, NULL); |
b03bc685 | 1940 | if (!btf_is_struct(def)) { |
be18010e KW |
1941 | pr_warn("map '%s': unexpected def kind %u.\n", |
1942 | map_name, btf_kind(var)); | |
abd29c93 AN |
1943 | return -EINVAL; |
1944 | } | |
1945 | if (def->size > vi->size) { | |
be18010e | 1946 | pr_warn("map '%s': invalid def size.\n", map_name); |
abd29c93 AN |
1947 | return -EINVAL; |
1948 | } | |
1949 | ||
1950 | map = bpf_object__add_map(obj); | |
1951 | if (IS_ERR(map)) | |
1952 | return PTR_ERR(map); | |
1953 | map->name = strdup(map_name); | |
1954 | if (!map->name) { | |
be18010e | 1955 | pr_warn("map '%s': failed to alloc map name.\n", map_name); |
abd29c93 AN |
1956 | return -ENOMEM; |
1957 | } | |
1958 | map->libbpf_type = LIBBPF_MAP_UNSPEC; | |
1959 | map->def.type = BPF_MAP_TYPE_UNSPEC; | |
1960 | map->sec_idx = sec_idx; | |
1961 | map->sec_offset = vi->offset; | |
1962 | pr_debug("map '%s': at sec_idx %d, offset %zu.\n", | |
1963 | map_name, map->sec_idx, map->sec_offset); | |
1964 | ||
b03bc685 AN |
1965 | vlen = btf_vlen(def); |
1966 | m = btf_members(def); | |
abd29c93 AN |
1967 | for (i = 0; i < vlen; i++, m++) { |
1968 | const char *name = btf__name_by_offset(obj->btf, m->name_off); | |
1969 | ||
1970 | if (!name) { | |
be18010e | 1971 | pr_warn("map '%s': invalid field #%d.\n", map_name, i); |
abd29c93 AN |
1972 | return -EINVAL; |
1973 | } | |
1974 | if (strcmp(name, "type") == 0) { | |
1975 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1976 | &map->def.type)) |
abd29c93 AN |
1977 | return -EINVAL; |
1978 | pr_debug("map '%s': found type = %u.\n", | |
1979 | map_name, map->def.type); | |
1980 | } else if (strcmp(name, "max_entries") == 0) { | |
1981 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1982 | &map->def.max_entries)) |
abd29c93 AN |
1983 | return -EINVAL; |
1984 | pr_debug("map '%s': found max_entries = %u.\n", | |
1985 | map_name, map->def.max_entries); | |
1986 | } else if (strcmp(name, "map_flags") == 0) { | |
1987 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1988 | &map->def.map_flags)) |
abd29c93 AN |
1989 | return -EINVAL; |
1990 | pr_debug("map '%s': found map_flags = %u.\n", | |
1991 | map_name, map->def.map_flags); | |
1992 | } else if (strcmp(name, "key_size") == 0) { | |
1993 | __u32 sz; | |
1994 | ||
1995 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 1996 | &sz)) |
abd29c93 AN |
1997 | return -EINVAL; |
1998 | pr_debug("map '%s': found key_size = %u.\n", | |
1999 | map_name, sz); | |
2000 | if (map->def.key_size && map->def.key_size != sz) { | |
be18010e KW |
2001 | pr_warn("map '%s': conflicting key size %u != %u.\n", |
2002 | map_name, map->def.key_size, sz); | |
abd29c93 AN |
2003 | return -EINVAL; |
2004 | } | |
2005 | map->def.key_size = sz; | |
2006 | } else if (strcmp(name, "key") == 0) { | |
2007 | __s64 sz; | |
2008 | ||
2009 | t = btf__type_by_id(obj->btf, m->type); | |
2010 | if (!t) { | |
be18010e KW |
2011 | pr_warn("map '%s': key type [%d] not found.\n", |
2012 | map_name, m->type); | |
abd29c93 AN |
2013 | return -EINVAL; |
2014 | } | |
b03bc685 | 2015 | if (!btf_is_ptr(t)) { |
be18010e KW |
2016 | pr_warn("map '%s': key spec is not PTR: %u.\n", |
2017 | map_name, btf_kind(t)); | |
abd29c93 AN |
2018 | return -EINVAL; |
2019 | } | |
2020 | sz = btf__resolve_size(obj->btf, t->type); | |
2021 | if (sz < 0) { | |
679152d3 AN |
2022 | pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", |
2023 | map_name, t->type, (ssize_t)sz); | |
abd29c93 AN |
2024 | return sz; |
2025 | } | |
679152d3 AN |
2026 | pr_debug("map '%s': found key [%u], sz = %zd.\n", |
2027 | map_name, t->type, (ssize_t)sz); | |
abd29c93 | 2028 | if (map->def.key_size && map->def.key_size != sz) { |
679152d3 AN |
2029 | pr_warn("map '%s': conflicting key size %u != %zd.\n", |
2030 | map_name, map->def.key_size, (ssize_t)sz); | |
abd29c93 AN |
2031 | return -EINVAL; |
2032 | } | |
2033 | map->def.key_size = sz; | |
2034 | map->btf_key_type_id = t->type; | |
2035 | } else if (strcmp(name, "value_size") == 0) { | |
2036 | __u32 sz; | |
2037 | ||
2038 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
ef99b02b | 2039 | &sz)) |
abd29c93 AN |
2040 | return -EINVAL; |
2041 | pr_debug("map '%s': found value_size = %u.\n", | |
2042 | map_name, sz); | |
2043 | if (map->def.value_size && map->def.value_size != sz) { | |
be18010e KW |
2044 | pr_warn("map '%s': conflicting value size %u != %u.\n", |
2045 | map_name, map->def.value_size, sz); | |
abd29c93 AN |
2046 | return -EINVAL; |
2047 | } | |
2048 | map->def.value_size = sz; | |
2049 | } else if (strcmp(name, "value") == 0) { | |
2050 | __s64 sz; | |
2051 | ||
2052 | t = btf__type_by_id(obj->btf, m->type); | |
2053 | if (!t) { | |
be18010e KW |
2054 | pr_warn("map '%s': value type [%d] not found.\n", |
2055 | map_name, m->type); | |
abd29c93 AN |
2056 | return -EINVAL; |
2057 | } | |
b03bc685 | 2058 | if (!btf_is_ptr(t)) { |
be18010e KW |
2059 | pr_warn("map '%s': value spec is not PTR: %u.\n", |
2060 | map_name, btf_kind(t)); | |
abd29c93 AN |
2061 | return -EINVAL; |
2062 | } | |
2063 | sz = btf__resolve_size(obj->btf, t->type); | |
2064 | if (sz < 0) { | |
679152d3 AN |
2065 | pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", |
2066 | map_name, t->type, (ssize_t)sz); | |
abd29c93 AN |
2067 | return sz; |
2068 | } | |
679152d3 AN |
2069 | pr_debug("map '%s': found value [%u], sz = %zd.\n", |
2070 | map_name, t->type, (ssize_t)sz); | |
abd29c93 | 2071 | if (map->def.value_size && map->def.value_size != sz) { |
679152d3 AN |
2072 | pr_warn("map '%s': conflicting value size %u != %zd.\n", |
2073 | map_name, map->def.value_size, (ssize_t)sz); | |
abd29c93 AN |
2074 | return -EINVAL; |
2075 | } | |
2076 | map->def.value_size = sz; | |
2077 | map->btf_value_type_id = t->type; | |
57a00f41 THJ |
2078 | } else if (strcmp(name, "pinning") == 0) { |
2079 | __u32 val; | |
2080 | int err; | |
2081 | ||
2082 | if (!get_map_field_int(map_name, obj->btf, def, m, | |
2083 | &val)) | |
2084 | return -EINVAL; | |
2085 | pr_debug("map '%s': found pinning = %u.\n", | |
2086 | map_name, val); | |
2087 | ||
2088 | if (val != LIBBPF_PIN_NONE && | |
2089 | val != LIBBPF_PIN_BY_NAME) { | |
2090 | pr_warn("map '%s': invalid pinning value %u.\n", | |
2091 | map_name, val); | |
2092 | return -EINVAL; | |
2093 | } | |
2094 | if (val == LIBBPF_PIN_BY_NAME) { | |
2095 | err = build_map_pin_path(map, pin_root_path); | |
2096 | if (err) { | |
2097 | pr_warn("map '%s': couldn't build pin path.\n", | |
2098 | map_name); | |
2099 | return err; | |
2100 | } | |
2101 | } | |
abd29c93 AN |
2102 | } else { |
2103 | if (strict) { | |
be18010e KW |
2104 | pr_warn("map '%s': unknown field '%s'.\n", |
2105 | map_name, name); | |
abd29c93 AN |
2106 | return -ENOTSUP; |
2107 | } | |
2108 | pr_debug("map '%s': ignoring unknown field '%s'.\n", | |
2109 | map_name, name); | |
2110 | } | |
2111 | } | |
2112 | ||
2113 | if (map->def.type == BPF_MAP_TYPE_UNSPEC) { | |
be18010e | 2114 | pr_warn("map '%s': map type isn't specified.\n", map_name); |
abd29c93 AN |
2115 | return -EINVAL; |
2116 | } | |
2117 | ||
2118 | return 0; | |
2119 | } | |
2120 | ||
57a00f41 THJ |
2121 | static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, |
2122 | const char *pin_root_path) | |
abd29c93 AN |
2123 | { |
2124 | const struct btf_type *sec = NULL; | |
2125 | int nr_types, i, vlen, err; | |
2126 | const struct btf_type *t; | |
2127 | const char *name; | |
2128 | Elf_Data *data; | |
2129 | Elf_Scn *scn; | |
2130 | ||
2131 | if (obj->efile.btf_maps_shndx < 0) | |
2132 | return 0; | |
2133 | ||
2134 | scn = elf_getscn(obj->efile.elf, obj->efile.btf_maps_shndx); | |
2135 | if (scn) | |
2136 | data = elf_getdata(scn, NULL); | |
2137 | if (!scn || !data) { | |
be18010e KW |
2138 | pr_warn("failed to get Elf_Data from map section %d (%s)\n", |
2139 | obj->efile.maps_shndx, MAPS_ELF_SEC); | |
abd29c93 AN |
2140 | return -EINVAL; |
2141 | } | |
2142 | ||
2143 | nr_types = btf__get_nr_types(obj->btf); | |
2144 | for (i = 1; i <= nr_types; i++) { | |
2145 | t = btf__type_by_id(obj->btf, i); | |
b03bc685 | 2146 | if (!btf_is_datasec(t)) |
abd29c93 AN |
2147 | continue; |
2148 | name = btf__name_by_offset(obj->btf, t->name_off); | |
2149 | if (strcmp(name, MAPS_ELF_SEC) == 0) { | |
2150 | sec = t; | |
2151 | break; | |
2152 | } | |
2153 | } | |
2154 | ||
2155 | if (!sec) { | |
be18010e | 2156 | pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); |
abd29c93 AN |
2157 | return -ENOENT; |
2158 | } | |
2159 | ||
b03bc685 | 2160 | vlen = btf_vlen(sec); |
abd29c93 AN |
2161 | for (i = 0; i < vlen; i++) { |
2162 | err = bpf_object__init_user_btf_map(obj, sec, i, | |
2163 | obj->efile.btf_maps_shndx, | |
8983b731 AN |
2164 | data, strict, |
2165 | pin_root_path); | |
abd29c93 AN |
2166 | if (err) |
2167 | return err; | |
2168 | } | |
2169 | ||
2170 | return 0; | |
2171 | } | |
2172 | ||
0d13bfce | 2173 | static int bpf_object__init_maps(struct bpf_object *obj, |
01af3bf0 | 2174 | const struct bpf_object_open_opts *opts) |
bf829271 | 2175 | { |
166750bc AN |
2176 | const char *pin_root_path; |
2177 | bool strict; | |
bf829271 | 2178 | int err; |
8837fe5d | 2179 | |
166750bc AN |
2180 | strict = !OPTS_GET(opts, relaxed_maps, false); |
2181 | pin_root_path = OPTS_GET(opts, pin_root_path, NULL); | |
bf829271 | 2182 | |
166750bc AN |
2183 | err = bpf_object__init_user_maps(obj, strict); |
2184 | err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path); | |
2185 | err = err ?: bpf_object__init_global_data_maps(obj); | |
81bfdd08 | 2186 | err = err ?: bpf_object__init_kconfig_map(obj); |
590a0088 | 2187 | err = err ?: bpf_object__init_struct_ops_maps(obj); |
bf829271 AN |
2188 | if (err) |
2189 | return err; | |
2190 | ||
bf829271 | 2191 | return 0; |
561bbcca WN |
2192 | } |
2193 | ||
e3d91b0c JDB |
2194 | static bool section_have_execinstr(struct bpf_object *obj, int idx) |
2195 | { | |
2196 | Elf_Scn *scn; | |
2197 | GElf_Shdr sh; | |
2198 | ||
2199 | scn = elf_getscn(obj->efile.elf, idx); | |
2200 | if (!scn) | |
2201 | return false; | |
2202 | ||
2203 | if (gelf_getshdr(scn, &sh) != &sh) | |
2204 | return false; | |
2205 | ||
2206 | if (sh.sh_flags & SHF_EXECINSTR) | |
2207 | return true; | |
2208 | ||
2209 | return false; | |
2210 | } | |
2211 | ||
d7c4b398 AN |
2212 | static void bpf_object__sanitize_btf(struct bpf_object *obj) |
2213 | { | |
2d3eb67f | 2214 | bool has_func_global = obj->caps.btf_func_global; |
d7c4b398 AN |
2215 | bool has_datasec = obj->caps.btf_datasec; |
2216 | bool has_func = obj->caps.btf_func; | |
2217 | struct btf *btf = obj->btf; | |
2218 | struct btf_type *t; | |
2219 | int i, j, vlen; | |
d7c4b398 | 2220 | |
2d3eb67f | 2221 | if (!obj->btf || (has_func && has_datasec && has_func_global)) |
d7c4b398 AN |
2222 | return; |
2223 | ||
2224 | for (i = 1; i <= btf__get_nr_types(btf); i++) { | |
2225 | t = (struct btf_type *)btf__type_by_id(btf, i); | |
d7c4b398 | 2226 | |
b03bc685 | 2227 | if (!has_datasec && btf_is_var(t)) { |
d7c4b398 AN |
2228 | /* replace VAR with INT */ |
2229 | t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); | |
1d4126c4 AN |
2230 | /* |
2231 | * using size = 1 is the safest choice, 4 will be too | |
2232 | * big and cause kernel BTF validation failure if | |
2233 | * original variable took less than 4 bytes | |
2234 | */ | |
2235 | t->size = 1; | |
708852dc | 2236 | *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); |
b03bc685 | 2237 | } else if (!has_datasec && btf_is_datasec(t)) { |
d7c4b398 | 2238 | /* replace DATASEC with STRUCT */ |
b03bc685 AN |
2239 | const struct btf_var_secinfo *v = btf_var_secinfos(t); |
2240 | struct btf_member *m = btf_members(t); | |
d7c4b398 AN |
2241 | struct btf_type *vt; |
2242 | char *name; | |
2243 | ||
2244 | name = (char *)btf__name_by_offset(btf, t->name_off); | |
2245 | while (*name) { | |
2246 | if (*name == '.') | |
2247 | *name = '_'; | |
2248 | name++; | |
2249 | } | |
2250 | ||
b03bc685 | 2251 | vlen = btf_vlen(t); |
d7c4b398 AN |
2252 | t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); |
2253 | for (j = 0; j < vlen; j++, v++, m++) { | |
2254 | /* order of field assignments is important */ | |
2255 | m->offset = v->offset * 8; | |
2256 | m->type = v->type; | |
2257 | /* preserve variable name as member name */ | |
2258 | vt = (void *)btf__type_by_id(btf, v->type); | |
2259 | m->name_off = vt->name_off; | |
2260 | } | |
b03bc685 | 2261 | } else if (!has_func && btf_is_func_proto(t)) { |
d7c4b398 | 2262 | /* replace FUNC_PROTO with ENUM */ |
b03bc685 | 2263 | vlen = btf_vlen(t); |
d7c4b398 AN |
2264 | t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); |
2265 | t->size = sizeof(__u32); /* kernel enforced */ | |
b03bc685 | 2266 | } else if (!has_func && btf_is_func(t)) { |
d7c4b398 AN |
2267 | /* replace FUNC with TYPEDEF */ |
2268 | t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); | |
2d3eb67f AS |
2269 | } else if (!has_func_global && btf_is_func(t)) { |
2270 | /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ | |
2271 | t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); | |
d7c4b398 AN |
2272 | } |
2273 | } | |
2274 | } | |
2275 | ||
2276 | static void bpf_object__sanitize_btf_ext(struct bpf_object *obj) | |
2277 | { | |
2278 | if (!obj->btf_ext) | |
2279 | return; | |
2280 | ||
2281 | if (!obj->caps.btf_func) { | |
2282 | btf_ext__free(obj->btf_ext); | |
2283 | obj->btf_ext = NULL; | |
2284 | } | |
2285 | } | |
2286 | ||
abd29c93 AN |
2287 | static bool bpf_object__is_btf_mandatory(const struct bpf_object *obj) |
2288 | { | |
53276446 | 2289 | return obj->efile.st_ops_shndx >= 0 || obj->nr_extern > 0; |
abd29c93 AN |
2290 | } |
2291 | ||
063183bf | 2292 | static int bpf_object__init_btf(struct bpf_object *obj, |
9c6660d0 AN |
2293 | Elf_Data *btf_data, |
2294 | Elf_Data *btf_ext_data) | |
2295 | { | |
b7d7f3e1 | 2296 | int err = -ENOENT; |
9c6660d0 AN |
2297 | |
2298 | if (btf_data) { | |
2299 | obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); | |
2300 | if (IS_ERR(obj->btf)) { | |
b7d7f3e1 AN |
2301 | err = PTR_ERR(obj->btf); |
2302 | obj->btf = NULL; | |
be18010e KW |
2303 | pr_warn("Error loading ELF section %s: %d.\n", |
2304 | BTF_ELF_SEC, err); | |
9c6660d0 AN |
2305 | goto out; |
2306 | } | |
b7d7f3e1 | 2307 | err = 0; |
9c6660d0 AN |
2308 | } |
2309 | if (btf_ext_data) { | |
2310 | if (!obj->btf) { | |
2311 | pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", | |
2312 | BTF_EXT_ELF_SEC, BTF_ELF_SEC); | |
2313 | goto out; | |
2314 | } | |
2315 | obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, | |
2316 | btf_ext_data->d_size); | |
2317 | if (IS_ERR(obj->btf_ext)) { | |
be18010e KW |
2318 | pr_warn("Error loading ELF section %s: %ld. Ignored and continue.\n", |
2319 | BTF_EXT_ELF_SEC, PTR_ERR(obj->btf_ext)); | |
9c6660d0 AN |
2320 | obj->btf_ext = NULL; |
2321 | goto out; | |
2322 | } | |
9c6660d0 AN |
2323 | } |
2324 | out: | |
b7d7f3e1 | 2325 | if (err && bpf_object__is_btf_mandatory(obj)) { |
be18010e | 2326 | pr_warn("BTF is required, but is missing or corrupted.\n"); |
b7d7f3e1 | 2327 | return err; |
abd29c93 | 2328 | } |
9c6660d0 AN |
2329 | return 0; |
2330 | } | |
2331 | ||
166750bc AN |
2332 | static int bpf_object__finalize_btf(struct bpf_object *obj) |
2333 | { | |
2334 | int err; | |
2335 | ||
2336 | if (!obj->btf) | |
2337 | return 0; | |
2338 | ||
2339 | err = btf__finalize_data(obj, obj->btf); | |
2340 | if (!err) | |
2341 | return 0; | |
2342 | ||
2343 | pr_warn("Error finalizing %s: %d.\n", BTF_ELF_SEC, err); | |
2344 | btf__free(obj->btf); | |
2345 | obj->btf = NULL; | |
2346 | btf_ext__free(obj->btf_ext); | |
2347 | obj->btf_ext = NULL; | |
2348 | ||
2349 | if (bpf_object__is_btf_mandatory(obj)) { | |
2350 | pr_warn("BTF is required, but is missing or corrupted.\n"); | |
2351 | return -ENOENT; | |
2352 | } | |
2353 | return 0; | |
2354 | } | |
2355 | ||
a6ed02ca KS |
2356 | static inline bool libbpf_prog_needs_vmlinux_btf(struct bpf_program *prog) |
2357 | { | |
2358 | if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) | |
2359 | return true; | |
2360 | ||
2361 | /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs | |
2362 | * also need vmlinux BTF | |
2363 | */ | |
2364 | if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) | |
2365 | return true; | |
2366 | ||
2367 | return false; | |
2368 | } | |
2369 | ||
2370 | static int bpf_object__load_vmlinux_btf(struct bpf_object *obj) | |
2371 | { | |
2372 | struct bpf_program *prog; | |
2373 | int err; | |
2374 | ||
2375 | bpf_object__for_each_program(prog, obj) { | |
2376 | if (libbpf_prog_needs_vmlinux_btf(prog)) { | |
2377 | obj->btf_vmlinux = libbpf_find_kernel_btf(); | |
2378 | if (IS_ERR(obj->btf_vmlinux)) { | |
2379 | err = PTR_ERR(obj->btf_vmlinux); | |
2380 | pr_warn("Error loading vmlinux BTF: %d\n", err); | |
2381 | obj->btf_vmlinux = NULL; | |
2382 | return err; | |
2383 | } | |
2384 | return 0; | |
2385 | } | |
2386 | } | |
2387 | ||
2388 | return 0; | |
2389 | } | |
2390 | ||
063183bf AN |
2391 | static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) |
2392 | { | |
2393 | int err = 0; | |
2394 | ||
2395 | if (!obj->btf) | |
2396 | return 0; | |
2397 | ||
2398 | bpf_object__sanitize_btf(obj); | |
2399 | bpf_object__sanitize_btf_ext(obj); | |
2400 | ||
2401 | err = btf__load(obj->btf); | |
2402 | if (err) { | |
be18010e KW |
2403 | pr_warn("Error loading %s into kernel: %d.\n", |
2404 | BTF_ELF_SEC, err); | |
063183bf AN |
2405 | btf__free(obj->btf); |
2406 | obj->btf = NULL; | |
04efe591 AN |
2407 | /* btf_ext can't exist without btf, so free it as well */ |
2408 | if (obj->btf_ext) { | |
2409 | btf_ext__free(obj->btf_ext); | |
2410 | obj->btf_ext = NULL; | |
2411 | } | |
2412 | ||
abd29c93 AN |
2413 | if (bpf_object__is_btf_mandatory(obj)) |
2414 | return err; | |
063183bf AN |
2415 | } |
2416 | return 0; | |
2417 | } | |
2418 | ||
0d13bfce | 2419 | static int bpf_object__elf_collect(struct bpf_object *obj) |
29603665 WN |
2420 | { |
2421 | Elf *elf = obj->efile.elf; | |
2422 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
f0187f0b | 2423 | Elf_Data *btf_ext_data = NULL; |
1713d68b | 2424 | Elf_Data *btf_data = NULL; |
29603665 | 2425 | Elf_Scn *scn = NULL; |
666810e8 | 2426 | int idx = 0, err = 0; |
29603665 WN |
2427 | |
2428 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
2429 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
be18010e | 2430 | pr_warn("failed to get e_shstrndx from %s\n", obj->path); |
6371ca3b | 2431 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
2432 | } |
2433 | ||
2434 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
2435 | char *name; | |
2436 | GElf_Shdr sh; | |
2437 | Elf_Data *data; | |
2438 | ||
2439 | idx++; | |
2440 | if (gelf_getshdr(scn, &sh) != &sh) { | |
be18010e KW |
2441 | pr_warn("failed to get section(%d) header from %s\n", |
2442 | idx, obj->path); | |
01b29d1d | 2443 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
2444 | } |
2445 | ||
2446 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
2447 | if (!name) { | |
be18010e KW |
2448 | pr_warn("failed to get section(%d) name from %s\n", |
2449 | idx, obj->path); | |
01b29d1d | 2450 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
2451 | } |
2452 | ||
2453 | data = elf_getdata(scn, 0); | |
2454 | if (!data) { | |
be18010e KW |
2455 | pr_warn("failed to get section(%d) data from %s(%s)\n", |
2456 | idx, name, obj->path); | |
01b29d1d | 2457 | return -LIBBPF_ERRNO__FORMAT; |
29603665 | 2458 | } |
077c066a JDB |
2459 | pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", |
2460 | idx, name, (unsigned long)data->d_size, | |
29603665 WN |
2461 | (int)sh.sh_link, (unsigned long)sh.sh_flags, |
2462 | (int)sh.sh_type); | |
cb1e5e96 | 2463 | |
1713d68b | 2464 | if (strcmp(name, "license") == 0) { |
cb1e5e96 WN |
2465 | err = bpf_object__init_license(obj, |
2466 | data->d_buf, | |
2467 | data->d_size); | |
01b29d1d AN |
2468 | if (err) |
2469 | return err; | |
1713d68b | 2470 | } else if (strcmp(name, "version") == 0) { |
54b8625c JF |
2471 | err = bpf_object__init_kversion(obj, |
2472 | data->d_buf, | |
2473 | data->d_size); | |
2474 | if (err) | |
2475 | return err; | |
1713d68b | 2476 | } else if (strcmp(name, "maps") == 0) { |
666810e8 | 2477 | obj->efile.maps_shndx = idx; |
abd29c93 AN |
2478 | } else if (strcmp(name, MAPS_ELF_SEC) == 0) { |
2479 | obj->efile.btf_maps_shndx = idx; | |
1713d68b DB |
2480 | } else if (strcmp(name, BTF_ELF_SEC) == 0) { |
2481 | btf_data = data; | |
2993e051 | 2482 | } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { |
f0187f0b | 2483 | btf_ext_data = data; |
8a138aed | 2484 | } else if (sh.sh_type == SHT_SYMTAB) { |
bec7d68c | 2485 | if (obj->efile.symbols) { |
be18010e KW |
2486 | pr_warn("bpf: multiple SYMTAB in %s\n", |
2487 | obj->path); | |
01b29d1d | 2488 | return -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 2489 | } |
01b29d1d | 2490 | obj->efile.symbols = data; |
166750bc | 2491 | obj->efile.symbols_shndx = idx; |
01b29d1d | 2492 | obj->efile.strtabidx = sh.sh_link; |
f8c7a4d4 JS |
2493 | } else if (sh.sh_type == SHT_PROGBITS && data->d_size > 0) { |
2494 | if (sh.sh_flags & SHF_EXECINSTR) { | |
2495 | if (strcmp(name, ".text") == 0) | |
2496 | obj->efile.text_shndx = idx; | |
2497 | err = bpf_object__add_program(obj, data->d_buf, | |
8983b731 AN |
2498 | data->d_size, |
2499 | name, idx); | |
f8c7a4d4 JS |
2500 | if (err) { |
2501 | char errmsg[STRERR_BUFSIZE]; | |
8983b731 | 2502 | char *cp; |
f8c7a4d4 | 2503 | |
8983b731 AN |
2504 | cp = libbpf_strerror_r(-err, errmsg, |
2505 | sizeof(errmsg)); | |
be18010e KW |
2506 | pr_warn("failed to alloc program %s (%s): %s", |
2507 | name, obj->path, cp); | |
01b29d1d | 2508 | return err; |
f8c7a4d4 | 2509 | } |
ac9d1389 | 2510 | } else if (strcmp(name, DATA_SEC) == 0) { |
d859900c DB |
2511 | obj->efile.data = data; |
2512 | obj->efile.data_shndx = idx; | |
ac9d1389 | 2513 | } else if (strcmp(name, RODATA_SEC) == 0) { |
d859900c DB |
2514 | obj->efile.rodata = data; |
2515 | obj->efile.rodata_shndx = idx; | |
590a0088 MKL |
2516 | } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { |
2517 | obj->efile.st_ops_data = data; | |
2518 | obj->efile.st_ops_shndx = idx; | |
d859900c DB |
2519 | } else { |
2520 | pr_debug("skip section(%d) %s\n", idx, name); | |
a5b8bd47 | 2521 | } |
b62f06e8 | 2522 | } else if (sh.sh_type == SHT_REL) { |
1f8e2bcb AN |
2523 | int nr_sects = obj->efile.nr_reloc_sects; |
2524 | void *sects = obj->efile.reloc_sects; | |
e3d91b0c JDB |
2525 | int sec = sh.sh_info; /* points to other section */ |
2526 | ||
2527 | /* Only do relo for section with exec instructions */ | |
590a0088 MKL |
2528 | if (!section_have_execinstr(obj, sec) && |
2529 | strcmp(name, ".rel" STRUCT_OPS_SEC)) { | |
e3d91b0c JDB |
2530 | pr_debug("skip relo %s(%d) for section(%d)\n", |
2531 | name, idx, sec); | |
2532 | continue; | |
2533 | } | |
b62f06e8 | 2534 | |
1f8e2bcb AN |
2535 | sects = reallocarray(sects, nr_sects + 1, |
2536 | sizeof(*obj->efile.reloc_sects)); | |
2537 | if (!sects) { | |
2538 | pr_warn("reloc_sects realloc failed\n"); | |
01b29d1d AN |
2539 | return -ENOMEM; |
2540 | } | |
b62f06e8 | 2541 | |
1f8e2bcb AN |
2542 | obj->efile.reloc_sects = sects; |
2543 | obj->efile.nr_reloc_sects++; | |
b62f06e8 | 2544 | |
1f8e2bcb AN |
2545 | obj->efile.reloc_sects[nr_sects].shdr = sh; |
2546 | obj->efile.reloc_sects[nr_sects].data = data; | |
ac9d1389 AN |
2547 | } else if (sh.sh_type == SHT_NOBITS && |
2548 | strcmp(name, BSS_SEC) == 0) { | |
d859900c DB |
2549 | obj->efile.bss = data; |
2550 | obj->efile.bss_shndx = idx; | |
077c066a JDB |
2551 | } else { |
2552 | pr_debug("skip section(%d) %s\n", idx, name); | |
bec7d68c | 2553 | } |
29603665 | 2554 | } |
561bbcca | 2555 | |
d3a3aa0c | 2556 | if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { |
be18010e | 2557 | pr_warn("Corrupted ELF file: index of strtab invalid\n"); |
f102154d | 2558 | return -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 2559 | } |
0d13bfce | 2560 | return bpf_object__init_btf(obj, btf_data, btf_ext_data); |
29603665 WN |
2561 | } |
2562 | ||
166750bc AN |
2563 | static bool sym_is_extern(const GElf_Sym *sym) |
2564 | { | |
2565 | int bind = GELF_ST_BIND(sym->st_info); | |
2566 | /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ | |
2567 | return sym->st_shndx == SHN_UNDEF && | |
2568 | (bind == STB_GLOBAL || bind == STB_WEAK) && | |
2569 | GELF_ST_TYPE(sym->st_info) == STT_NOTYPE; | |
2570 | } | |
2571 | ||
2572 | static int find_extern_btf_id(const struct btf *btf, const char *ext_name) | |
2573 | { | |
2574 | const struct btf_type *t; | |
2575 | const char *var_name; | |
2576 | int i, n; | |
2577 | ||
2578 | if (!btf) | |
2579 | return -ESRCH; | |
2580 | ||
2581 | n = btf__get_nr_types(btf); | |
2582 | for (i = 1; i <= n; i++) { | |
2583 | t = btf__type_by_id(btf, i); | |
2584 | ||
2585 | if (!btf_is_var(t)) | |
2586 | continue; | |
2587 | ||
2588 | var_name = btf__name_by_offset(btf, t->name_off); | |
2589 | if (strcmp(var_name, ext_name)) | |
2590 | continue; | |
2591 | ||
2592 | if (btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) | |
2593 | return -EINVAL; | |
2594 | ||
2595 | return i; | |
2596 | } | |
2597 | ||
2598 | return -ENOENT; | |
2599 | } | |
2600 | ||
2601 | static enum extern_type find_extern_type(const struct btf *btf, int id, | |
2602 | bool *is_signed) | |
2603 | { | |
2604 | const struct btf_type *t; | |
2605 | const char *name; | |
2606 | ||
2607 | t = skip_mods_and_typedefs(btf, id, NULL); | |
2608 | name = btf__name_by_offset(btf, t->name_off); | |
2609 | ||
2610 | if (is_signed) | |
2611 | *is_signed = false; | |
2612 | switch (btf_kind(t)) { | |
2613 | case BTF_KIND_INT: { | |
2614 | int enc = btf_int_encoding(t); | |
2615 | ||
2616 | if (enc & BTF_INT_BOOL) | |
2617 | return t->size == 1 ? EXT_BOOL : EXT_UNKNOWN; | |
2618 | if (is_signed) | |
2619 | *is_signed = enc & BTF_INT_SIGNED; | |
2620 | if (t->size == 1) | |
2621 | return EXT_CHAR; | |
2622 | if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) | |
2623 | return EXT_UNKNOWN; | |
2624 | return EXT_INT; | |
2625 | } | |
2626 | case BTF_KIND_ENUM: | |
2627 | if (t->size != 4) | |
2628 | return EXT_UNKNOWN; | |
2629 | if (strcmp(name, "libbpf_tristate")) | |
2630 | return EXT_UNKNOWN; | |
2631 | return EXT_TRISTATE; | |
2632 | case BTF_KIND_ARRAY: | |
2633 | if (btf_array(t)->nelems == 0) | |
2634 | return EXT_UNKNOWN; | |
2635 | if (find_extern_type(btf, btf_array(t)->type, NULL) != EXT_CHAR) | |
2636 | return EXT_UNKNOWN; | |
2637 | return EXT_CHAR_ARR; | |
2638 | default: | |
2639 | return EXT_UNKNOWN; | |
2640 | } | |
2641 | } | |
2642 | ||
2643 | static int cmp_externs(const void *_a, const void *_b) | |
2644 | { | |
2645 | const struct extern_desc *a = _a; | |
2646 | const struct extern_desc *b = _b; | |
2647 | ||
2648 | /* descending order by alignment requirements */ | |
2649 | if (a->align != b->align) | |
2650 | return a->align > b->align ? -1 : 1; | |
2651 | /* ascending order by size, within same alignment class */ | |
2652 | if (a->sz != b->sz) | |
2653 | return a->sz < b->sz ? -1 : 1; | |
2654 | /* resolve ties by name */ | |
2655 | return strcmp(a->name, b->name); | |
2656 | } | |
2657 | ||
2658 | static int bpf_object__collect_externs(struct bpf_object *obj) | |
2659 | { | |
2660 | const struct btf_type *t; | |
2661 | struct extern_desc *ext; | |
2662 | int i, n, off, btf_id; | |
2663 | struct btf_type *sec; | |
2664 | const char *ext_name; | |
2665 | Elf_Scn *scn; | |
2666 | GElf_Shdr sh; | |
2667 | ||
2668 | if (!obj->efile.symbols) | |
2669 | return 0; | |
2670 | ||
2671 | scn = elf_getscn(obj->efile.elf, obj->efile.symbols_shndx); | |
2672 | if (!scn) | |
2673 | return -LIBBPF_ERRNO__FORMAT; | |
2674 | if (gelf_getshdr(scn, &sh) != &sh) | |
2675 | return -LIBBPF_ERRNO__FORMAT; | |
2676 | n = sh.sh_size / sh.sh_entsize; | |
2677 | ||
2678 | pr_debug("looking for externs among %d symbols...\n", n); | |
2679 | for (i = 0; i < n; i++) { | |
2680 | GElf_Sym sym; | |
2681 | ||
2682 | if (!gelf_getsym(obj->efile.symbols, i, &sym)) | |
2683 | return -LIBBPF_ERRNO__FORMAT; | |
2684 | if (!sym_is_extern(&sym)) | |
2685 | continue; | |
2686 | ext_name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
2687 | sym.st_name); | |
2688 | if (!ext_name || !ext_name[0]) | |
2689 | continue; | |
2690 | ||
2691 | ext = obj->externs; | |
2692 | ext = reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); | |
2693 | if (!ext) | |
2694 | return -ENOMEM; | |
2695 | obj->externs = ext; | |
2696 | ext = &ext[obj->nr_extern]; | |
2697 | memset(ext, 0, sizeof(*ext)); | |
2698 | obj->nr_extern++; | |
2699 | ||
2700 | ext->btf_id = find_extern_btf_id(obj->btf, ext_name); | |
2701 | if (ext->btf_id <= 0) { | |
2702 | pr_warn("failed to find BTF for extern '%s': %d\n", | |
2703 | ext_name, ext->btf_id); | |
2704 | return ext->btf_id; | |
2705 | } | |
2706 | t = btf__type_by_id(obj->btf, ext->btf_id); | |
2707 | ext->name = btf__name_by_offset(obj->btf, t->name_off); | |
2708 | ext->sym_idx = i; | |
2709 | ext->is_weak = GELF_ST_BIND(sym.st_info) == STB_WEAK; | |
2710 | ext->sz = btf__resolve_size(obj->btf, t->type); | |
2711 | if (ext->sz <= 0) { | |
2712 | pr_warn("failed to resolve size of extern '%s': %d\n", | |
2713 | ext_name, ext->sz); | |
2714 | return ext->sz; | |
2715 | } | |
2716 | ext->align = btf__align_of(obj->btf, t->type); | |
2717 | if (ext->align <= 0) { | |
2718 | pr_warn("failed to determine alignment of extern '%s': %d\n", | |
2719 | ext_name, ext->align); | |
2720 | return -EINVAL; | |
2721 | } | |
2722 | ext->type = find_extern_type(obj->btf, t->type, | |
2723 | &ext->is_signed); | |
2724 | if (ext->type == EXT_UNKNOWN) { | |
2725 | pr_warn("extern '%s' type is unsupported\n", ext_name); | |
2726 | return -ENOTSUP; | |
2727 | } | |
2728 | } | |
2729 | pr_debug("collected %d externs total\n", obj->nr_extern); | |
2730 | ||
2731 | if (!obj->nr_extern) | |
2732 | return 0; | |
2733 | ||
2734 | /* sort externs by (alignment, size, name) and calculate their offsets | |
2735 | * within a map */ | |
2736 | qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); | |
2737 | off = 0; | |
2738 | for (i = 0; i < obj->nr_extern; i++) { | |
2739 | ext = &obj->externs[i]; | |
2740 | ext->data_off = roundup(off, ext->align); | |
2741 | off = ext->data_off + ext->sz; | |
2742 | pr_debug("extern #%d: symbol %d, off %u, name %s\n", | |
2743 | i, ext->sym_idx, ext->data_off, ext->name); | |
2744 | } | |
2745 | ||
81bfdd08 | 2746 | btf_id = btf__find_by_name(obj->btf, KCONFIG_SEC); |
166750bc | 2747 | if (btf_id <= 0) { |
81bfdd08 | 2748 | pr_warn("no BTF info found for '%s' datasec\n", KCONFIG_SEC); |
166750bc AN |
2749 | return -ESRCH; |
2750 | } | |
2751 | ||
2752 | sec = (struct btf_type *)btf__type_by_id(obj->btf, btf_id); | |
2753 | sec->size = off; | |
2754 | n = btf_vlen(sec); | |
2755 | for (i = 0; i < n; i++) { | |
2756 | struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; | |
2757 | ||
2758 | t = btf__type_by_id(obj->btf, vs->type); | |
2759 | ext_name = btf__name_by_offset(obj->btf, t->name_off); | |
2760 | ext = find_extern_by_name(obj, ext_name); | |
2761 | if (!ext) { | |
2762 | pr_warn("failed to find extern definition for BTF var '%s'\n", | |
2763 | ext_name); | |
2764 | return -ESRCH; | |
2765 | } | |
2766 | vs->offset = ext->data_off; | |
2767 | btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; | |
2768 | } | |
2769 | ||
2770 | return 0; | |
2771 | } | |
2772 | ||
34090915 WN |
2773 | static struct bpf_program * |
2774 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
2775 | { | |
2776 | struct bpf_program *prog; | |
2777 | size_t i; | |
2778 | ||
2779 | for (i = 0; i < obj->nr_programs; i++) { | |
2780 | prog = &obj->programs[i]; | |
2781 | if (prog->idx == idx) | |
2782 | return prog; | |
2783 | } | |
2784 | return NULL; | |
2785 | } | |
2786 | ||
6d4b198b | 2787 | struct bpf_program * |
a324aae3 AN |
2788 | bpf_object__find_program_by_title(const struct bpf_object *obj, |
2789 | const char *title) | |
6d4b198b JK |
2790 | { |
2791 | struct bpf_program *pos; | |
2792 | ||
2793 | bpf_object__for_each_program(pos, obj) { | |
2794 | if (pos->section_name && !strcmp(pos->section_name, title)) | |
2795 | return pos; | |
2796 | } | |
2797 | return NULL; | |
2798 | } | |
2799 | ||
01af3bf0 AN |
2800 | struct bpf_program * |
2801 | bpf_object__find_program_by_name(const struct bpf_object *obj, | |
2802 | const char *name) | |
2803 | { | |
2804 | struct bpf_program *prog; | |
2805 | ||
2806 | bpf_object__for_each_program(prog, obj) { | |
2807 | if (!strcmp(prog->name, name)) | |
2808 | return prog; | |
2809 | } | |
2810 | return NULL; | |
2811 | } | |
2812 | ||
d859900c DB |
2813 | static bool bpf_object__shndx_is_data(const struct bpf_object *obj, |
2814 | int shndx) | |
2815 | { | |
2816 | return shndx == obj->efile.data_shndx || | |
2817 | shndx == obj->efile.bss_shndx || | |
2818 | shndx == obj->efile.rodata_shndx; | |
2819 | } | |
2820 | ||
2821 | static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, | |
2822 | int shndx) | |
2823 | { | |
abd29c93 AN |
2824 | return shndx == obj->efile.maps_shndx || |
2825 | shndx == obj->efile.btf_maps_shndx; | |
d859900c DB |
2826 | } |
2827 | ||
d859900c DB |
2828 | static enum libbpf_map_type |
2829 | bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) | |
2830 | { | |
2831 | if (shndx == obj->efile.data_shndx) | |
2832 | return LIBBPF_MAP_DATA; | |
2833 | else if (shndx == obj->efile.bss_shndx) | |
2834 | return LIBBPF_MAP_BSS; | |
2835 | else if (shndx == obj->efile.rodata_shndx) | |
2836 | return LIBBPF_MAP_RODATA; | |
166750bc | 2837 | else if (shndx == obj->efile.symbols_shndx) |
81bfdd08 | 2838 | return LIBBPF_MAP_KCONFIG; |
d859900c DB |
2839 | else |
2840 | return LIBBPF_MAP_UNSPEC; | |
2841 | } | |
2842 | ||
1f8e2bcb AN |
2843 | static int bpf_program__record_reloc(struct bpf_program *prog, |
2844 | struct reloc_desc *reloc_desc, | |
2845 | __u32 insn_idx, const char *name, | |
2846 | const GElf_Sym *sym, const GElf_Rel *rel) | |
2847 | { | |
2848 | struct bpf_insn *insn = &prog->insns[insn_idx]; | |
2849 | size_t map_idx, nr_maps = prog->obj->nr_maps; | |
2850 | struct bpf_object *obj = prog->obj; | |
2851 | __u32 shdr_idx = sym->st_shndx; | |
2852 | enum libbpf_map_type type; | |
2853 | struct bpf_map *map; | |
2854 | ||
2855 | /* sub-program call relocation */ | |
2856 | if (insn->code == (BPF_JMP | BPF_CALL)) { | |
2857 | if (insn->src_reg != BPF_PSEUDO_CALL) { | |
2858 | pr_warn("incorrect bpf_call opcode\n"); | |
2859 | return -LIBBPF_ERRNO__RELOC; | |
2860 | } | |
2861 | /* text_shndx can be 0, if no default "main" program exists */ | |
2862 | if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { | |
2863 | pr_warn("bad call relo against section %u\n", shdr_idx); | |
2864 | return -LIBBPF_ERRNO__RELOC; | |
2865 | } | |
2866 | if (sym->st_value % 8) { | |
679152d3 AN |
2867 | pr_warn("bad call relo offset: %zu\n", |
2868 | (size_t)sym->st_value); | |
1f8e2bcb AN |
2869 | return -LIBBPF_ERRNO__RELOC; |
2870 | } | |
2871 | reloc_desc->type = RELO_CALL; | |
2872 | reloc_desc->insn_idx = insn_idx; | |
53f8dd43 | 2873 | reloc_desc->sym_off = sym->st_value; |
1f8e2bcb AN |
2874 | obj->has_pseudo_calls = true; |
2875 | return 0; | |
2876 | } | |
2877 | ||
2878 | if (insn->code != (BPF_LD | BPF_IMM | BPF_DW)) { | |
8983b731 | 2879 | pr_warn("invalid relo for insns[%d].code 0x%x\n", |
1f8e2bcb AN |
2880 | insn_idx, insn->code); |
2881 | return -LIBBPF_ERRNO__RELOC; | |
2882 | } | |
166750bc AN |
2883 | |
2884 | if (sym_is_extern(sym)) { | |
2885 | int sym_idx = GELF_R_SYM(rel->r_info); | |
2886 | int i, n = obj->nr_extern; | |
2887 | struct extern_desc *ext; | |
2888 | ||
2889 | for (i = 0; i < n; i++) { | |
2890 | ext = &obj->externs[i]; | |
2891 | if (ext->sym_idx == sym_idx) | |
2892 | break; | |
2893 | } | |
2894 | if (i >= n) { | |
2895 | pr_warn("extern relo failed to find extern for sym %d\n", | |
2896 | sym_idx); | |
2897 | return -LIBBPF_ERRNO__RELOC; | |
2898 | } | |
2899 | pr_debug("found extern #%d '%s' (sym %d, off %u) for insn %u\n", | |
2900 | i, ext->name, ext->sym_idx, ext->data_off, insn_idx); | |
2901 | reloc_desc->type = RELO_EXTERN; | |
2902 | reloc_desc->insn_idx = insn_idx; | |
2903 | reloc_desc->sym_off = ext->data_off; | |
2904 | return 0; | |
2905 | } | |
2906 | ||
1f8e2bcb | 2907 | if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { |
393cdfbe AN |
2908 | pr_warn("invalid relo for \'%s\' in special section 0x%x; forgot to initialize global var?..\n", |
2909 | name, shdr_idx); | |
1f8e2bcb AN |
2910 | return -LIBBPF_ERRNO__RELOC; |
2911 | } | |
2912 | ||
2913 | type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); | |
2914 | ||
2915 | /* generic map reference relocation */ | |
2916 | if (type == LIBBPF_MAP_UNSPEC) { | |
2917 | if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { | |
2918 | pr_warn("bad map relo against section %u\n", | |
2919 | shdr_idx); | |
2920 | return -LIBBPF_ERRNO__RELOC; | |
2921 | } | |
2922 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { | |
2923 | map = &obj->maps[map_idx]; | |
2924 | if (map->libbpf_type != type || | |
2925 | map->sec_idx != sym->st_shndx || | |
2926 | map->sec_offset != sym->st_value) | |
2927 | continue; | |
2928 | pr_debug("found map %zd (%s, sec %d, off %zu) for insn %u\n", | |
2929 | map_idx, map->name, map->sec_idx, | |
2930 | map->sec_offset, insn_idx); | |
2931 | break; | |
2932 | } | |
2933 | if (map_idx >= nr_maps) { | |
679152d3 AN |
2934 | pr_warn("map relo failed to find map for sec %u, off %zu\n", |
2935 | shdr_idx, (size_t)sym->st_value); | |
1f8e2bcb AN |
2936 | return -LIBBPF_ERRNO__RELOC; |
2937 | } | |
2938 | reloc_desc->type = RELO_LD64; | |
2939 | reloc_desc->insn_idx = insn_idx; | |
2940 | reloc_desc->map_idx = map_idx; | |
53f8dd43 | 2941 | reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ |
1f8e2bcb AN |
2942 | return 0; |
2943 | } | |
2944 | ||
2945 | /* global data map relocation */ | |
2946 | if (!bpf_object__shndx_is_data(obj, shdr_idx)) { | |
2947 | pr_warn("bad data relo against section %u\n", shdr_idx); | |
2948 | return -LIBBPF_ERRNO__RELOC; | |
1f8e2bcb | 2949 | } |
1f8e2bcb AN |
2950 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { |
2951 | map = &obj->maps[map_idx]; | |
2952 | if (map->libbpf_type != type) | |
2953 | continue; | |
2954 | pr_debug("found data map %zd (%s, sec %d, off %zu) for insn %u\n", | |
2955 | map_idx, map->name, map->sec_idx, map->sec_offset, | |
2956 | insn_idx); | |
2957 | break; | |
2958 | } | |
2959 | if (map_idx >= nr_maps) { | |
2960 | pr_warn("data relo failed to find map for sec %u\n", | |
2961 | shdr_idx); | |
2962 | return -LIBBPF_ERRNO__RELOC; | |
2963 | } | |
2964 | ||
2965 | reloc_desc->type = RELO_DATA; | |
2966 | reloc_desc->insn_idx = insn_idx; | |
2967 | reloc_desc->map_idx = map_idx; | |
53f8dd43 | 2968 | reloc_desc->sym_off = sym->st_value; |
1f8e2bcb AN |
2969 | return 0; |
2970 | } | |
2971 | ||
34090915 | 2972 | static int |
48cca7e4 AS |
2973 | bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, |
2974 | Elf_Data *data, struct bpf_object *obj) | |
34090915 | 2975 | { |
48cca7e4 | 2976 | Elf_Data *symbols = obj->efile.symbols; |
1f8e2bcb | 2977 | int err, i, nrels; |
34090915 | 2978 | |
399dc65e | 2979 | pr_debug("collecting relocating info for: '%s'\n", prog->section_name); |
34090915 WN |
2980 | nrels = shdr->sh_size / shdr->sh_entsize; |
2981 | ||
2982 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
2983 | if (!prog->reloc_desc) { | |
be18010e | 2984 | pr_warn("failed to alloc memory in relocation\n"); |
34090915 WN |
2985 | return -ENOMEM; |
2986 | } | |
2987 | prog->nr_reloc = nrels; | |
2988 | ||
2989 | for (i = 0; i < nrels; i++) { | |
d859900c | 2990 | const char *name; |
1f8e2bcb | 2991 | __u32 insn_idx; |
abd29c93 AN |
2992 | GElf_Sym sym; |
2993 | GElf_Rel rel; | |
34090915 WN |
2994 | |
2995 | if (!gelf_getrel(data, i, &rel)) { | |
be18010e | 2996 | pr_warn("relocation: failed to get %d reloc\n", i); |
6371ca3b | 2997 | return -LIBBPF_ERRNO__FORMAT; |
34090915 | 2998 | } |
399dc65e | 2999 | if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { |
be18010e KW |
3000 | pr_warn("relocation: symbol %"PRIx64" not found\n", |
3001 | GELF_R_SYM(rel.r_info)); | |
6371ca3b | 3002 | return -LIBBPF_ERRNO__FORMAT; |
34090915 | 3003 | } |
1f8e2bcb AN |
3004 | if (rel.r_offset % sizeof(struct bpf_insn)) |
3005 | return -LIBBPF_ERRNO__FORMAT; | |
d859900c | 3006 | |
1f8e2bcb | 3007 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); |
d859900c DB |
3008 | name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, |
3009 | sym.st_name) ? : "<?>"; | |
3010 | ||
679152d3 AN |
3011 | pr_debug("relo for shdr %u, symb %zu, value %zu, type %d, bind %d, name %d (\'%s\'), insn %u\n", |
3012 | (__u32)sym.st_shndx, (size_t)GELF_R_SYM(rel.r_info), | |
3013 | (size_t)sym.st_value, GELF_ST_TYPE(sym.st_info), | |
1f8e2bcb AN |
3014 | GELF_ST_BIND(sym.st_info), sym.st_name, name, |
3015 | insn_idx); | |
666810e8 | 3016 | |
1f8e2bcb AN |
3017 | err = bpf_program__record_reloc(prog, &prog->reloc_desc[i], |
3018 | insn_idx, name, &sym, &rel); | |
3019 | if (err) | |
3020 | return err; | |
34090915 WN |
3021 | } |
3022 | return 0; | |
3023 | } | |
3024 | ||
abd29c93 | 3025 | static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) |
8a138aed MKL |
3026 | { |
3027 | struct bpf_map_def *def = &map->def; | |
d859900c | 3028 | __u32 key_type_id = 0, value_type_id = 0; |
96408c43 | 3029 | int ret; |
8a138aed | 3030 | |
590a0088 MKL |
3031 | /* if it's BTF-defined map, we don't need to search for type IDs. |
3032 | * For struct_ops map, it does not need btf_key_type_id and | |
3033 | * btf_value_type_id. | |
3034 | */ | |
3035 | if (map->sec_idx == obj->efile.btf_maps_shndx || | |
3036 | bpf_map__is_struct_ops(map)) | |
abd29c93 AN |
3037 | return 0; |
3038 | ||
d859900c | 3039 | if (!bpf_map__is_internal(map)) { |
abd29c93 | 3040 | ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, |
d859900c DB |
3041 | def->value_size, &key_type_id, |
3042 | &value_type_id); | |
3043 | } else { | |
3044 | /* | |
3045 | * LLVM annotates global data differently in BTF, that is, | |
3046 | * only as '.data', '.bss' or '.rodata'. | |
3047 | */ | |
abd29c93 | 3048 | ret = btf__find_by_name(obj->btf, |
d859900c DB |
3049 | libbpf_type_to_btf_name[map->libbpf_type]); |
3050 | } | |
3051 | if (ret < 0) | |
96408c43 | 3052 | return ret; |
8a138aed | 3053 | |
96408c43 | 3054 | map->btf_key_type_id = key_type_id; |
d859900c DB |
3055 | map->btf_value_type_id = bpf_map__is_internal(map) ? |
3056 | ret : value_type_id; | |
8a138aed MKL |
3057 | return 0; |
3058 | } | |
3059 | ||
26736eb9 JK |
3060 | int bpf_map__reuse_fd(struct bpf_map *map, int fd) |
3061 | { | |
3062 | struct bpf_map_info info = {}; | |
3063 | __u32 len = sizeof(info); | |
3064 | int new_fd, err; | |
3065 | char *new_name; | |
3066 | ||
3067 | err = bpf_obj_get_info_by_fd(fd, &info, &len); | |
3068 | if (err) | |
3069 | return err; | |
3070 | ||
3071 | new_name = strdup(info.name); | |
3072 | if (!new_name) | |
3073 | return -errno; | |
3074 | ||
3075 | new_fd = open("/", O_RDONLY | O_CLOEXEC); | |
d1b4574a THJ |
3076 | if (new_fd < 0) { |
3077 | err = -errno; | |
26736eb9 | 3078 | goto err_free_new_name; |
d1b4574a | 3079 | } |
26736eb9 JK |
3080 | |
3081 | new_fd = dup3(fd, new_fd, O_CLOEXEC); | |
d1b4574a THJ |
3082 | if (new_fd < 0) { |
3083 | err = -errno; | |
26736eb9 | 3084 | goto err_close_new_fd; |
d1b4574a | 3085 | } |
26736eb9 JK |
3086 | |
3087 | err = zclose(map->fd); | |
d1b4574a THJ |
3088 | if (err) { |
3089 | err = -errno; | |
26736eb9 | 3090 | goto err_close_new_fd; |
d1b4574a | 3091 | } |
26736eb9 JK |
3092 | free(map->name); |
3093 | ||
3094 | map->fd = new_fd; | |
3095 | map->name = new_name; | |
3096 | map->def.type = info.type; | |
3097 | map->def.key_size = info.key_size; | |
3098 | map->def.value_size = info.value_size; | |
3099 | map->def.max_entries = info.max_entries; | |
3100 | map->def.map_flags = info.map_flags; | |
3101 | map->btf_key_type_id = info.btf_key_type_id; | |
3102 | map->btf_value_type_id = info.btf_value_type_id; | |
ec6d5f47 | 3103 | map->reused = true; |
26736eb9 JK |
3104 | |
3105 | return 0; | |
3106 | ||
3107 | err_close_new_fd: | |
3108 | close(new_fd); | |
3109 | err_free_new_name: | |
3110 | free(new_name); | |
d1b4574a | 3111 | return err; |
26736eb9 JK |
3112 | } |
3113 | ||
1a11a4c7 AI |
3114 | int bpf_map__resize(struct bpf_map *map, __u32 max_entries) |
3115 | { | |
3116 | if (!map || !max_entries) | |
3117 | return -EINVAL; | |
3118 | ||
3119 | /* If map already created, its attributes can't be changed. */ | |
3120 | if (map->fd >= 0) | |
3121 | return -EBUSY; | |
3122 | ||
3123 | map->def.max_entries = max_entries; | |
3124 | ||
3125 | return 0; | |
3126 | } | |
3127 | ||
47eff617 SF |
3128 | static int |
3129 | bpf_object__probe_name(struct bpf_object *obj) | |
3130 | { | |
3131 | struct bpf_load_program_attr attr; | |
3132 | char *cp, errmsg[STRERR_BUFSIZE]; | |
3133 | struct bpf_insn insns[] = { | |
3134 | BPF_MOV64_IMM(BPF_REG_0, 0), | |
3135 | BPF_EXIT_INSN(), | |
3136 | }; | |
3137 | int ret; | |
3138 | ||
3139 | /* make sure basic loading works */ | |
3140 | ||
3141 | memset(&attr, 0, sizeof(attr)); | |
3142 | attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; | |
3143 | attr.insns = insns; | |
3144 | attr.insns_cnt = ARRAY_SIZE(insns); | |
3145 | attr.license = "GPL"; | |
3146 | ||
3147 | ret = bpf_load_program_xattr(&attr, NULL, 0); | |
3148 | if (ret < 0) { | |
3149 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
be18010e KW |
3150 | pr_warn("Error in %s():%s(%d). Couldn't load basic 'r0 = 0' BPF program.\n", |
3151 | __func__, cp, errno); | |
47eff617 SF |
3152 | return -errno; |
3153 | } | |
3154 | close(ret); | |
3155 | ||
3156 | /* now try the same program, but with the name */ | |
3157 | ||
3158 | attr.name = "test"; | |
3159 | ret = bpf_load_program_xattr(&attr, NULL, 0); | |
3160 | if (ret >= 0) { | |
3161 | obj->caps.name = 1; | |
3162 | close(ret); | |
3163 | } | |
3164 | ||
3165 | return 0; | |
3166 | } | |
3167 | ||
8837fe5d DB |
3168 | static int |
3169 | bpf_object__probe_global_data(struct bpf_object *obj) | |
3170 | { | |
3171 | struct bpf_load_program_attr prg_attr; | |
3172 | struct bpf_create_map_attr map_attr; | |
3173 | char *cp, errmsg[STRERR_BUFSIZE]; | |
3174 | struct bpf_insn insns[] = { | |
3175 | BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), | |
3176 | BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), | |
3177 | BPF_MOV64_IMM(BPF_REG_0, 0), | |
3178 | BPF_EXIT_INSN(), | |
3179 | }; | |
3180 | int ret, map; | |
3181 | ||
3182 | memset(&map_attr, 0, sizeof(map_attr)); | |
3183 | map_attr.map_type = BPF_MAP_TYPE_ARRAY; | |
3184 | map_attr.key_size = sizeof(int); | |
3185 | map_attr.value_size = 32; | |
3186 | map_attr.max_entries = 1; | |
3187 | ||
3188 | map = bpf_create_map_xattr(&map_attr); | |
3189 | if (map < 0) { | |
3190 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); | |
be18010e KW |
3191 | pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", |
3192 | __func__, cp, errno); | |
8837fe5d DB |
3193 | return -errno; |
3194 | } | |
3195 | ||
3196 | insns[0].imm = map; | |
3197 | ||
3198 | memset(&prg_attr, 0, sizeof(prg_attr)); | |
3199 | prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; | |
3200 | prg_attr.insns = insns; | |
3201 | prg_attr.insns_cnt = ARRAY_SIZE(insns); | |
3202 | prg_attr.license = "GPL"; | |
3203 | ||
3204 | ret = bpf_load_program_xattr(&prg_attr, NULL, 0); | |
3205 | if (ret >= 0) { | |
3206 | obj->caps.global_data = 1; | |
3207 | close(ret); | |
3208 | } | |
3209 | ||
3210 | close(map); | |
3211 | return 0; | |
3212 | } | |
3213 | ||
d7c4b398 AN |
3214 | static int bpf_object__probe_btf_func(struct bpf_object *obj) |
3215 | { | |
8983b731 | 3216 | static const char strs[] = "\0int\0x\0a"; |
d7c4b398 AN |
3217 | /* void x(int a) {} */ |
3218 | __u32 types[] = { | |
3219 | /* int */ | |
3220 | BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ | |
3221 | /* FUNC_PROTO */ /* [2] */ | |
3222 | BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), | |
3223 | BTF_PARAM_ENC(7, 1), | |
3224 | /* FUNC x */ /* [3] */ | |
3225 | BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), | |
3226 | }; | |
cfd49210 | 3227 | int btf_fd; |
d7c4b398 | 3228 | |
cfd49210 MR |
3229 | btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), |
3230 | strs, sizeof(strs)); | |
3231 | if (btf_fd >= 0) { | |
d7c4b398 | 3232 | obj->caps.btf_func = 1; |
cfd49210 MR |
3233 | close(btf_fd); |
3234 | return 1; | |
3235 | } | |
3236 | ||
d7c4b398 AN |
3237 | return 0; |
3238 | } | |
3239 | ||
2d3eb67f AS |
3240 | static int bpf_object__probe_btf_func_global(struct bpf_object *obj) |
3241 | { | |
3242 | static const char strs[] = "\0int\0x\0a"; | |
3243 | /* static void x(int a) {} */ | |
3244 | __u32 types[] = { | |
3245 | /* int */ | |
3246 | BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ | |
3247 | /* FUNC_PROTO */ /* [2] */ | |
3248 | BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), | |
3249 | BTF_PARAM_ENC(7, 1), | |
3250 | /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ | |
3251 | BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), | |
3252 | }; | |
3253 | int btf_fd; | |
3254 | ||
3255 | btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), | |
3256 | strs, sizeof(strs)); | |
3257 | if (btf_fd >= 0) { | |
3258 | obj->caps.btf_func_global = 1; | |
3259 | close(btf_fd); | |
3260 | return 1; | |
3261 | } | |
3262 | ||
3263 | return 0; | |
3264 | } | |
3265 | ||
d7c4b398 AN |
3266 | static int bpf_object__probe_btf_datasec(struct bpf_object *obj) |
3267 | { | |
8983b731 | 3268 | static const char strs[] = "\0x\0.data"; |
d7c4b398 AN |
3269 | /* static int a; */ |
3270 | __u32 types[] = { | |
3271 | /* int */ | |
3272 | BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ | |
3273 | /* VAR x */ /* [2] */ | |
3274 | BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), | |
3275 | BTF_VAR_STATIC, | |
3276 | /* DATASEC val */ /* [3] */ | |
3277 | BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), | |
3278 | BTF_VAR_SECINFO_ENC(2, 0, 4), | |
3279 | }; | |
cfd49210 | 3280 | int btf_fd; |
d7c4b398 | 3281 | |
cfd49210 MR |
3282 | btf_fd = libbpf__load_raw_btf((char *)types, sizeof(types), |
3283 | strs, sizeof(strs)); | |
3284 | if (btf_fd >= 0) { | |
d7c4b398 | 3285 | obj->caps.btf_datasec = 1; |
cfd49210 MR |
3286 | close(btf_fd); |
3287 | return 1; | |
3288 | } | |
3289 | ||
d7c4b398 AN |
3290 | return 0; |
3291 | } | |
3292 | ||
7fe74b43 AN |
3293 | static int bpf_object__probe_array_mmap(struct bpf_object *obj) |
3294 | { | |
3295 | struct bpf_create_map_attr attr = { | |
3296 | .map_type = BPF_MAP_TYPE_ARRAY, | |
3297 | .map_flags = BPF_F_MMAPABLE, | |
3298 | .key_size = sizeof(int), | |
3299 | .value_size = sizeof(int), | |
3300 | .max_entries = 1, | |
3301 | }; | |
3302 | int fd; | |
3303 | ||
3304 | fd = bpf_create_map_xattr(&attr); | |
3305 | if (fd >= 0) { | |
3306 | obj->caps.array_mmap = 1; | |
3307 | close(fd); | |
3308 | return 1; | |
3309 | } | |
3310 | ||
3311 | return 0; | |
3312 | } | |
3313 | ||
47eff617 SF |
3314 | static int |
3315 | bpf_object__probe_caps(struct bpf_object *obj) | |
3316 | { | |
8837fe5d DB |
3317 | int (*probe_fn[])(struct bpf_object *obj) = { |
3318 | bpf_object__probe_name, | |
3319 | bpf_object__probe_global_data, | |
d7c4b398 | 3320 | bpf_object__probe_btf_func, |
2d3eb67f | 3321 | bpf_object__probe_btf_func_global, |
d7c4b398 | 3322 | bpf_object__probe_btf_datasec, |
7fe74b43 | 3323 | bpf_object__probe_array_mmap, |
8837fe5d DB |
3324 | }; |
3325 | int i, ret; | |
3326 | ||
3327 | for (i = 0; i < ARRAY_SIZE(probe_fn); i++) { | |
3328 | ret = probe_fn[i](obj); | |
3329 | if (ret < 0) | |
15ea164e | 3330 | pr_debug("Probe #%d failed with %d.\n", i, ret); |
8837fe5d DB |
3331 | } |
3332 | ||
3333 | return 0; | |
47eff617 SF |
3334 | } |
3335 | ||
57a00f41 THJ |
3336 | static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) |
3337 | { | |
3338 | struct bpf_map_info map_info = {}; | |
3339 | char msg[STRERR_BUFSIZE]; | |
3340 | __u32 map_info_len; | |
3341 | ||
3342 | map_info_len = sizeof(map_info); | |
3343 | ||
3344 | if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len)) { | |
3345 | pr_warn("failed to get map info for map FD %d: %s\n", | |
3346 | map_fd, libbpf_strerror_r(errno, msg, sizeof(msg))); | |
3347 | return false; | |
3348 | } | |
3349 | ||
3350 | return (map_info.type == map->def.type && | |
3351 | map_info.key_size == map->def.key_size && | |
3352 | map_info.value_size == map->def.value_size && | |
3353 | map_info.max_entries == map->def.max_entries && | |
3354 | map_info.map_flags == map->def.map_flags); | |
3355 | } | |
3356 | ||
3357 | static int | |
3358 | bpf_object__reuse_map(struct bpf_map *map) | |
3359 | { | |
3360 | char *cp, errmsg[STRERR_BUFSIZE]; | |
3361 | int err, pin_fd; | |
3362 | ||
3363 | pin_fd = bpf_obj_get(map->pin_path); | |
3364 | if (pin_fd < 0) { | |
3365 | err = -errno; | |
3366 | if (err == -ENOENT) { | |
3367 | pr_debug("found no pinned map to reuse at '%s'\n", | |
3368 | map->pin_path); | |
3369 | return 0; | |
3370 | } | |
3371 | ||
3372 | cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); | |
3373 | pr_warn("couldn't retrieve pinned map '%s': %s\n", | |
3374 | map->pin_path, cp); | |
3375 | return err; | |
3376 | } | |
3377 | ||
3378 | if (!map_is_reuse_compat(map, pin_fd)) { | |
3379 | pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", | |
3380 | map->pin_path); | |
3381 | close(pin_fd); | |
3382 | return -EINVAL; | |
3383 | } | |
3384 | ||
3385 | err = bpf_map__reuse_fd(map, pin_fd); | |
3386 | if (err) { | |
3387 | close(pin_fd); | |
3388 | return err; | |
3389 | } | |
3390 | map->pinned = true; | |
3391 | pr_debug("reused pinned map at '%s'\n", map->pin_path); | |
3392 | ||
3393 | return 0; | |
3394 | } | |
3395 | ||
d859900c DB |
3396 | static int |
3397 | bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) | |
3398 | { | |
166750bc | 3399 | enum libbpf_map_type map_type = map->libbpf_type; |
d859900c DB |
3400 | char *cp, errmsg[STRERR_BUFSIZE]; |
3401 | int err, zero = 0; | |
d859900c | 3402 | |
166750bc AN |
3403 | /* kernel already zero-initializes .bss map. */ |
3404 | if (map_type == LIBBPF_MAP_BSS) | |
d859900c DB |
3405 | return 0; |
3406 | ||
eba9c5f4 AN |
3407 | err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); |
3408 | if (err) { | |
3409 | err = -errno; | |
3410 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
3411 | pr_warn("Error setting initial map(%s) contents: %s\n", | |
3412 | map->name, cp); | |
3413 | return err; | |
3414 | } | |
d859900c | 3415 | |
81bfdd08 AN |
3416 | /* Freeze .rodata and .kconfig map as read-only from syscall side. */ |
3417 | if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { | |
d859900c DB |
3418 | err = bpf_map_freeze(map->fd); |
3419 | if (err) { | |
eba9c5f4 AN |
3420 | err = -errno; |
3421 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
be18010e KW |
3422 | pr_warn("Error freezing map(%s) as read-only: %s\n", |
3423 | map->name, cp); | |
eba9c5f4 | 3424 | return err; |
d859900c DB |
3425 | } |
3426 | } | |
eba9c5f4 | 3427 | return 0; |
d859900c DB |
3428 | } |
3429 | ||
52d3352e WN |
3430 | static int |
3431 | bpf_object__create_maps(struct bpf_object *obj) | |
3432 | { | |
8a138aed | 3433 | struct bpf_create_map_attr create_attr = {}; |
d7ff34d5 | 3434 | int nr_cpus = 0; |
52d3352e | 3435 | unsigned int i; |
8a138aed | 3436 | int err; |
52d3352e | 3437 | |
9d759a9b | 3438 | for (i = 0; i < obj->nr_maps; i++) { |
8a138aed MKL |
3439 | struct bpf_map *map = &obj->maps[i]; |
3440 | struct bpf_map_def *def = &map->def; | |
1ce6a9fc | 3441 | char *cp, errmsg[STRERR_BUFSIZE]; |
8a138aed MKL |
3442 | int *pfd = &map->fd; |
3443 | ||
57a00f41 THJ |
3444 | if (map->pin_path) { |
3445 | err = bpf_object__reuse_map(map); | |
3446 | if (err) { | |
3447 | pr_warn("error reusing pinned map %s\n", | |
3448 | map->name); | |
3449 | return err; | |
3450 | } | |
3451 | } | |
3452 | ||
26736eb9 JK |
3453 | if (map->fd >= 0) { |
3454 | pr_debug("skip map create (preset) %s: fd=%d\n", | |
3455 | map->name, map->fd); | |
3456 | continue; | |
3457 | } | |
3458 | ||
94cb310c SF |
3459 | if (obj->caps.name) |
3460 | create_attr.name = map->name; | |
f0307a7e | 3461 | create_attr.map_ifindex = map->map_ifindex; |
8a138aed MKL |
3462 | create_attr.map_type = def->type; |
3463 | create_attr.map_flags = def->map_flags; | |
3464 | create_attr.key_size = def->key_size; | |
3465 | create_attr.value_size = def->value_size; | |
d7ff34d5 AN |
3466 | if (def->type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && |
3467 | !def->max_entries) { | |
3468 | if (!nr_cpus) | |
3469 | nr_cpus = libbpf_num_possible_cpus(); | |
3470 | if (nr_cpus < 0) { | |
be18010e KW |
3471 | pr_warn("failed to determine number of system CPUs: %d\n", |
3472 | nr_cpus); | |
d7ff34d5 AN |
3473 | err = nr_cpus; |
3474 | goto err_out; | |
3475 | } | |
3476 | pr_debug("map '%s': setting size to %d\n", | |
3477 | map->name, nr_cpus); | |
3478 | create_attr.max_entries = nr_cpus; | |
3479 | } else { | |
3480 | create_attr.max_entries = def->max_entries; | |
3481 | } | |
e55d54f4 | 3482 | create_attr.btf_fd = 0; |
61746dbe MKL |
3483 | create_attr.btf_key_type_id = 0; |
3484 | create_attr.btf_value_type_id = 0; | |
addb9fc9 NS |
3485 | if (bpf_map_type__is_map_in_map(def->type) && |
3486 | map->inner_map_fd >= 0) | |
3487 | create_attr.inner_map_fd = map->inner_map_fd; | |
590a0088 MKL |
3488 | if (bpf_map__is_struct_ops(map)) |
3489 | create_attr.btf_vmlinux_value_type_id = | |
3490 | map->btf_vmlinux_value_type_id; | |
8a138aed | 3491 | |
abd29c93 | 3492 | if (obj->btf && !bpf_map_find_btf_info(obj, map)) { |
8a138aed | 3493 | create_attr.btf_fd = btf__fd(obj->btf); |
61746dbe MKL |
3494 | create_attr.btf_key_type_id = map->btf_key_type_id; |
3495 | create_attr.btf_value_type_id = map->btf_value_type_id; | |
8a138aed MKL |
3496 | } |
3497 | ||
3498 | *pfd = bpf_create_map_xattr(&create_attr); | |
e55d54f4 AN |
3499 | if (*pfd < 0 && (create_attr.btf_key_type_id || |
3500 | create_attr.btf_value_type_id)) { | |
d7ff34d5 AN |
3501 | err = -errno; |
3502 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); | |
be18010e KW |
3503 | pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", |
3504 | map->name, cp, err); | |
e55d54f4 | 3505 | create_attr.btf_fd = 0; |
61746dbe MKL |
3506 | create_attr.btf_key_type_id = 0; |
3507 | create_attr.btf_value_type_id = 0; | |
3508 | map->btf_key_type_id = 0; | |
3509 | map->btf_value_type_id = 0; | |
8a138aed MKL |
3510 | *pfd = bpf_create_map_xattr(&create_attr); |
3511 | } | |
3512 | ||
52d3352e WN |
3513 | if (*pfd < 0) { |
3514 | size_t j; | |
52d3352e | 3515 | |
d7ff34d5 | 3516 | err = -errno; |
d859900c | 3517 | err_out: |
d7ff34d5 | 3518 | cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); |
be18010e KW |
3519 | pr_warn("failed to create map (name: '%s'): %s(%d)\n", |
3520 | map->name, cp, err); | |
dc3a2d25 | 3521 | pr_perm_msg(err); |
52d3352e | 3522 | for (j = 0; j < i; j++) |
9d759a9b | 3523 | zclose(obj->maps[j].fd); |
52d3352e WN |
3524 | return err; |
3525 | } | |
d859900c DB |
3526 | |
3527 | if (bpf_map__is_internal(map)) { | |
3528 | err = bpf_object__populate_internal_map(obj, map); | |
3529 | if (err < 0) { | |
3530 | zclose(*pfd); | |
3531 | goto err_out; | |
3532 | } | |
3533 | } | |
3534 | ||
57a00f41 THJ |
3535 | if (map->pin_path && !map->pinned) { |
3536 | err = bpf_map__pin(map, NULL); | |
3537 | if (err) { | |
3538 | pr_warn("failed to auto-pin map name '%s' at '%s'\n", | |
3539 | map->name, map->pin_path); | |
3540 | return err; | |
3541 | } | |
3542 | } | |
3543 | ||
76e1022b | 3544 | pr_debug("created map %s: fd=%d\n", map->name, *pfd); |
52d3352e WN |
3545 | } |
3546 | ||
52d3352e WN |
3547 | return 0; |
3548 | } | |
3549 | ||
f0187f0b MKL |
3550 | static int |
3551 | check_btf_ext_reloc_err(struct bpf_program *prog, int err, | |
3552 | void *btf_prog_info, const char *info_name) | |
3553 | { | |
3554 | if (err != -ENOENT) { | |
be18010e KW |
3555 | pr_warn("Error in loading %s for sec %s.\n", |
3556 | info_name, prog->section_name); | |
f0187f0b MKL |
3557 | return err; |
3558 | } | |
3559 | ||
3560 | /* err == -ENOENT (i.e. prog->section_name not found in btf_ext) */ | |
3561 | ||
3562 | if (btf_prog_info) { | |
3563 | /* | |
3564 | * Some info has already been found but has problem | |
399dc65e | 3565 | * in the last btf_ext reloc. Must have to error out. |
f0187f0b | 3566 | */ |
be18010e KW |
3567 | pr_warn("Error in relocating %s for sec %s.\n", |
3568 | info_name, prog->section_name); | |
f0187f0b MKL |
3569 | return err; |
3570 | } | |
3571 | ||
399dc65e | 3572 | /* Have problem loading the very first info. Ignore the rest. */ |
be18010e KW |
3573 | pr_warn("Cannot find %s for main program sec %s. Ignore all %s.\n", |
3574 | info_name, prog->section_name, info_name); | |
f0187f0b MKL |
3575 | return 0; |
3576 | } | |
3577 | ||
3578 | static int | |
3579 | bpf_program_reloc_btf_ext(struct bpf_program *prog, struct bpf_object *obj, | |
3580 | const char *section_name, __u32 insn_offset) | |
3581 | { | |
3582 | int err; | |
3583 | ||
3584 | if (!insn_offset || prog->func_info) { | |
3585 | /* | |
3586 | * !insn_offset => main program | |
3587 | * | |
3588 | * For sub prog, the main program's func_info has to | |
3589 | * be loaded first (i.e. prog->func_info != NULL) | |
3590 | */ | |
3591 | err = btf_ext__reloc_func_info(obj->btf, obj->btf_ext, | |
3592 | section_name, insn_offset, | |
3593 | &prog->func_info, | |
3594 | &prog->func_info_cnt); | |
3595 | if (err) | |
3596 | return check_btf_ext_reloc_err(prog, err, | |
3597 | prog->func_info, | |
3598 | "bpf_func_info"); | |
3599 | ||
3600 | prog->func_info_rec_size = btf_ext__func_info_rec_size(obj->btf_ext); | |
3601 | } | |
3602 | ||
3d650141 MKL |
3603 | if (!insn_offset || prog->line_info) { |
3604 | err = btf_ext__reloc_line_info(obj->btf, obj->btf_ext, | |
3605 | section_name, insn_offset, | |
3606 | &prog->line_info, | |
3607 | &prog->line_info_cnt); | |
3608 | if (err) | |
3609 | return check_btf_ext_reloc_err(prog, err, | |
3610 | prog->line_info, | |
3611 | "bpf_line_info"); | |
3612 | ||
3613 | prog->line_info_rec_size = btf_ext__line_info_rec_size(obj->btf_ext); | |
3614 | } | |
3615 | ||
f0187f0b MKL |
3616 | return 0; |
3617 | } | |
3618 | ||
ddc7c304 AN |
3619 | #define BPF_CORE_SPEC_MAX_LEN 64 |
3620 | ||
3621 | /* represents BPF CO-RE field or array element accessor */ | |
3622 | struct bpf_core_accessor { | |
3623 | __u32 type_id; /* struct/union type or array element type */ | |
3624 | __u32 idx; /* field index or array index */ | |
3625 | const char *name; /* field name or NULL for array accessor */ | |
3626 | }; | |
3627 | ||
3628 | struct bpf_core_spec { | |
3629 | const struct btf *btf; | |
3630 | /* high-level spec: named fields and array indices only */ | |
3631 | struct bpf_core_accessor spec[BPF_CORE_SPEC_MAX_LEN]; | |
3632 | /* high-level spec length */ | |
3633 | int len; | |
3634 | /* raw, low-level spec: 1-to-1 with accessor spec string */ | |
3635 | int raw_spec[BPF_CORE_SPEC_MAX_LEN]; | |
3636 | /* raw spec length */ | |
3637 | int raw_len; | |
ee26dade AN |
3638 | /* field bit offset represented by spec */ |
3639 | __u32 bit_offset; | |
ddc7c304 AN |
3640 | }; |
3641 | ||
3642 | static bool str_is_empty(const char *s) | |
3643 | { | |
3644 | return !s || !s[0]; | |
3645 | } | |
3646 | ||
1b484b30 AN |
3647 | static bool is_flex_arr(const struct btf *btf, |
3648 | const struct bpf_core_accessor *acc, | |
3649 | const struct btf_array *arr) | |
3650 | { | |
3651 | const struct btf_type *t; | |
3652 | ||
3653 | /* not a flexible array, if not inside a struct or has non-zero size */ | |
3654 | if (!acc->name || arr->nelems > 0) | |
3655 | return false; | |
3656 | ||
3657 | /* has to be the last member of enclosing struct */ | |
3658 | t = btf__type_by_id(btf, acc->type_id); | |
3659 | return acc->idx == btf_vlen(t) - 1; | |
3660 | } | |
3661 | ||
ddc7c304 | 3662 | /* |
511bb008 | 3663 | * Turn bpf_field_reloc into a low- and high-level spec representation, |
ddc7c304 | 3664 | * validating correctness along the way, as well as calculating resulting |
ee26dade AN |
3665 | * field bit offset, specified by accessor string. Low-level spec captures |
3666 | * every single level of nestedness, including traversing anonymous | |
ddc7c304 AN |
3667 | * struct/union members. High-level one only captures semantically meaningful |
3668 | * "turning points": named fields and array indicies. | |
3669 | * E.g., for this case: | |
3670 | * | |
3671 | * struct sample { | |
3672 | * int __unimportant; | |
3673 | * struct { | |
3674 | * int __1; | |
3675 | * int __2; | |
3676 | * int a[7]; | |
3677 | * }; | |
3678 | * }; | |
3679 | * | |
3680 | * struct sample *s = ...; | |
3681 | * | |
3682 | * int x = &s->a[3]; // access string = '0:1:2:3' | |
3683 | * | |
3684 | * Low-level spec has 1:1 mapping with each element of access string (it's | |
3685 | * just a parsed access string representation): [0, 1, 2, 3]. | |
3686 | * | |
3687 | * High-level spec will capture only 3 points: | |
3688 | * - intial zero-index access by pointer (&s->... is the same as &s[0]...); | |
3689 | * - field 'a' access (corresponds to '2' in low-level spec); | |
3690 | * - array element #3 access (corresponds to '3' in low-level spec). | |
3691 | * | |
3692 | */ | |
3693 | static int bpf_core_spec_parse(const struct btf *btf, | |
3694 | __u32 type_id, | |
3695 | const char *spec_str, | |
3696 | struct bpf_core_spec *spec) | |
3697 | { | |
3698 | int access_idx, parsed_len, i; | |
1b484b30 | 3699 | struct bpf_core_accessor *acc; |
ddc7c304 AN |
3700 | const struct btf_type *t; |
3701 | const char *name; | |
3702 | __u32 id; | |
3703 | __s64 sz; | |
3704 | ||
3705 | if (str_is_empty(spec_str) || *spec_str == ':') | |
3706 | return -EINVAL; | |
3707 | ||
3708 | memset(spec, 0, sizeof(*spec)); | |
3709 | spec->btf = btf; | |
3710 | ||
3711 | /* parse spec_str="0:1:2:3:4" into array raw_spec=[0, 1, 2, 3, 4] */ | |
3712 | while (*spec_str) { | |
3713 | if (*spec_str == ':') | |
3714 | ++spec_str; | |
3715 | if (sscanf(spec_str, "%d%n", &access_idx, &parsed_len) != 1) | |
3716 | return -EINVAL; | |
3717 | if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) | |
3718 | return -E2BIG; | |
3719 | spec_str += parsed_len; | |
3720 | spec->raw_spec[spec->raw_len++] = access_idx; | |
3721 | } | |
3722 | ||
3723 | if (spec->raw_len == 0) | |
3724 | return -EINVAL; | |
3725 | ||
3726 | /* first spec value is always reloc type array index */ | |
3727 | t = skip_mods_and_typedefs(btf, type_id, &id); | |
3728 | if (!t) | |
3729 | return -EINVAL; | |
3730 | ||
3731 | access_idx = spec->raw_spec[0]; | |
3732 | spec->spec[0].type_id = id; | |
3733 | spec->spec[0].idx = access_idx; | |
3734 | spec->len++; | |
3735 | ||
3736 | sz = btf__resolve_size(btf, id); | |
3737 | if (sz < 0) | |
3738 | return sz; | |
ee26dade | 3739 | spec->bit_offset = access_idx * sz * 8; |
ddc7c304 AN |
3740 | |
3741 | for (i = 1; i < spec->raw_len; i++) { | |
3742 | t = skip_mods_and_typedefs(btf, id, &id); | |
3743 | if (!t) | |
3744 | return -EINVAL; | |
3745 | ||
3746 | access_idx = spec->raw_spec[i]; | |
1b484b30 | 3747 | acc = &spec->spec[spec->len]; |
ddc7c304 AN |
3748 | |
3749 | if (btf_is_composite(t)) { | |
3750 | const struct btf_member *m; | |
ee26dade | 3751 | __u32 bit_offset; |
ddc7c304 AN |
3752 | |
3753 | if (access_idx >= btf_vlen(t)) | |
3754 | return -EINVAL; | |
ddc7c304 | 3755 | |
ee26dade AN |
3756 | bit_offset = btf_member_bit_offset(t, access_idx); |
3757 | spec->bit_offset += bit_offset; | |
ddc7c304 AN |
3758 | |
3759 | m = btf_members(t) + access_idx; | |
3760 | if (m->name_off) { | |
3761 | name = btf__name_by_offset(btf, m->name_off); | |
3762 | if (str_is_empty(name)) | |
3763 | return -EINVAL; | |
3764 | ||
1b484b30 AN |
3765 | acc->type_id = id; |
3766 | acc->idx = access_idx; | |
3767 | acc->name = name; | |
ddc7c304 AN |
3768 | spec->len++; |
3769 | } | |
3770 | ||
3771 | id = m->type; | |
3772 | } else if (btf_is_array(t)) { | |
3773 | const struct btf_array *a = btf_array(t); | |
1b484b30 | 3774 | bool flex; |
ddc7c304 AN |
3775 | |
3776 | t = skip_mods_and_typedefs(btf, a->type, &id); | |
1b484b30 AN |
3777 | if (!t) |
3778 | return -EINVAL; | |
3779 | ||
3780 | flex = is_flex_arr(btf, acc - 1, a); | |
3781 | if (!flex && access_idx >= a->nelems) | |
ddc7c304 AN |
3782 | return -EINVAL; |
3783 | ||
3784 | spec->spec[spec->len].type_id = id; | |
3785 | spec->spec[spec->len].idx = access_idx; | |
3786 | spec->len++; | |
3787 | ||
3788 | sz = btf__resolve_size(btf, id); | |
3789 | if (sz < 0) | |
3790 | return sz; | |
ee26dade | 3791 | spec->bit_offset += access_idx * sz * 8; |
ddc7c304 | 3792 | } else { |
be18010e KW |
3793 | pr_warn("relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %d\n", |
3794 | type_id, spec_str, i, id, btf_kind(t)); | |
ddc7c304 AN |
3795 | return -EINVAL; |
3796 | } | |
3797 | } | |
3798 | ||
3799 | return 0; | |
3800 | } | |
3801 | ||
3802 | static bool bpf_core_is_flavor_sep(const char *s) | |
3803 | { | |
3804 | /* check X___Y name pattern, where X and Y are not underscores */ | |
3805 | return s[0] != '_' && /* X */ | |
3806 | s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ | |
3807 | s[4] != '_'; /* Y */ | |
3808 | } | |
3809 | ||
3810 | /* Given 'some_struct_name___with_flavor' return the length of a name prefix | |
3811 | * before last triple underscore. Struct name part after last triple | |
3812 | * underscore is ignored by BPF CO-RE relocation during relocation matching. | |
3813 | */ | |
3814 | static size_t bpf_core_essential_name_len(const char *name) | |
3815 | { | |
3816 | size_t n = strlen(name); | |
3817 | int i; | |
3818 | ||
3819 | for (i = n - 5; i >= 0; i--) { | |
3820 | if (bpf_core_is_flavor_sep(name + i)) | |
3821 | return i + 1; | |
3822 | } | |
3823 | return n; | |
3824 | } | |
3825 | ||
3826 | /* dynamically sized list of type IDs */ | |
3827 | struct ids_vec { | |
3828 | __u32 *data; | |
3829 | int len; | |
3830 | }; | |
3831 | ||
3832 | static void bpf_core_free_cands(struct ids_vec *cand_ids) | |
3833 | { | |
3834 | free(cand_ids->data); | |
3835 | free(cand_ids); | |
3836 | } | |
3837 | ||
3838 | static struct ids_vec *bpf_core_find_cands(const struct btf *local_btf, | |
3839 | __u32 local_type_id, | |
3840 | const struct btf *targ_btf) | |
3841 | { | |
3842 | size_t local_essent_len, targ_essent_len; | |
3843 | const char *local_name, *targ_name; | |
3844 | const struct btf_type *t; | |
3845 | struct ids_vec *cand_ids; | |
3846 | __u32 *new_ids; | |
3847 | int i, err, n; | |
3848 | ||
3849 | t = btf__type_by_id(local_btf, local_type_id); | |
3850 | if (!t) | |
3851 | return ERR_PTR(-EINVAL); | |
3852 | ||
3853 | local_name = btf__name_by_offset(local_btf, t->name_off); | |
3854 | if (str_is_empty(local_name)) | |
3855 | return ERR_PTR(-EINVAL); | |
3856 | local_essent_len = bpf_core_essential_name_len(local_name); | |
3857 | ||
3858 | cand_ids = calloc(1, sizeof(*cand_ids)); | |
3859 | if (!cand_ids) | |
3860 | return ERR_PTR(-ENOMEM); | |
3861 | ||
3862 | n = btf__get_nr_types(targ_btf); | |
3863 | for (i = 1; i <= n; i++) { | |
3864 | t = btf__type_by_id(targ_btf, i); | |
3865 | targ_name = btf__name_by_offset(targ_btf, t->name_off); | |
3866 | if (str_is_empty(targ_name)) | |
3867 | continue; | |
3868 | ||
3869 | targ_essent_len = bpf_core_essential_name_len(targ_name); | |
3870 | if (targ_essent_len != local_essent_len) | |
3871 | continue; | |
3872 | ||
3873 | if (strncmp(local_name, targ_name, local_essent_len) == 0) { | |
3874 | pr_debug("[%d] %s: found candidate [%d] %s\n", | |
3875 | local_type_id, local_name, i, targ_name); | |
35b9211c AN |
3876 | new_ids = reallocarray(cand_ids->data, |
3877 | cand_ids->len + 1, | |
3878 | sizeof(*cand_ids->data)); | |
ddc7c304 AN |
3879 | if (!new_ids) { |
3880 | err = -ENOMEM; | |
3881 | goto err_out; | |
3882 | } | |
3883 | cand_ids->data = new_ids; | |
3884 | cand_ids->data[cand_ids->len++] = i; | |
3885 | } | |
3886 | } | |
3887 | return cand_ids; | |
3888 | err_out: | |
3889 | bpf_core_free_cands(cand_ids); | |
3890 | return ERR_PTR(err); | |
3891 | } | |
3892 | ||
3893 | /* Check two types for compatibility, skipping const/volatile/restrict and | |
ee26dade | 3894 | * typedefs, to ensure we are relocating compatible entities: |
ddc7c304 | 3895 | * - any two STRUCTs/UNIONs are compatible and can be mixed; |
94f060e9 | 3896 | * - any two FWDs are compatible, if their names match (modulo flavor suffix); |
ddc7c304 | 3897 | * - any two PTRs are always compatible; |
94f060e9 AN |
3898 | * - for ENUMs, names should be the same (ignoring flavor suffix) or at |
3899 | * least one of enums should be anonymous; | |
ddc7c304 | 3900 | * - for ENUMs, check sizes, names are ignored; |
ee26dade | 3901 | * - for INT, size and signedness are ignored; |
ddc7c304 AN |
3902 | * - for ARRAY, dimensionality is ignored, element types are checked for |
3903 | * compatibility recursively; | |
3904 | * - everything else shouldn't be ever a target of relocation. | |
3905 | * These rules are not set in stone and probably will be adjusted as we get | |
3906 | * more experience with using BPF CO-RE relocations. | |
3907 | */ | |
3908 | static int bpf_core_fields_are_compat(const struct btf *local_btf, | |
3909 | __u32 local_id, | |
3910 | const struct btf *targ_btf, | |
3911 | __u32 targ_id) | |
3912 | { | |
3913 | const struct btf_type *local_type, *targ_type; | |
3914 | ||
3915 | recur: | |
3916 | local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); | |
3917 | targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); | |
3918 | if (!local_type || !targ_type) | |
3919 | return -EINVAL; | |
3920 | ||
3921 | if (btf_is_composite(local_type) && btf_is_composite(targ_type)) | |
3922 | return 1; | |
3923 | if (btf_kind(local_type) != btf_kind(targ_type)) | |
3924 | return 0; | |
3925 | ||
3926 | switch (btf_kind(local_type)) { | |
ddc7c304 AN |
3927 | case BTF_KIND_PTR: |
3928 | return 1; | |
94f060e9 AN |
3929 | case BTF_KIND_FWD: |
3930 | case BTF_KIND_ENUM: { | |
3931 | const char *local_name, *targ_name; | |
3932 | size_t local_len, targ_len; | |
3933 | ||
3934 | local_name = btf__name_by_offset(local_btf, | |
3935 | local_type->name_off); | |
3936 | targ_name = btf__name_by_offset(targ_btf, targ_type->name_off); | |
3937 | local_len = bpf_core_essential_name_len(local_name); | |
3938 | targ_len = bpf_core_essential_name_len(targ_name); | |
3939 | /* one of them is anonymous or both w/ same flavor-less names */ | |
3940 | return local_len == 0 || targ_len == 0 || | |
3941 | (local_len == targ_len && | |
3942 | strncmp(local_name, targ_name, local_len) == 0); | |
3943 | } | |
ddc7c304 | 3944 | case BTF_KIND_INT: |
ee26dade AN |
3945 | /* just reject deprecated bitfield-like integers; all other |
3946 | * integers are by default compatible between each other | |
3947 | */ | |
ddc7c304 | 3948 | return btf_int_offset(local_type) == 0 && |
ee26dade | 3949 | btf_int_offset(targ_type) == 0; |
ddc7c304 AN |
3950 | case BTF_KIND_ARRAY: |
3951 | local_id = btf_array(local_type)->type; | |
3952 | targ_id = btf_array(targ_type)->type; | |
3953 | goto recur; | |
3954 | default: | |
be18010e KW |
3955 | pr_warn("unexpected kind %d relocated, local [%d], target [%d]\n", |
3956 | btf_kind(local_type), local_id, targ_id); | |
ddc7c304 AN |
3957 | return 0; |
3958 | } | |
3959 | } | |
3960 | ||
3961 | /* | |
3962 | * Given single high-level named field accessor in local type, find | |
3963 | * corresponding high-level accessor for a target type. Along the way, | |
3964 | * maintain low-level spec for target as well. Also keep updating target | |
ee26dade | 3965 | * bit offset. |
ddc7c304 AN |
3966 | * |
3967 | * Searching is performed through recursive exhaustive enumeration of all | |
3968 | * fields of a struct/union. If there are any anonymous (embedded) | |
3969 | * structs/unions, they are recursively searched as well. If field with | |
3970 | * desired name is found, check compatibility between local and target types, | |
3971 | * before returning result. | |
3972 | * | |
3973 | * 1 is returned, if field is found. | |
3974 | * 0 is returned if no compatible field is found. | |
3975 | * <0 is returned on error. | |
3976 | */ | |
3977 | static int bpf_core_match_member(const struct btf *local_btf, | |
3978 | const struct bpf_core_accessor *local_acc, | |
3979 | const struct btf *targ_btf, | |
3980 | __u32 targ_id, | |
3981 | struct bpf_core_spec *spec, | |
3982 | __u32 *next_targ_id) | |
3983 | { | |
3984 | const struct btf_type *local_type, *targ_type; | |
3985 | const struct btf_member *local_member, *m; | |
3986 | const char *local_name, *targ_name; | |
3987 | __u32 local_id; | |
3988 | int i, n, found; | |
3989 | ||
3990 | targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); | |
3991 | if (!targ_type) | |
3992 | return -EINVAL; | |
3993 | if (!btf_is_composite(targ_type)) | |
3994 | return 0; | |
3995 | ||
3996 | local_id = local_acc->type_id; | |
3997 | local_type = btf__type_by_id(local_btf, local_id); | |
3998 | local_member = btf_members(local_type) + local_acc->idx; | |
3999 | local_name = btf__name_by_offset(local_btf, local_member->name_off); | |
4000 | ||
4001 | n = btf_vlen(targ_type); | |
4002 | m = btf_members(targ_type); | |
4003 | for (i = 0; i < n; i++, m++) { | |
ee26dade | 4004 | __u32 bit_offset; |
ddc7c304 | 4005 | |
ee26dade | 4006 | bit_offset = btf_member_bit_offset(targ_type, i); |
ddc7c304 AN |
4007 | |
4008 | /* too deep struct/union/array nesting */ | |
4009 | if (spec->raw_len == BPF_CORE_SPEC_MAX_LEN) | |
4010 | return -E2BIG; | |
4011 | ||
4012 | /* speculate this member will be the good one */ | |
ee26dade | 4013 | spec->bit_offset += bit_offset; |
ddc7c304 AN |
4014 | spec->raw_spec[spec->raw_len++] = i; |
4015 | ||
4016 | targ_name = btf__name_by_offset(targ_btf, m->name_off); | |
4017 | if (str_is_empty(targ_name)) { | |
4018 | /* embedded struct/union, we need to go deeper */ | |
4019 | found = bpf_core_match_member(local_btf, local_acc, | |
4020 | targ_btf, m->type, | |
4021 | spec, next_targ_id); | |
4022 | if (found) /* either found or error */ | |
4023 | return found; | |
4024 | } else if (strcmp(local_name, targ_name) == 0) { | |
4025 | /* matching named field */ | |
4026 | struct bpf_core_accessor *targ_acc; | |
4027 | ||
4028 | targ_acc = &spec->spec[spec->len++]; | |
4029 | targ_acc->type_id = targ_id; | |
4030 | targ_acc->idx = i; | |
4031 | targ_acc->name = targ_name; | |
4032 | ||
4033 | *next_targ_id = m->type; | |
4034 | found = bpf_core_fields_are_compat(local_btf, | |
4035 | local_member->type, | |
4036 | targ_btf, m->type); | |
4037 | if (!found) | |
4038 | spec->len--; /* pop accessor */ | |
4039 | return found; | |
4040 | } | |
4041 | /* member turned out not to be what we looked for */ | |
ee26dade | 4042 | spec->bit_offset -= bit_offset; |
ddc7c304 AN |
4043 | spec->raw_len--; |
4044 | } | |
4045 | ||
4046 | return 0; | |
4047 | } | |
4048 | ||
4049 | /* | |
4050 | * Try to match local spec to a target type and, if successful, produce full | |
ee26dade | 4051 | * target spec (high-level, low-level + bit offset). |
ddc7c304 AN |
4052 | */ |
4053 | static int bpf_core_spec_match(struct bpf_core_spec *local_spec, | |
4054 | const struct btf *targ_btf, __u32 targ_id, | |
4055 | struct bpf_core_spec *targ_spec) | |
4056 | { | |
4057 | const struct btf_type *targ_type; | |
4058 | const struct bpf_core_accessor *local_acc; | |
4059 | struct bpf_core_accessor *targ_acc; | |
4060 | int i, sz, matched; | |
4061 | ||
4062 | memset(targ_spec, 0, sizeof(*targ_spec)); | |
4063 | targ_spec->btf = targ_btf; | |
4064 | ||
4065 | local_acc = &local_spec->spec[0]; | |
4066 | targ_acc = &targ_spec->spec[0]; | |
4067 | ||
4068 | for (i = 0; i < local_spec->len; i++, local_acc++, targ_acc++) { | |
4069 | targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, | |
4070 | &targ_id); | |
4071 | if (!targ_type) | |
4072 | return -EINVAL; | |
4073 | ||
4074 | if (local_acc->name) { | |
4075 | matched = bpf_core_match_member(local_spec->btf, | |
4076 | local_acc, | |
4077 | targ_btf, targ_id, | |
4078 | targ_spec, &targ_id); | |
4079 | if (matched <= 0) | |
4080 | return matched; | |
4081 | } else { | |
4082 | /* for i=0, targ_id is already treated as array element | |
4083 | * type (because it's the original struct), for others | |
4084 | * we should find array element type first | |
4085 | */ | |
4086 | if (i > 0) { | |
4087 | const struct btf_array *a; | |
1b484b30 | 4088 | bool flex; |
ddc7c304 AN |
4089 | |
4090 | if (!btf_is_array(targ_type)) | |
4091 | return 0; | |
4092 | ||
4093 | a = btf_array(targ_type); | |
1b484b30 AN |
4094 | flex = is_flex_arr(targ_btf, targ_acc - 1, a); |
4095 | if (!flex && local_acc->idx >= a->nelems) | |
ddc7c304 AN |
4096 | return 0; |
4097 | if (!skip_mods_and_typedefs(targ_btf, a->type, | |
4098 | &targ_id)) | |
4099 | return -EINVAL; | |
4100 | } | |
4101 | ||
4102 | /* too deep struct/union/array nesting */ | |
4103 | if (targ_spec->raw_len == BPF_CORE_SPEC_MAX_LEN) | |
4104 | return -E2BIG; | |
4105 | ||
4106 | targ_acc->type_id = targ_id; | |
4107 | targ_acc->idx = local_acc->idx; | |
4108 | targ_acc->name = NULL; | |
4109 | targ_spec->len++; | |
4110 | targ_spec->raw_spec[targ_spec->raw_len] = targ_acc->idx; | |
4111 | targ_spec->raw_len++; | |
4112 | ||
4113 | sz = btf__resolve_size(targ_btf, targ_id); | |
4114 | if (sz < 0) | |
4115 | return sz; | |
ee26dade | 4116 | targ_spec->bit_offset += local_acc->idx * sz * 8; |
ddc7c304 AN |
4117 | } |
4118 | } | |
4119 | ||
4120 | return 1; | |
4121 | } | |
4122 | ||
ee26dade AN |
4123 | static int bpf_core_calc_field_relo(const struct bpf_program *prog, |
4124 | const struct bpf_field_reloc *relo, | |
4125 | const struct bpf_core_spec *spec, | |
4126 | __u32 *val, bool *validate) | |
4127 | { | |
4128 | const struct bpf_core_accessor *acc = &spec->spec[spec->len - 1]; | |
4129 | const struct btf_type *t = btf__type_by_id(spec->btf, acc->type_id); | |
4130 | __u32 byte_off, byte_sz, bit_off, bit_sz; | |
4131 | const struct btf_member *m; | |
4132 | const struct btf_type *mt; | |
4133 | bool bitfield; | |
94f060e9 | 4134 | __s64 sz; |
ee26dade AN |
4135 | |
4136 | /* a[n] accessor needs special handling */ | |
4137 | if (!acc->name) { | |
94f060e9 AN |
4138 | if (relo->kind == BPF_FIELD_BYTE_OFFSET) { |
4139 | *val = spec->bit_offset / 8; | |
4140 | } else if (relo->kind == BPF_FIELD_BYTE_SIZE) { | |
4141 | sz = btf__resolve_size(spec->btf, acc->type_id); | |
4142 | if (sz < 0) | |
4143 | return -EINVAL; | |
4144 | *val = sz; | |
4145 | } else { | |
4146 | pr_warn("prog '%s': relo %d at insn #%d can't be applied to array access\n", | |
ee26dade AN |
4147 | bpf_program__title(prog, false), |
4148 | relo->kind, relo->insn_off / 8); | |
4149 | return -EINVAL; | |
4150 | } | |
ee26dade AN |
4151 | if (validate) |
4152 | *validate = true; | |
4153 | return 0; | |
4154 | } | |
4155 | ||
4156 | m = btf_members(t) + acc->idx; | |
4157 | mt = skip_mods_and_typedefs(spec->btf, m->type, NULL); | |
4158 | bit_off = spec->bit_offset; | |
4159 | bit_sz = btf_member_bitfield_size(t, acc->idx); | |
4160 | ||
4161 | bitfield = bit_sz > 0; | |
4162 | if (bitfield) { | |
4163 | byte_sz = mt->size; | |
4164 | byte_off = bit_off / 8 / byte_sz * byte_sz; | |
4165 | /* figure out smallest int size necessary for bitfield load */ | |
4166 | while (bit_off + bit_sz - byte_off * 8 > byte_sz * 8) { | |
4167 | if (byte_sz >= 8) { | |
4168 | /* bitfield can't be read with 64-bit read */ | |
4169 | pr_warn("prog '%s': relo %d at insn #%d can't be satisfied for bitfield\n", | |
4170 | bpf_program__title(prog, false), | |
4171 | relo->kind, relo->insn_off / 8); | |
4172 | return -E2BIG; | |
4173 | } | |
4174 | byte_sz *= 2; | |
4175 | byte_off = bit_off / 8 / byte_sz * byte_sz; | |
4176 | } | |
4177 | } else { | |
94f060e9 AN |
4178 | sz = btf__resolve_size(spec->btf, m->type); |
4179 | if (sz < 0) | |
4180 | return -EINVAL; | |
4181 | byte_sz = sz; | |
ee26dade AN |
4182 | byte_off = spec->bit_offset / 8; |
4183 | bit_sz = byte_sz * 8; | |
4184 | } | |
4185 | ||
4186 | /* for bitfields, all the relocatable aspects are ambiguous and we | |
4187 | * might disagree with compiler, so turn off validation of expected | |
4188 | * value, except for signedness | |
4189 | */ | |
4190 | if (validate) | |
4191 | *validate = !bitfield; | |
4192 | ||
4193 | switch (relo->kind) { | |
4194 | case BPF_FIELD_BYTE_OFFSET: | |
4195 | *val = byte_off; | |
4196 | break; | |
4197 | case BPF_FIELD_BYTE_SIZE: | |
4198 | *val = byte_sz; | |
4199 | break; | |
4200 | case BPF_FIELD_SIGNED: | |
4201 | /* enums will be assumed unsigned */ | |
4202 | *val = btf_is_enum(mt) || | |
4203 | (btf_int_encoding(mt) & BTF_INT_SIGNED); | |
4204 | if (validate) | |
4205 | *validate = true; /* signedness is never ambiguous */ | |
4206 | break; | |
4207 | case BPF_FIELD_LSHIFT_U64: | |
4208 | #if __BYTE_ORDER == __LITTLE_ENDIAN | |
4209 | *val = 64 - (bit_off + bit_sz - byte_off * 8); | |
4210 | #else | |
4211 | *val = (8 - byte_sz) * 8 + (bit_off - byte_off * 8); | |
4212 | #endif | |
4213 | break; | |
4214 | case BPF_FIELD_RSHIFT_U64: | |
4215 | *val = 64 - bit_sz; | |
4216 | if (validate) | |
4217 | *validate = true; /* right shift is never ambiguous */ | |
4218 | break; | |
4219 | case BPF_FIELD_EXISTS: | |
4220 | default: | |
4221 | pr_warn("prog '%s': unknown relo %d at insn #%d\n", | |
4222 | bpf_program__title(prog, false), | |
4223 | relo->kind, relo->insn_off / 8); | |
4224 | return -EINVAL; | |
4225 | } | |
4226 | ||
4227 | return 0; | |
4228 | } | |
4229 | ||
ddc7c304 AN |
4230 | /* |
4231 | * Patch relocatable BPF instruction. | |
62561eb4 AN |
4232 | * |
4233 | * Patched value is determined by relocation kind and target specification. | |
4234 | * For field existence relocation target spec will be NULL if field is not | |
4235 | * found. | |
4236 | * Expected insn->imm value is determined using relocation kind and local | |
4237 | * spec, and is checked before patching instruction. If actual insn->imm value | |
4238 | * is wrong, bail out with error. | |
ddc7c304 AN |
4239 | * |
4240 | * Currently three kinds of BPF instructions are supported: | |
4241 | * 1. rX = <imm> (assignment with immediate operand); | |
4242 | * 2. rX += <imm> (arithmetic operations with immediate operand); | |
ddc7c304 | 4243 | */ |
62561eb4 AN |
4244 | static int bpf_core_reloc_insn(struct bpf_program *prog, |
4245 | const struct bpf_field_reloc *relo, | |
d7a25270 | 4246 | int relo_idx, |
62561eb4 AN |
4247 | const struct bpf_core_spec *local_spec, |
4248 | const struct bpf_core_spec *targ_spec) | |
ddc7c304 | 4249 | { |
62561eb4 | 4250 | __u32 orig_val, new_val; |
ddc7c304 | 4251 | struct bpf_insn *insn; |
d7a25270 | 4252 | bool validate = true; |
ee26dade | 4253 | int insn_idx, err; |
ddc7c304 AN |
4254 | __u8 class; |
4255 | ||
62561eb4 AN |
4256 | if (relo->insn_off % sizeof(struct bpf_insn)) |
4257 | return -EINVAL; | |
4258 | insn_idx = relo->insn_off / sizeof(struct bpf_insn); | |
d7a25270 AN |
4259 | insn = &prog->insns[insn_idx]; |
4260 | class = BPF_CLASS(insn->code); | |
62561eb4 | 4261 | |
ee26dade | 4262 | if (relo->kind == BPF_FIELD_EXISTS) { |
62561eb4 AN |
4263 | orig_val = 1; /* can't generate EXISTS relo w/o local field */ |
4264 | new_val = targ_spec ? 1 : 0; | |
ee26dade | 4265 | } else if (!targ_spec) { |
d7a25270 AN |
4266 | pr_debug("prog '%s': relo #%d: substituting insn #%d w/ invalid insn\n", |
4267 | bpf_program__title(prog, false), relo_idx, insn_idx); | |
4268 | insn->code = BPF_JMP | BPF_CALL; | |
4269 | insn->dst_reg = 0; | |
4270 | insn->src_reg = 0; | |
4271 | insn->off = 0; | |
4272 | /* if this instruction is reachable (not a dead code), | |
4273 | * verifier will complain with the following message: | |
4274 | * invalid func unknown#195896080 | |
4275 | */ | |
4276 | insn->imm = 195896080; /* => 0xbad2310 => "bad relo" */ | |
4277 | return 0; | |
ee26dade AN |
4278 | } else { |
4279 | err = bpf_core_calc_field_relo(prog, relo, local_spec, | |
4280 | &orig_val, &validate); | |
4281 | if (err) | |
4282 | return err; | |
4283 | err = bpf_core_calc_field_relo(prog, relo, targ_spec, | |
4284 | &new_val, NULL); | |
4285 | if (err) | |
4286 | return err; | |
62561eb4 | 4287 | } |
ddc7c304 | 4288 | |
8ab9da57 AN |
4289 | switch (class) { |
4290 | case BPF_ALU: | |
4291 | case BPF_ALU64: | |
ddc7c304 AN |
4292 | if (BPF_SRC(insn->code) != BPF_K) |
4293 | return -EINVAL; | |
d7a25270 AN |
4294 | if (validate && insn->imm != orig_val) { |
4295 | pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n", | |
4296 | bpf_program__title(prog, false), relo_idx, | |
4297 | insn_idx, insn->imm, orig_val, new_val); | |
ddc7c304 | 4298 | return -EINVAL; |
ee26dade AN |
4299 | } |
4300 | orig_val = insn->imm; | |
62561eb4 | 4301 | insn->imm = new_val; |
d7a25270 AN |
4302 | pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n", |
4303 | bpf_program__title(prog, false), relo_idx, insn_idx, | |
4304 | orig_val, new_val); | |
8ab9da57 AN |
4305 | break; |
4306 | case BPF_LDX: | |
4307 | case BPF_ST: | |
4308 | case BPF_STX: | |
d7a25270 AN |
4309 | if (validate && insn->off != orig_val) { |
4310 | pr_warn("prog '%s': relo #%d: unexpected insn #%d (LD/LDX/ST/STX) value: got %u, exp %u -> %u\n", | |
4311 | bpf_program__title(prog, false), relo_idx, | |
4312 | insn_idx, insn->off, orig_val, new_val); | |
8ab9da57 AN |
4313 | return -EINVAL; |
4314 | } | |
4315 | if (new_val > SHRT_MAX) { | |
d7a25270 AN |
4316 | pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n", |
4317 | bpf_program__title(prog, false), relo_idx, | |
4318 | insn_idx, new_val); | |
8ab9da57 AN |
4319 | return -ERANGE; |
4320 | } | |
4321 | orig_val = insn->off; | |
4322 | insn->off = new_val; | |
d7a25270 AN |
4323 | pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n", |
4324 | bpf_program__title(prog, false), relo_idx, insn_idx, | |
4325 | orig_val, new_val); | |
8ab9da57 AN |
4326 | break; |
4327 | default: | |
d7a25270 AN |
4328 | pr_warn("prog '%s': relo #%d: trying to relocate unrecognized insn #%d, code:%x, src:%x, dst:%x, off:%x, imm:%x\n", |
4329 | bpf_program__title(prog, false), relo_idx, | |
be18010e KW |
4330 | insn_idx, insn->code, insn->src_reg, insn->dst_reg, |
4331 | insn->off, insn->imm); | |
ddc7c304 AN |
4332 | return -EINVAL; |
4333 | } | |
62561eb4 | 4334 | |
ddc7c304 AN |
4335 | return 0; |
4336 | } | |
4337 | ||
ddc7c304 AN |
4338 | /* Output spec definition in the format: |
4339 | * [<type-id>] (<type-name>) + <raw-spec> => <offset>@<spec>, | |
4340 | * where <spec> is a C-syntax view of recorded field access, e.g.: x.a[3].b | |
4341 | */ | |
4342 | static void bpf_core_dump_spec(int level, const struct bpf_core_spec *spec) | |
4343 | { | |
4344 | const struct btf_type *t; | |
4345 | const char *s; | |
4346 | __u32 type_id; | |
4347 | int i; | |
4348 | ||
4349 | type_id = spec->spec[0].type_id; | |
4350 | t = btf__type_by_id(spec->btf, type_id); | |
4351 | s = btf__name_by_offset(spec->btf, t->name_off); | |
4352 | libbpf_print(level, "[%u] %s + ", type_id, s); | |
4353 | ||
4354 | for (i = 0; i < spec->raw_len; i++) | |
4355 | libbpf_print(level, "%d%s", spec->raw_spec[i], | |
4356 | i == spec->raw_len - 1 ? " => " : ":"); | |
4357 | ||
ee26dade AN |
4358 | libbpf_print(level, "%u.%u @ &x", |
4359 | spec->bit_offset / 8, spec->bit_offset % 8); | |
ddc7c304 AN |
4360 | |
4361 | for (i = 0; i < spec->len; i++) { | |
4362 | if (spec->spec[i].name) | |
4363 | libbpf_print(level, ".%s", spec->spec[i].name); | |
4364 | else | |
4365 | libbpf_print(level, "[%u]", spec->spec[i].idx); | |
4366 | } | |
4367 | ||
4368 | } | |
4369 | ||
4370 | static size_t bpf_core_hash_fn(const void *key, void *ctx) | |
4371 | { | |
4372 | return (size_t)key; | |
4373 | } | |
4374 | ||
4375 | static bool bpf_core_equal_fn(const void *k1, const void *k2, void *ctx) | |
4376 | { | |
4377 | return k1 == k2; | |
4378 | } | |
4379 | ||
4380 | static void *u32_as_hash_key(__u32 x) | |
4381 | { | |
4382 | return (void *)(uintptr_t)x; | |
4383 | } | |
4384 | ||
4385 | /* | |
4386 | * CO-RE relocate single instruction. | |
4387 | * | |
4388 | * The outline and important points of the algorithm: | |
4389 | * 1. For given local type, find corresponding candidate target types. | |
4390 | * Candidate type is a type with the same "essential" name, ignoring | |
4391 | * everything after last triple underscore (___). E.g., `sample`, | |
4392 | * `sample___flavor_one`, `sample___flavor_another_one`, are all candidates | |
4393 | * for each other. Names with triple underscore are referred to as | |
4394 | * "flavors" and are useful, among other things, to allow to | |
4395 | * specify/support incompatible variations of the same kernel struct, which | |
4396 | * might differ between different kernel versions and/or build | |
4397 | * configurations. | |
4398 | * | |
4399 | * N.B. Struct "flavors" could be generated by bpftool's BTF-to-C | |
4400 | * converter, when deduplicated BTF of a kernel still contains more than | |
4401 | * one different types with the same name. In that case, ___2, ___3, etc | |
4402 | * are appended starting from second name conflict. But start flavors are | |
4403 | * also useful to be defined "locally", in BPF program, to extract same | |
4404 | * data from incompatible changes between different kernel | |
4405 | * versions/configurations. For instance, to handle field renames between | |
4406 | * kernel versions, one can use two flavors of the struct name with the | |
4407 | * same common name and use conditional relocations to extract that field, | |
4408 | * depending on target kernel version. | |
4409 | * 2. For each candidate type, try to match local specification to this | |
4410 | * candidate target type. Matching involves finding corresponding | |
4411 | * high-level spec accessors, meaning that all named fields should match, | |
4412 | * as well as all array accesses should be within the actual bounds. Also, | |
4413 | * types should be compatible (see bpf_core_fields_are_compat for details). | |
4414 | * 3. It is supported and expected that there might be multiple flavors | |
4415 | * matching the spec. As long as all the specs resolve to the same set of | |
511bb008 | 4416 | * offsets across all candidates, there is no error. If there is any |
ddc7c304 AN |
4417 | * ambiguity, CO-RE relocation will fail. This is necessary to accomodate |
4418 | * imprefection of BTF deduplication, which can cause slight duplication of | |
4419 | * the same BTF type, if some directly or indirectly referenced (by | |
4420 | * pointer) type gets resolved to different actual types in different | |
4421 | * object files. If such situation occurs, deduplicated BTF will end up | |
4422 | * with two (or more) structurally identical types, which differ only in | |
4423 | * types they refer to through pointer. This should be OK in most cases and | |
4424 | * is not an error. | |
4425 | * 4. Candidate types search is performed by linearly scanning through all | |
4426 | * types in target BTF. It is anticipated that this is overall more | |
4427 | * efficient memory-wise and not significantly worse (if not better) | |
4428 | * CPU-wise compared to prebuilding a map from all local type names to | |
4429 | * a list of candidate type names. It's also sped up by caching resolved | |
4430 | * list of matching candidates per each local "root" type ID, that has at | |
511bb008 | 4431 | * least one bpf_field_reloc associated with it. This list is shared |
ddc7c304 AN |
4432 | * between multiple relocations for the same type ID and is updated as some |
4433 | * of the candidates are pruned due to structural incompatibility. | |
4434 | */ | |
511bb008 AN |
4435 | static int bpf_core_reloc_field(struct bpf_program *prog, |
4436 | const struct bpf_field_reloc *relo, | |
ddc7c304 AN |
4437 | int relo_idx, |
4438 | const struct btf *local_btf, | |
4439 | const struct btf *targ_btf, | |
4440 | struct hashmap *cand_cache) | |
4441 | { | |
4442 | const char *prog_name = bpf_program__title(prog, false); | |
4443 | struct bpf_core_spec local_spec, cand_spec, targ_spec; | |
4444 | const void *type_key = u32_as_hash_key(relo->type_id); | |
4445 | const struct btf_type *local_type, *cand_type; | |
4446 | const char *local_name, *cand_name; | |
4447 | struct ids_vec *cand_ids; | |
4448 | __u32 local_id, cand_id; | |
4449 | const char *spec_str; | |
4450 | int i, j, err; | |
4451 | ||
4452 | local_id = relo->type_id; | |
4453 | local_type = btf__type_by_id(local_btf, local_id); | |
4454 | if (!local_type) | |
4455 | return -EINVAL; | |
4456 | ||
4457 | local_name = btf__name_by_offset(local_btf, local_type->name_off); | |
4458 | if (str_is_empty(local_name)) | |
4459 | return -EINVAL; | |
4460 | ||
4461 | spec_str = btf__name_by_offset(local_btf, relo->access_str_off); | |
4462 | if (str_is_empty(spec_str)) | |
4463 | return -EINVAL; | |
4464 | ||
4465 | err = bpf_core_spec_parse(local_btf, local_id, spec_str, &local_spec); | |
4466 | if (err) { | |
be18010e KW |
4467 | pr_warn("prog '%s': relo #%d: parsing [%d] %s + %s failed: %d\n", |
4468 | prog_name, relo_idx, local_id, local_name, spec_str, | |
4469 | err); | |
ddc7c304 AN |
4470 | return -EINVAL; |
4471 | } | |
4472 | ||
ee26dade AN |
4473 | pr_debug("prog '%s': relo #%d: kind %d, spec is ", prog_name, relo_idx, |
4474 | relo->kind); | |
ddc7c304 AN |
4475 | bpf_core_dump_spec(LIBBPF_DEBUG, &local_spec); |
4476 | libbpf_print(LIBBPF_DEBUG, "\n"); | |
4477 | ||
4478 | if (!hashmap__find(cand_cache, type_key, (void **)&cand_ids)) { | |
4479 | cand_ids = bpf_core_find_cands(local_btf, local_id, targ_btf); | |
4480 | if (IS_ERR(cand_ids)) { | |
be18010e KW |
4481 | pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s: %ld", |
4482 | prog_name, relo_idx, local_id, local_name, | |
4483 | PTR_ERR(cand_ids)); | |
ddc7c304 AN |
4484 | return PTR_ERR(cand_ids); |
4485 | } | |
4486 | err = hashmap__set(cand_cache, type_key, cand_ids, NULL, NULL); | |
4487 | if (err) { | |
4488 | bpf_core_free_cands(cand_ids); | |
4489 | return err; | |
4490 | } | |
4491 | } | |
4492 | ||
4493 | for (i = 0, j = 0; i < cand_ids->len; i++) { | |
4494 | cand_id = cand_ids->data[i]; | |
4495 | cand_type = btf__type_by_id(targ_btf, cand_id); | |
4496 | cand_name = btf__name_by_offset(targ_btf, cand_type->name_off); | |
4497 | ||
4498 | err = bpf_core_spec_match(&local_spec, targ_btf, | |
4499 | cand_id, &cand_spec); | |
4500 | pr_debug("prog '%s': relo #%d: matching candidate #%d %s against spec ", | |
4501 | prog_name, relo_idx, i, cand_name); | |
4502 | bpf_core_dump_spec(LIBBPF_DEBUG, &cand_spec); | |
4503 | libbpf_print(LIBBPF_DEBUG, ": %d\n", err); | |
4504 | if (err < 0) { | |
be18010e KW |
4505 | pr_warn("prog '%s': relo #%d: matching error: %d\n", |
4506 | prog_name, relo_idx, err); | |
ddc7c304 AN |
4507 | return err; |
4508 | } | |
4509 | if (err == 0) | |
4510 | continue; | |
4511 | ||
4512 | if (j == 0) { | |
4513 | targ_spec = cand_spec; | |
ee26dade | 4514 | } else if (cand_spec.bit_offset != targ_spec.bit_offset) { |
ddc7c304 | 4515 | /* if there are many candidates, they should all |
ee26dade | 4516 | * resolve to the same bit offset |
ddc7c304 | 4517 | */ |
be18010e | 4518 | pr_warn("prog '%s': relo #%d: offset ambiguity: %u != %u\n", |
ee26dade AN |
4519 | prog_name, relo_idx, cand_spec.bit_offset, |
4520 | targ_spec.bit_offset); | |
ddc7c304 AN |
4521 | return -EINVAL; |
4522 | } | |
4523 | ||
4524 | cand_ids->data[j++] = cand_spec.spec[0].type_id; | |
4525 | } | |
4526 | ||
62561eb4 | 4527 | /* |
d7a25270 AN |
4528 | * For BPF_FIELD_EXISTS relo or when used BPF program has field |
4529 | * existence checks or kernel version/config checks, it's expected | |
4530 | * that we might not find any candidates. In this case, if field | |
4531 | * wasn't found in any candidate, the list of candidates shouldn't | |
4532 | * change at all, we'll just handle relocating appropriately, | |
4533 | * depending on relo's kind. | |
62561eb4 AN |
4534 | */ |
4535 | if (j > 0) | |
4536 | cand_ids->len = j; | |
4537 | ||
d7a25270 AN |
4538 | /* |
4539 | * If no candidates were found, it might be both a programmer error, | |
4540 | * as well as expected case, depending whether instruction w/ | |
4541 | * relocation is guarded in some way that makes it unreachable (dead | |
4542 | * code) if relocation can't be resolved. This is handled in | |
4543 | * bpf_core_reloc_insn() uniformly by replacing that instruction with | |
4544 | * BPF helper call insn (using invalid helper ID). If that instruction | |
4545 | * is indeed unreachable, then it will be ignored and eliminated by | |
4546 | * verifier. If it was an error, then verifier will complain and point | |
4547 | * to a specific instruction number in its log. | |
4548 | */ | |
4549 | if (j == 0) | |
4550 | pr_debug("prog '%s': relo #%d: no matching targets found for [%d] %s + %s\n", | |
4551 | prog_name, relo_idx, local_id, local_name, spec_str); | |
ddc7c304 | 4552 | |
62561eb4 | 4553 | /* bpf_core_reloc_insn should know how to handle missing targ_spec */ |
d7a25270 | 4554 | err = bpf_core_reloc_insn(prog, relo, relo_idx, &local_spec, |
62561eb4 | 4555 | j ? &targ_spec : NULL); |
ddc7c304 | 4556 | if (err) { |
be18010e KW |
4557 | pr_warn("prog '%s': relo #%d: failed to patch insn at offset %d: %d\n", |
4558 | prog_name, relo_idx, relo->insn_off, err); | |
ddc7c304 AN |
4559 | return -EINVAL; |
4560 | } | |
4561 | ||
4562 | return 0; | |
4563 | } | |
4564 | ||
4565 | static int | |
511bb008 | 4566 | bpf_core_reloc_fields(struct bpf_object *obj, const char *targ_btf_path) |
ddc7c304 AN |
4567 | { |
4568 | const struct btf_ext_info_sec *sec; | |
511bb008 | 4569 | const struct bpf_field_reloc *rec; |
ddc7c304 AN |
4570 | const struct btf_ext_info *seg; |
4571 | struct hashmap_entry *entry; | |
4572 | struct hashmap *cand_cache = NULL; | |
4573 | struct bpf_program *prog; | |
4574 | struct btf *targ_btf; | |
4575 | const char *sec_name; | |
4576 | int i, err = 0; | |
4577 | ||
4578 | if (targ_btf_path) | |
4579 | targ_btf = btf__parse_elf(targ_btf_path, NULL); | |
4580 | else | |
fb2426ad | 4581 | targ_btf = libbpf_find_kernel_btf(); |
ddc7c304 | 4582 | if (IS_ERR(targ_btf)) { |
be18010e | 4583 | pr_warn("failed to get target BTF: %ld\n", PTR_ERR(targ_btf)); |
ddc7c304 AN |
4584 | return PTR_ERR(targ_btf); |
4585 | } | |
4586 | ||
4587 | cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); | |
4588 | if (IS_ERR(cand_cache)) { | |
4589 | err = PTR_ERR(cand_cache); | |
4590 | goto out; | |
4591 | } | |
4592 | ||
511bb008 | 4593 | seg = &obj->btf_ext->field_reloc_info; |
ddc7c304 AN |
4594 | for_each_btf_ext_sec(seg, sec) { |
4595 | sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); | |
4596 | if (str_is_empty(sec_name)) { | |
4597 | err = -EINVAL; | |
4598 | goto out; | |
4599 | } | |
4600 | prog = bpf_object__find_program_by_title(obj, sec_name); | |
4601 | if (!prog) { | |
be18010e KW |
4602 | pr_warn("failed to find program '%s' for CO-RE offset relocation\n", |
4603 | sec_name); | |
ddc7c304 AN |
4604 | err = -EINVAL; |
4605 | goto out; | |
4606 | } | |
4607 | ||
4608 | pr_debug("prog '%s': performing %d CO-RE offset relocs\n", | |
4609 | sec_name, sec->num_info); | |
4610 | ||
4611 | for_each_btf_ext_rec(seg, sec, i, rec) { | |
511bb008 AN |
4612 | err = bpf_core_reloc_field(prog, rec, i, obj->btf, |
4613 | targ_btf, cand_cache); | |
ddc7c304 | 4614 | if (err) { |
be18010e KW |
4615 | pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", |
4616 | sec_name, i, err); | |
ddc7c304 AN |
4617 | goto out; |
4618 | } | |
4619 | } | |
4620 | } | |
4621 | ||
4622 | out: | |
4623 | btf__free(targ_btf); | |
4624 | if (!IS_ERR_OR_NULL(cand_cache)) { | |
4625 | hashmap__for_each_entry(cand_cache, entry, i) { | |
4626 | bpf_core_free_cands(entry->value); | |
4627 | } | |
4628 | hashmap__free(cand_cache); | |
4629 | } | |
4630 | return err; | |
4631 | } | |
4632 | ||
4633 | static int | |
4634 | bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) | |
4635 | { | |
4636 | int err = 0; | |
4637 | ||
511bb008 AN |
4638 | if (obj->btf_ext->field_reloc_info.len) |
4639 | err = bpf_core_reloc_fields(obj, targ_btf_path); | |
ddc7c304 AN |
4640 | |
4641 | return err; | |
4642 | } | |
4643 | ||
48cca7e4 AS |
4644 | static int |
4645 | bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, | |
4646 | struct reloc_desc *relo) | |
4647 | { | |
4648 | struct bpf_insn *insn, *new_insn; | |
4649 | struct bpf_program *text; | |
4650 | size_t new_cnt; | |
2993e051 | 4651 | int err; |
48cca7e4 | 4652 | |
9173cac3 | 4653 | if (prog->idx != obj->efile.text_shndx && prog->main_prog_cnt == 0) { |
48cca7e4 AS |
4654 | text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); |
4655 | if (!text) { | |
be18010e | 4656 | pr_warn("no .text section found yet relo into text exist\n"); |
48cca7e4 AS |
4657 | return -LIBBPF_ERRNO__RELOC; |
4658 | } | |
4659 | new_cnt = prog->insns_cnt + text->insns_cnt; | |
531b014e | 4660 | new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); |
48cca7e4 | 4661 | if (!new_insn) { |
be18010e | 4662 | pr_warn("oom in prog realloc\n"); |
48cca7e4 AS |
4663 | return -ENOMEM; |
4664 | } | |
3dc5e059 | 4665 | prog->insns = new_insn; |
2993e051 | 4666 | |
f0187f0b MKL |
4667 | if (obj->btf_ext) { |
4668 | err = bpf_program_reloc_btf_ext(prog, obj, | |
4669 | text->section_name, | |
4670 | prog->insns_cnt); | |
4671 | if (err) | |
2993e051 | 4672 | return err; |
2993e051 YS |
4673 | } |
4674 | ||
48cca7e4 AS |
4675 | memcpy(new_insn + prog->insns_cnt, text->insns, |
4676 | text->insns_cnt * sizeof(*insn)); | |
48cca7e4 AS |
4677 | prog->main_prog_cnt = prog->insns_cnt; |
4678 | prog->insns_cnt = new_cnt; | |
b1a2ce82 JC |
4679 | pr_debug("added %zd insn from %s to prog %s\n", |
4680 | text->insns_cnt, text->section_name, | |
4681 | prog->section_name); | |
48cca7e4 | 4682 | } |
9173cac3 | 4683 | |
48cca7e4 | 4684 | insn = &prog->insns[relo->insn_idx]; |
53f8dd43 | 4685 | insn->imm += relo->sym_off / 8 + prog->main_prog_cnt - relo->insn_idx; |
48cca7e4 AS |
4686 | return 0; |
4687 | } | |
4688 | ||
8a47a6c5 | 4689 | static int |
9d759a9b | 4690 | bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) |
8a47a6c5 | 4691 | { |
48cca7e4 | 4692 | int i, err; |
8a47a6c5 | 4693 | |
2993e051 YS |
4694 | if (!prog) |
4695 | return 0; | |
4696 | ||
f0187f0b MKL |
4697 | if (obj->btf_ext) { |
4698 | err = bpf_program_reloc_btf_ext(prog, obj, | |
4699 | prog->section_name, 0); | |
4700 | if (err) | |
2993e051 | 4701 | return err; |
2993e051 YS |
4702 | } |
4703 | ||
4704 | if (!prog->reloc_desc) | |
8a47a6c5 WN |
4705 | return 0; |
4706 | ||
4707 | for (i = 0; i < prog->nr_reloc; i++) { | |
53f8dd43 | 4708 | struct reloc_desc *relo = &prog->reloc_desc[i]; |
166750bc | 4709 | struct bpf_insn *insn = &prog->insns[relo->insn_idx]; |
8a47a6c5 | 4710 | |
166750bc AN |
4711 | if (relo->insn_idx + 1 >= (int)prog->insns_cnt) { |
4712 | pr_warn("relocation out of range: '%s'\n", | |
4713 | prog->section_name); | |
4714 | return -LIBBPF_ERRNO__RELOC; | |
4715 | } | |
d859900c | 4716 | |
166750bc AN |
4717 | switch (relo->type) { |
4718 | case RELO_LD64: | |
4719 | insn[0].src_reg = BPF_PSEUDO_MAP_FD; | |
4720 | insn[0].imm = obj->maps[relo->map_idx].fd; | |
4721 | break; | |
4722 | case RELO_DATA: | |
4723 | insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; | |
4724 | insn[1].imm = insn[0].imm + relo->sym_off; | |
53f8dd43 | 4725 | insn[0].imm = obj->maps[relo->map_idx].fd; |
166750bc AN |
4726 | break; |
4727 | case RELO_EXTERN: | |
4728 | insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; | |
81bfdd08 | 4729 | insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; |
166750bc AN |
4730 | insn[1].imm = relo->sym_off; |
4731 | break; | |
4732 | case RELO_CALL: | |
53f8dd43 | 4733 | err = bpf_program__reloc_text(prog, obj, relo); |
48cca7e4 AS |
4734 | if (err) |
4735 | return err; | |
166750bc AN |
4736 | break; |
4737 | default: | |
4738 | pr_warn("relo #%d: bad relo type %d\n", i, relo->type); | |
4739 | return -EINVAL; | |
8a47a6c5 | 4740 | } |
8a47a6c5 WN |
4741 | } |
4742 | ||
4743 | zfree(&prog->reloc_desc); | |
4744 | prog->nr_reloc = 0; | |
4745 | return 0; | |
4746 | } | |
4747 | ||
8a47a6c5 | 4748 | static int |
ddc7c304 | 4749 | bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) |
8a47a6c5 WN |
4750 | { |
4751 | struct bpf_program *prog; | |
4752 | size_t i; | |
4753 | int err; | |
4754 | ||
ddc7c304 AN |
4755 | if (obj->btf_ext) { |
4756 | err = bpf_object__relocate_core(obj, targ_btf_path); | |
4757 | if (err) { | |
be18010e KW |
4758 | pr_warn("failed to perform CO-RE relocations: %d\n", |
4759 | err); | |
ddc7c304 AN |
4760 | return err; |
4761 | } | |
4762 | } | |
9173cac3 AN |
4763 | /* ensure .text is relocated first, as it's going to be copied as-is |
4764 | * later for sub-program calls | |
4765 | */ | |
4766 | for (i = 0; i < obj->nr_programs; i++) { | |
4767 | prog = &obj->programs[i]; | |
4768 | if (prog->idx != obj->efile.text_shndx) | |
4769 | continue; | |
4770 | ||
4771 | err = bpf_program__relocate(prog, obj); | |
4772 | if (err) { | |
4773 | pr_warn("failed to relocate '%s'\n", prog->section_name); | |
4774 | return err; | |
4775 | } | |
4776 | break; | |
4777 | } | |
4778 | /* now relocate everything but .text, which by now is relocated | |
4779 | * properly, so we can copy raw sub-program instructions as is safely | |
4780 | */ | |
8a47a6c5 WN |
4781 | for (i = 0; i < obj->nr_programs; i++) { |
4782 | prog = &obj->programs[i]; | |
9173cac3 AN |
4783 | if (prog->idx == obj->efile.text_shndx) |
4784 | continue; | |
8a47a6c5 | 4785 | |
9d759a9b | 4786 | err = bpf_program__relocate(prog, obj); |
8a47a6c5 | 4787 | if (err) { |
be18010e | 4788 | pr_warn("failed to relocate '%s'\n", prog->section_name); |
8a47a6c5 WN |
4789 | return err; |
4790 | } | |
4791 | } | |
4792 | return 0; | |
4793 | } | |
4794 | ||
590a0088 MKL |
4795 | static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj, |
4796 | GElf_Shdr *shdr, | |
4797 | Elf_Data *data); | |
4798 | ||
34090915 WN |
4799 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
4800 | { | |
4801 | int i, err; | |
4802 | ||
4803 | if (!obj_elf_valid(obj)) { | |
be18010e | 4804 | pr_warn("Internal error: elf object is closed\n"); |
6371ca3b | 4805 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
4806 | } |
4807 | ||
1f8e2bcb AN |
4808 | for (i = 0; i < obj->efile.nr_reloc_sects; i++) { |
4809 | GElf_Shdr *shdr = &obj->efile.reloc_sects[i].shdr; | |
4810 | Elf_Data *data = obj->efile.reloc_sects[i].data; | |
34090915 WN |
4811 | int idx = shdr->sh_info; |
4812 | struct bpf_program *prog; | |
34090915 WN |
4813 | |
4814 | if (shdr->sh_type != SHT_REL) { | |
be18010e | 4815 | pr_warn("internal error at %d\n", __LINE__); |
6371ca3b | 4816 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
4817 | } |
4818 | ||
590a0088 MKL |
4819 | if (idx == obj->efile.st_ops_shndx) { |
4820 | err = bpf_object__collect_struct_ops_map_reloc(obj, | |
4821 | shdr, | |
4822 | data); | |
4823 | if (err) | |
4824 | return err; | |
4825 | continue; | |
4826 | } | |
4827 | ||
34090915 WN |
4828 | prog = bpf_object__find_prog_by_idx(obj, idx); |
4829 | if (!prog) { | |
be18010e | 4830 | pr_warn("relocation failed: no section(%d)\n", idx); |
6371ca3b | 4831 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
4832 | } |
4833 | ||
399dc65e | 4834 | err = bpf_program__collect_reloc(prog, shdr, data, obj); |
34090915 | 4835 | if (err) |
6371ca3b | 4836 | return err; |
34090915 WN |
4837 | } |
4838 | return 0; | |
4839 | } | |
4840 | ||
55cffde2 | 4841 | static int |
2993e051 | 4842 | load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, |
f0187f0b | 4843 | char *license, __u32 kern_version, int *pfd) |
55cffde2 | 4844 | { |
d7be143b | 4845 | struct bpf_load_program_attr load_attr; |
1ce6a9fc | 4846 | char *cp, errmsg[STRERR_BUFSIZE]; |
da11b417 | 4847 | int log_buf_size = BPF_LOG_BUF_SIZE; |
55cffde2 | 4848 | char *log_buf; |
5d01ab7b | 4849 | int btf_fd, ret; |
55cffde2 | 4850 | |
fba01a06 AN |
4851 | if (!insns || !insns_cnt) |
4852 | return -EINVAL; | |
4853 | ||
d7be143b | 4854 | memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); |
2993e051 YS |
4855 | load_attr.prog_type = prog->type; |
4856 | load_attr.expected_attach_type = prog->expected_attach_type; | |
5b32a23e SF |
4857 | if (prog->caps->name) |
4858 | load_attr.name = prog->name; | |
d7be143b AI |
4859 | load_attr.insns = insns; |
4860 | load_attr.insns_cnt = insns_cnt; | |
4861 | load_attr.license = license; | |
590a0088 MKL |
4862 | if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) { |
4863 | load_attr.attach_btf_id = prog->attach_btf_id; | |
2db6eab1 AS |
4864 | } else if (prog->type == BPF_PROG_TYPE_TRACING || |
4865 | prog->type == BPF_PROG_TYPE_EXT) { | |
e7bf94db AS |
4866 | load_attr.attach_prog_fd = prog->attach_prog_fd; |
4867 | load_attr.attach_btf_id = prog->attach_btf_id; | |
4868 | } else { | |
4869 | load_attr.kern_version = kern_version; | |
4870 | load_attr.prog_ifindex = prog->prog_ifindex; | |
4871 | } | |
3415ec64 AN |
4872 | /* if .BTF.ext was loaded, kernel supports associated BTF for prog */ |
4873 | if (prog->obj->btf_ext) | |
4874 | btf_fd = bpf_object__btf_fd(prog->obj); | |
4875 | else | |
4876 | btf_fd = -1; | |
5d01ab7b | 4877 | load_attr.prog_btf_fd = btf_fd >= 0 ? btf_fd : 0; |
2993e051 YS |
4878 | load_attr.func_info = prog->func_info; |
4879 | load_attr.func_info_rec_size = prog->func_info_rec_size; | |
f0187f0b | 4880 | load_attr.func_info_cnt = prog->func_info_cnt; |
3d650141 MKL |
4881 | load_attr.line_info = prog->line_info; |
4882 | load_attr.line_info_rec_size = prog->line_info_rec_size; | |
4883 | load_attr.line_info_cnt = prog->line_info_cnt; | |
da11b417 | 4884 | load_attr.log_level = prog->log_level; |
04656198 | 4885 | load_attr.prog_flags = prog->prog_flags; |
55cffde2 | 4886 | |
da11b417 AS |
4887 | retry_load: |
4888 | log_buf = malloc(log_buf_size); | |
55cffde2 | 4889 | if (!log_buf) |
be18010e | 4890 | pr_warn("Alloc log buffer for bpf loader error, continue without log\n"); |
55cffde2 | 4891 | |
da11b417 | 4892 | ret = bpf_load_program_xattr(&load_attr, log_buf, log_buf_size); |
55cffde2 WN |
4893 | |
4894 | if (ret >= 0) { | |
da11b417 AS |
4895 | if (load_attr.log_level) |
4896 | pr_debug("verifier log:\n%s", log_buf); | |
55cffde2 WN |
4897 | *pfd = ret; |
4898 | ret = 0; | |
4899 | goto out; | |
4900 | } | |
4901 | ||
da11b417 AS |
4902 | if (errno == ENOSPC) { |
4903 | log_buf_size <<= 1; | |
4904 | free(log_buf); | |
4905 | goto retry_load; | |
4906 | } | |
4f33ddb4 | 4907 | ret = -errno; |
24d6a808 | 4908 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
be18010e | 4909 | pr_warn("load bpf program failed: %s\n", cp); |
dc3a2d25 | 4910 | pr_perm_msg(ret); |
55cffde2 | 4911 | |
6371ca3b WN |
4912 | if (log_buf && log_buf[0] != '\0') { |
4913 | ret = -LIBBPF_ERRNO__VERIFY; | |
be18010e KW |
4914 | pr_warn("-- BEGIN DUMP LOG ---\n"); |
4915 | pr_warn("\n%s\n", log_buf); | |
4916 | pr_warn("-- END LOG --\n"); | |
d7be143b | 4917 | } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { |
be18010e KW |
4918 | pr_warn("Program too large (%zu insns), at most %d insns\n", |
4919 | load_attr.insns_cnt, BPF_MAXINSNS); | |
705fa219 | 4920 | ret = -LIBBPF_ERRNO__PROG2BIG; |
4f33ddb4 | 4921 | } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { |
705fa219 | 4922 | /* Wrong program type? */ |
4f33ddb4 | 4923 | int fd; |
705fa219 | 4924 | |
4f33ddb4 THJ |
4925 | load_attr.prog_type = BPF_PROG_TYPE_KPROBE; |
4926 | load_attr.expected_attach_type = 0; | |
4927 | fd = bpf_load_program_xattr(&load_attr, NULL, 0); | |
4928 | if (fd >= 0) { | |
4929 | close(fd); | |
4930 | ret = -LIBBPF_ERRNO__PROGTYPE; | |
4931 | goto out; | |
4932 | } | |
55cffde2 WN |
4933 | } |
4934 | ||
4935 | out: | |
4936 | free(log_buf); | |
4937 | return ret; | |
4938 | } | |
4939 | ||
a6ed02ca | 4940 | static int libbpf_find_attach_btf_id(struct bpf_program *prog); |
13acb508 AN |
4941 | |
4942 | int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) | |
55cffde2 | 4943 | { |
13acb508 AN |
4944 | int err = 0, fd, i, btf_id; |
4945 | ||
ff26ce5c EC |
4946 | if ((prog->type == BPF_PROG_TYPE_TRACING || |
4947 | prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { | |
a6ed02ca | 4948 | btf_id = libbpf_find_attach_btf_id(prog); |
13acb508 AN |
4949 | if (btf_id <= 0) |
4950 | return btf_id; | |
4951 | prog->attach_btf_id = btf_id; | |
4952 | } | |
55cffde2 | 4953 | |
b580563e WN |
4954 | if (prog->instances.nr < 0 || !prog->instances.fds) { |
4955 | if (prog->preprocessor) { | |
be18010e KW |
4956 | pr_warn("Internal error: can't load program '%s'\n", |
4957 | prog->section_name); | |
b580563e WN |
4958 | return -LIBBPF_ERRNO__INTERNAL; |
4959 | } | |
55cffde2 | 4960 | |
b580563e WN |
4961 | prog->instances.fds = malloc(sizeof(int)); |
4962 | if (!prog->instances.fds) { | |
be18010e | 4963 | pr_warn("Not enough memory for BPF fds\n"); |
b580563e WN |
4964 | return -ENOMEM; |
4965 | } | |
4966 | prog->instances.nr = 1; | |
4967 | prog->instances.fds[0] = -1; | |
4968 | } | |
4969 | ||
4970 | if (!prog->preprocessor) { | |
4971 | if (prog->instances.nr != 1) { | |
be18010e KW |
4972 | pr_warn("Program '%s' is inconsistent: nr(%d) != 1\n", |
4973 | prog->section_name, prog->instances.nr); | |
b580563e | 4974 | } |
2993e051 | 4975 | err = load_program(prog, prog->insns, prog->insns_cnt, |
13acb508 | 4976 | license, kern_ver, &fd); |
b580563e WN |
4977 | if (!err) |
4978 | prog->instances.fds[0] = fd; | |
4979 | goto out; | |
4980 | } | |
4981 | ||
4982 | for (i = 0; i < prog->instances.nr; i++) { | |
4983 | struct bpf_prog_prep_result result; | |
4984 | bpf_program_prep_t preprocessor = prog->preprocessor; | |
4985 | ||
1ad9cbb8 | 4986 | memset(&result, 0, sizeof(result)); |
b580563e WN |
4987 | err = preprocessor(prog, i, prog->insns, |
4988 | prog->insns_cnt, &result); | |
4989 | if (err) { | |
be18010e KW |
4990 | pr_warn("Preprocessing the %dth instance of program '%s' failed\n", |
4991 | i, prog->section_name); | |
b580563e WN |
4992 | goto out; |
4993 | } | |
4994 | ||
4995 | if (!result.new_insn_ptr || !result.new_insn_cnt) { | |
4996 | pr_debug("Skip loading the %dth instance of program '%s'\n", | |
4997 | i, prog->section_name); | |
4998 | prog->instances.fds[i] = -1; | |
4999 | if (result.pfd) | |
5000 | *result.pfd = -1; | |
5001 | continue; | |
5002 | } | |
5003 | ||
2993e051 | 5004 | err = load_program(prog, result.new_insn_ptr, |
13acb508 | 5005 | result.new_insn_cnt, license, kern_ver, &fd); |
b580563e | 5006 | if (err) { |
be18010e KW |
5007 | pr_warn("Loading the %dth instance of program '%s' failed\n", |
5008 | i, prog->section_name); | |
b580563e WN |
5009 | goto out; |
5010 | } | |
5011 | ||
5012 | if (result.pfd) | |
5013 | *result.pfd = fd; | |
5014 | prog->instances.fds[i] = fd; | |
5015 | } | |
5016 | out: | |
55cffde2 | 5017 | if (err) |
be18010e | 5018 | pr_warn("failed to load program '%s'\n", prog->section_name); |
55cffde2 WN |
5019 | zfree(&prog->insns); |
5020 | prog->insns_cnt = 0; | |
5021 | return err; | |
5022 | } | |
5023 | ||
a324aae3 AN |
5024 | static bool bpf_program__is_function_storage(const struct bpf_program *prog, |
5025 | const struct bpf_object *obj) | |
9a94f277 JK |
5026 | { |
5027 | return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; | |
5028 | } | |
5029 | ||
55cffde2 | 5030 | static int |
60276f98 | 5031 | bpf_object__load_progs(struct bpf_object *obj, int log_level) |
55cffde2 WN |
5032 | { |
5033 | size_t i; | |
5034 | int err; | |
5035 | ||
5036 | for (i = 0; i < obj->nr_programs; i++) { | |
9a94f277 | 5037 | if (bpf_program__is_function_storage(&obj->programs[i], obj)) |
48cca7e4 | 5038 | continue; |
501b125a | 5039 | obj->programs[i].log_level |= log_level; |
55cffde2 WN |
5040 | err = bpf_program__load(&obj->programs[i], |
5041 | obj->license, | |
5042 | obj->kern_version); | |
5043 | if (err) | |
5044 | return err; | |
5045 | } | |
5046 | return 0; | |
5047 | } | |
5048 | ||
1a5e3fb1 | 5049 | static struct bpf_object * |
5e61f270 | 5050 | __bpf_object__open(const char *path, const void *obj_buf, size_t obj_buf_sz, |
01af3bf0 | 5051 | const struct bpf_object_open_opts *opts) |
1a5e3fb1 | 5052 | { |
8601fd42 | 5053 | const char *obj_name, *kconfig; |
dd4436bb | 5054 | struct bpf_program *prog; |
1a5e3fb1 | 5055 | struct bpf_object *obj; |
291ee02b | 5056 | char tmp_name[64]; |
6371ca3b | 5057 | int err; |
1a5e3fb1 WN |
5058 | |
5059 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
be18010e KW |
5060 | pr_warn("failed to init libelf for %s\n", |
5061 | path ? : "(mem buf)"); | |
6371ca3b | 5062 | return ERR_PTR(-LIBBPF_ERRNO__LIBELF); |
1a5e3fb1 WN |
5063 | } |
5064 | ||
291ee02b AN |
5065 | if (!OPTS_VALID(opts, bpf_object_open_opts)) |
5066 | return ERR_PTR(-EINVAL); | |
5067 | ||
1aace10f | 5068 | obj_name = OPTS_GET(opts, object_name, NULL); |
291ee02b AN |
5069 | if (obj_buf) { |
5070 | if (!obj_name) { | |
5071 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
5072 | (unsigned long)obj_buf, | |
5073 | (unsigned long)obj_buf_sz); | |
5074 | obj_name = tmp_name; | |
5075 | } | |
5076 | path = obj_name; | |
5077 | pr_debug("loading object '%s' from buffer\n", obj_name); | |
5078 | } | |
5079 | ||
2ce8450e | 5080 | obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); |
6371ca3b WN |
5081 | if (IS_ERR(obj)) |
5082 | return obj; | |
1a5e3fb1 | 5083 | |
8601fd42 AN |
5084 | kconfig = OPTS_GET(opts, kconfig, NULL); |
5085 | if (kconfig) { | |
5086 | obj->kconfig = strdup(kconfig); | |
5087 | if (!obj->kconfig) | |
166750bc AN |
5088 | return ERR_PTR(-ENOMEM); |
5089 | } | |
291ee02b | 5090 | |
0d13bfce AN |
5091 | err = bpf_object__elf_init(obj); |
5092 | err = err ? : bpf_object__check_endianness(obj); | |
5093 | err = err ? : bpf_object__elf_collect(obj); | |
166750bc AN |
5094 | err = err ? : bpf_object__collect_externs(obj); |
5095 | err = err ? : bpf_object__finalize_btf(obj); | |
0d13bfce AN |
5096 | err = err ? : bpf_object__init_maps(obj, opts); |
5097 | err = err ? : bpf_object__init_prog_names(obj); | |
5098 | err = err ? : bpf_object__collect_reloc(obj); | |
5099 | if (err) | |
5100 | goto out; | |
1a5e3fb1 | 5101 | bpf_object__elf_finish(obj); |
dd4436bb AN |
5102 | |
5103 | bpf_object__for_each_program(prog, obj) { | |
5104 | enum bpf_prog_type prog_type; | |
5105 | enum bpf_attach_type attach_type; | |
5106 | ||
590a0088 MKL |
5107 | if (prog->type != BPF_PROG_TYPE_UNSPEC) |
5108 | continue; | |
5109 | ||
dd4436bb AN |
5110 | err = libbpf_prog_type_by_name(prog->section_name, &prog_type, |
5111 | &attach_type); | |
5112 | if (err == -ESRCH) | |
5113 | /* couldn't guess, but user might manually specify */ | |
5114 | continue; | |
5115 | if (err) | |
5116 | goto out; | |
5117 | ||
5118 | bpf_program__set_type(prog, prog_type); | |
5119 | bpf_program__set_expected_attach_type(prog, attach_type); | |
2db6eab1 AS |
5120 | if (prog_type == BPF_PROG_TYPE_TRACING || |
5121 | prog_type == BPF_PROG_TYPE_EXT) | |
166750bc | 5122 | prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); |
dd4436bb AN |
5123 | } |
5124 | ||
1a5e3fb1 WN |
5125 | return obj; |
5126 | out: | |
5127 | bpf_object__close(obj); | |
6371ca3b | 5128 | return ERR_PTR(err); |
1a5e3fb1 WN |
5129 | } |
5130 | ||
5e61f270 AN |
5131 | static struct bpf_object * |
5132 | __bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) | |
1a5e3fb1 | 5133 | { |
e00aca65 | 5134 | DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, |
291ee02b AN |
5135 | .relaxed_maps = flags & MAPS_RELAX_COMPAT, |
5136 | ); | |
5137 | ||
1a5e3fb1 | 5138 | /* param validation */ |
07f2d4ea | 5139 | if (!attr->file) |
1a5e3fb1 WN |
5140 | return NULL; |
5141 | ||
07f2d4ea | 5142 | pr_debug("loading %s\n", attr->file); |
291ee02b | 5143 | return __bpf_object__open(attr->file, NULL, 0, &opts); |
c034a177 JF |
5144 | } |
5145 | ||
5146 | struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) | |
5147 | { | |
5148 | return __bpf_object__open_xattr(attr, 0); | |
07f2d4ea JK |
5149 | } |
5150 | ||
5151 | struct bpf_object *bpf_object__open(const char *path) | |
5152 | { | |
5153 | struct bpf_object_open_attr attr = { | |
5154 | .file = path, | |
5155 | .prog_type = BPF_PROG_TYPE_UNSPEC, | |
5156 | }; | |
1a5e3fb1 | 5157 | |
07f2d4ea | 5158 | return bpf_object__open_xattr(&attr); |
6c956392 WN |
5159 | } |
5160 | ||
2ce8450e | 5161 | struct bpf_object * |
01af3bf0 | 5162 | bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) |
2ce8450e | 5163 | { |
2ce8450e AN |
5164 | if (!path) |
5165 | return ERR_PTR(-EINVAL); | |
5166 | ||
5167 | pr_debug("loading %s\n", path); | |
5168 | ||
291ee02b | 5169 | return __bpf_object__open(path, NULL, 0, opts); |
2ce8450e AN |
5170 | } |
5171 | ||
5172 | struct bpf_object * | |
5173 | bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, | |
01af3bf0 | 5174 | const struct bpf_object_open_opts *opts) |
6c956392 | 5175 | { |
2ce8450e AN |
5176 | if (!obj_buf || obj_buf_sz == 0) |
5177 | return ERR_PTR(-EINVAL); | |
6c956392 | 5178 | |
291ee02b | 5179 | return __bpf_object__open(NULL, obj_buf, obj_buf_sz, opts); |
2ce8450e AN |
5180 | } |
5181 | ||
5182 | struct bpf_object * | |
5183 | bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, | |
5184 | const char *name) | |
5185 | { | |
e00aca65 | 5186 | DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, |
2ce8450e AN |
5187 | .object_name = name, |
5188 | /* wrong default, but backwards-compatible */ | |
5189 | .relaxed_maps = true, | |
5190 | ); | |
5191 | ||
5192 | /* returning NULL is wrong, but backwards-compatible */ | |
5193 | if (!obj_buf || obj_buf_sz == 0) | |
5194 | return NULL; | |
6c956392 | 5195 | |
2ce8450e | 5196 | return bpf_object__open_mem(obj_buf, obj_buf_sz, &opts); |
1a5e3fb1 WN |
5197 | } |
5198 | ||
52d3352e WN |
5199 | int bpf_object__unload(struct bpf_object *obj) |
5200 | { | |
5201 | size_t i; | |
5202 | ||
5203 | if (!obj) | |
5204 | return -EINVAL; | |
5205 | ||
590a0088 | 5206 | for (i = 0; i < obj->nr_maps; i++) { |
9d759a9b | 5207 | zclose(obj->maps[i].fd); |
590a0088 MKL |
5208 | if (obj->maps[i].st_ops) |
5209 | zfree(&obj->maps[i].st_ops->kern_vdata); | |
5210 | } | |
52d3352e | 5211 | |
55cffde2 WN |
5212 | for (i = 0; i < obj->nr_programs; i++) |
5213 | bpf_program__unload(&obj->programs[i]); | |
5214 | ||
52d3352e WN |
5215 | return 0; |
5216 | } | |
5217 | ||
0d13bfce AN |
5218 | static int bpf_object__sanitize_maps(struct bpf_object *obj) |
5219 | { | |
5220 | struct bpf_map *m; | |
5221 | ||
5222 | bpf_object__for_each_map(m, obj) { | |
5223 | if (!bpf_map__is_internal(m)) | |
5224 | continue; | |
5225 | if (!obj->caps.global_data) { | |
5226 | pr_warn("kernel doesn't support global data\n"); | |
5227 | return -ENOTSUP; | |
5228 | } | |
5229 | if (!obj->caps.array_mmap) | |
5230 | m->def.map_flags ^= BPF_F_MMAPABLE; | |
5231 | } | |
5232 | ||
5233 | return 0; | |
5234 | } | |
5235 | ||
166750bc | 5236 | static int bpf_object__resolve_externs(struct bpf_object *obj, |
8601fd42 | 5237 | const char *extra_kconfig) |
166750bc AN |
5238 | { |
5239 | bool need_config = false; | |
5240 | struct extern_desc *ext; | |
5241 | int err, i; | |
5242 | void *data; | |
5243 | ||
5244 | if (obj->nr_extern == 0) | |
5245 | return 0; | |
5246 | ||
81bfdd08 | 5247 | data = obj->maps[obj->kconfig_map_idx].mmaped; |
166750bc AN |
5248 | |
5249 | for (i = 0; i < obj->nr_extern; i++) { | |
5250 | ext = &obj->externs[i]; | |
5251 | ||
5252 | if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { | |
5253 | void *ext_val = data + ext->data_off; | |
5254 | __u32 kver = get_kernel_version(); | |
5255 | ||
5256 | if (!kver) { | |
5257 | pr_warn("failed to get kernel version\n"); | |
5258 | return -EINVAL; | |
5259 | } | |
5260 | err = set_ext_value_num(ext, ext_val, kver); | |
5261 | if (err) | |
5262 | return err; | |
5263 | pr_debug("extern %s=0x%x\n", ext->name, kver); | |
5264 | } else if (strncmp(ext->name, "CONFIG_", 7) == 0) { | |
5265 | need_config = true; | |
5266 | } else { | |
5267 | pr_warn("unrecognized extern '%s'\n", ext->name); | |
5268 | return -EINVAL; | |
5269 | } | |
5270 | } | |
8601fd42 AN |
5271 | if (need_config && extra_kconfig) { |
5272 | err = bpf_object__read_kconfig_mem(obj, extra_kconfig, data); | |
5273 | if (err) | |
5274 | return -EINVAL; | |
5275 | need_config = false; | |
5276 | for (i = 0; i < obj->nr_extern; i++) { | |
5277 | ext = &obj->externs[i]; | |
5278 | if (!ext->is_set) { | |
5279 | need_config = true; | |
5280 | break; | |
5281 | } | |
5282 | } | |
5283 | } | |
166750bc | 5284 | if (need_config) { |
8601fd42 | 5285 | err = bpf_object__read_kconfig_file(obj, data); |
166750bc AN |
5286 | if (err) |
5287 | return -EINVAL; | |
5288 | } | |
5289 | for (i = 0; i < obj->nr_extern; i++) { | |
5290 | ext = &obj->externs[i]; | |
5291 | ||
5292 | if (!ext->is_set && !ext->is_weak) { | |
5293 | pr_warn("extern %s (strong) not resolved\n", ext->name); | |
5294 | return -ESRCH; | |
5295 | } else if (!ext->is_set) { | |
5296 | pr_debug("extern %s (weak) not resolved, defaulting to zero\n", | |
5297 | ext->name); | |
5298 | } | |
5299 | } | |
5300 | ||
5301 | return 0; | |
5302 | } | |
5303 | ||
60276f98 | 5304 | int bpf_object__load_xattr(struct bpf_object_load_attr *attr) |
52d3352e | 5305 | { |
60276f98 | 5306 | struct bpf_object *obj; |
ec6d5f47 | 5307 | int err, i; |
6371ca3b | 5308 | |
60276f98 QM |
5309 | if (!attr) |
5310 | return -EINVAL; | |
5311 | obj = attr->obj; | |
52d3352e WN |
5312 | if (!obj) |
5313 | return -EINVAL; | |
5314 | ||
5315 | if (obj->loaded) { | |
be18010e | 5316 | pr_warn("object should not be loaded twice\n"); |
52d3352e WN |
5317 | return -EINVAL; |
5318 | } | |
5319 | ||
5320 | obj->loaded = true; | |
6371ca3b | 5321 | |
0d13bfce | 5322 | err = bpf_object__probe_caps(obj); |
8601fd42 | 5323 | err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); |
0d13bfce AN |
5324 | err = err ? : bpf_object__sanitize_and_load_btf(obj); |
5325 | err = err ? : bpf_object__sanitize_maps(obj); | |
a6ed02ca | 5326 | err = err ? : bpf_object__load_vmlinux_btf(obj); |
590a0088 | 5327 | err = err ? : bpf_object__init_kern_struct_ops_maps(obj); |
0d13bfce AN |
5328 | err = err ? : bpf_object__create_maps(obj); |
5329 | err = err ? : bpf_object__relocate(obj, attr->target_btf_path); | |
5330 | err = err ? : bpf_object__load_progs(obj, attr->log_level); | |
a6ed02ca KS |
5331 | |
5332 | btf__free(obj->btf_vmlinux); | |
5333 | obj->btf_vmlinux = NULL; | |
5334 | ||
0d13bfce AN |
5335 | if (err) |
5336 | goto out; | |
52d3352e WN |
5337 | |
5338 | return 0; | |
5339 | out: | |
ec6d5f47 THJ |
5340 | /* unpin any maps that were auto-pinned during load */ |
5341 | for (i = 0; i < obj->nr_maps; i++) | |
5342 | if (obj->maps[i].pinned && !obj->maps[i].reused) | |
5343 | bpf_map__unpin(&obj->maps[i], NULL); | |
5344 | ||
52d3352e | 5345 | bpf_object__unload(obj); |
be18010e | 5346 | pr_warn("failed to load object '%s'\n", obj->path); |
6371ca3b | 5347 | return err; |
52d3352e WN |
5348 | } |
5349 | ||
60276f98 QM |
5350 | int bpf_object__load(struct bpf_object *obj) |
5351 | { | |
5352 | struct bpf_object_load_attr attr = { | |
5353 | .obj = obj, | |
5354 | }; | |
5355 | ||
5356 | return bpf_object__load_xattr(&attr); | |
5357 | } | |
5358 | ||
196f8487 THJ |
5359 | static int make_parent_dir(const char *path) |
5360 | { | |
5361 | char *cp, errmsg[STRERR_BUFSIZE]; | |
5362 | char *dname, *dir; | |
5363 | int err = 0; | |
5364 | ||
5365 | dname = strdup(path); | |
5366 | if (dname == NULL) | |
5367 | return -ENOMEM; | |
5368 | ||
5369 | dir = dirname(dname); | |
5370 | if (mkdir(dir, 0700) && errno != EEXIST) | |
5371 | err = -errno; | |
5372 | ||
5373 | free(dname); | |
5374 | if (err) { | |
5375 | cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); | |
5376 | pr_warn("failed to mkdir %s: %s\n", path, cp); | |
5377 | } | |
5378 | return err; | |
5379 | } | |
5380 | ||
f367540c JS |
5381 | static int check_path(const char *path) |
5382 | { | |
1ce6a9fc | 5383 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
5384 | struct statfs st_fs; |
5385 | char *dname, *dir; | |
5386 | int err = 0; | |
5387 | ||
5388 | if (path == NULL) | |
5389 | return -EINVAL; | |
5390 | ||
5391 | dname = strdup(path); | |
5392 | if (dname == NULL) | |
5393 | return -ENOMEM; | |
5394 | ||
5395 | dir = dirname(dname); | |
5396 | if (statfs(dir, &st_fs)) { | |
24d6a808 | 5397 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
be18010e | 5398 | pr_warn("failed to statfs %s: %s\n", dir, cp); |
f367540c JS |
5399 | err = -errno; |
5400 | } | |
5401 | free(dname); | |
5402 | ||
5403 | if (!err && st_fs.f_type != BPF_FS_MAGIC) { | |
be18010e | 5404 | pr_warn("specified path %s is not on BPF FS\n", path); |
f367540c JS |
5405 | err = -EINVAL; |
5406 | } | |
5407 | ||
5408 | return err; | |
5409 | } | |
5410 | ||
5411 | int bpf_program__pin_instance(struct bpf_program *prog, const char *path, | |
5412 | int instance) | |
5413 | { | |
1ce6a9fc | 5414 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
5415 | int err; |
5416 | ||
196f8487 THJ |
5417 | err = make_parent_dir(path); |
5418 | if (err) | |
5419 | return err; | |
5420 | ||
f367540c JS |
5421 | err = check_path(path); |
5422 | if (err) | |
5423 | return err; | |
5424 | ||
5425 | if (prog == NULL) { | |
be18010e | 5426 | pr_warn("invalid program pointer\n"); |
f367540c JS |
5427 | return -EINVAL; |
5428 | } | |
5429 | ||
5430 | if (instance < 0 || instance >= prog->instances.nr) { | |
be18010e KW |
5431 | pr_warn("invalid prog instance %d of prog %s (max %d)\n", |
5432 | instance, prog->section_name, prog->instances.nr); | |
f367540c JS |
5433 | return -EINVAL; |
5434 | } | |
5435 | ||
5436 | if (bpf_obj_pin(prog->instances.fds[instance], path)) { | |
24d6a808 | 5437 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
be18010e | 5438 | pr_warn("failed to pin program: %s\n", cp); |
f367540c JS |
5439 | return -errno; |
5440 | } | |
5441 | pr_debug("pinned program '%s'\n", path); | |
5442 | ||
5443 | return 0; | |
5444 | } | |
5445 | ||
0c19a9fb SF |
5446 | int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, |
5447 | int instance) | |
5448 | { | |
5449 | int err; | |
5450 | ||
5451 | err = check_path(path); | |
5452 | if (err) | |
5453 | return err; | |
5454 | ||
5455 | if (prog == NULL) { | |
be18010e | 5456 | pr_warn("invalid program pointer\n"); |
0c19a9fb SF |
5457 | return -EINVAL; |
5458 | } | |
5459 | ||
5460 | if (instance < 0 || instance >= prog->instances.nr) { | |
be18010e KW |
5461 | pr_warn("invalid prog instance %d of prog %s (max %d)\n", |
5462 | instance, prog->section_name, prog->instances.nr); | |
0c19a9fb SF |
5463 | return -EINVAL; |
5464 | } | |
5465 | ||
5466 | err = unlink(path); | |
5467 | if (err != 0) | |
5468 | return -errno; | |
5469 | pr_debug("unpinned program '%s'\n", path); | |
5470 | ||
5471 | return 0; | |
5472 | } | |
5473 | ||
f367540c JS |
5474 | int bpf_program__pin(struct bpf_program *prog, const char *path) |
5475 | { | |
5476 | int i, err; | |
5477 | ||
196f8487 THJ |
5478 | err = make_parent_dir(path); |
5479 | if (err) | |
5480 | return err; | |
5481 | ||
f367540c JS |
5482 | err = check_path(path); |
5483 | if (err) | |
5484 | return err; | |
5485 | ||
5486 | if (prog == NULL) { | |
be18010e | 5487 | pr_warn("invalid program pointer\n"); |
f367540c JS |
5488 | return -EINVAL; |
5489 | } | |
5490 | ||
5491 | if (prog->instances.nr <= 0) { | |
be18010e | 5492 | pr_warn("no instances of prog %s to pin\n", |
f367540c JS |
5493 | prog->section_name); |
5494 | return -EINVAL; | |
5495 | } | |
5496 | ||
fd734c5c SF |
5497 | if (prog->instances.nr == 1) { |
5498 | /* don't create subdirs when pinning single instance */ | |
5499 | return bpf_program__pin_instance(prog, path, 0); | |
5500 | } | |
5501 | ||
0c19a9fb SF |
5502 | for (i = 0; i < prog->instances.nr; i++) { |
5503 | char buf[PATH_MAX]; | |
5504 | int len; | |
5505 | ||
5506 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
5507 | if (len < 0) { | |
5508 | err = -EINVAL; | |
5509 | goto err_unpin; | |
5510 | } else if (len >= PATH_MAX) { | |
5511 | err = -ENAMETOOLONG; | |
5512 | goto err_unpin; | |
5513 | } | |
5514 | ||
5515 | err = bpf_program__pin_instance(prog, buf, i); | |
5516 | if (err) | |
5517 | goto err_unpin; | |
5518 | } | |
5519 | ||
5520 | return 0; | |
5521 | ||
5522 | err_unpin: | |
5523 | for (i = i - 1; i >= 0; i--) { | |
5524 | char buf[PATH_MAX]; | |
5525 | int len; | |
5526 | ||
5527 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
5528 | if (len < 0) | |
5529 | continue; | |
5530 | else if (len >= PATH_MAX) | |
5531 | continue; | |
5532 | ||
5533 | bpf_program__unpin_instance(prog, buf, i); | |
5534 | } | |
5535 | ||
5536 | rmdir(path); | |
5537 | ||
5538 | return err; | |
5539 | } | |
5540 | ||
5541 | int bpf_program__unpin(struct bpf_program *prog, const char *path) | |
5542 | { | |
5543 | int i, err; | |
5544 | ||
5545 | err = check_path(path); | |
5546 | if (err) | |
5547 | return err; | |
5548 | ||
5549 | if (prog == NULL) { | |
be18010e | 5550 | pr_warn("invalid program pointer\n"); |
0c19a9fb SF |
5551 | return -EINVAL; |
5552 | } | |
5553 | ||
5554 | if (prog->instances.nr <= 0) { | |
be18010e | 5555 | pr_warn("no instances of prog %s to pin\n", |
0c19a9fb SF |
5556 | prog->section_name); |
5557 | return -EINVAL; | |
fd734c5c SF |
5558 | } |
5559 | ||
5560 | if (prog->instances.nr == 1) { | |
5561 | /* don't create subdirs when pinning single instance */ | |
5562 | return bpf_program__unpin_instance(prog, path, 0); | |
0c19a9fb SF |
5563 | } |
5564 | ||
f367540c JS |
5565 | for (i = 0; i < prog->instances.nr; i++) { |
5566 | char buf[PATH_MAX]; | |
5567 | int len; | |
5568 | ||
5569 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
5570 | if (len < 0) | |
5571 | return -EINVAL; | |
5572 | else if (len >= PATH_MAX) | |
5573 | return -ENAMETOOLONG; | |
5574 | ||
0c19a9fb | 5575 | err = bpf_program__unpin_instance(prog, buf, i); |
f367540c JS |
5576 | if (err) |
5577 | return err; | |
5578 | } | |
5579 | ||
0c19a9fb SF |
5580 | err = rmdir(path); |
5581 | if (err) | |
5582 | return -errno; | |
5583 | ||
f367540c JS |
5584 | return 0; |
5585 | } | |
5586 | ||
b6989f35 JS |
5587 | int bpf_map__pin(struct bpf_map *map, const char *path) |
5588 | { | |
1ce6a9fc | 5589 | char *cp, errmsg[STRERR_BUFSIZE]; |
b6989f35 JS |
5590 | int err; |
5591 | ||
b6989f35 | 5592 | if (map == NULL) { |
be18010e | 5593 | pr_warn("invalid map pointer\n"); |
b6989f35 JS |
5594 | return -EINVAL; |
5595 | } | |
5596 | ||
4580b25f THJ |
5597 | if (map->pin_path) { |
5598 | if (path && strcmp(path, map->pin_path)) { | |
5599 | pr_warn("map '%s' already has pin path '%s' different from '%s'\n", | |
5600 | bpf_map__name(map), map->pin_path, path); | |
5601 | return -EINVAL; | |
5602 | } else if (map->pinned) { | |
5603 | pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", | |
5604 | bpf_map__name(map), map->pin_path); | |
5605 | return 0; | |
5606 | } | |
5607 | } else { | |
5608 | if (!path) { | |
5609 | pr_warn("missing a path to pin map '%s' at\n", | |
5610 | bpf_map__name(map)); | |
5611 | return -EINVAL; | |
5612 | } else if (map->pinned) { | |
5613 | pr_warn("map '%s' already pinned\n", bpf_map__name(map)); | |
5614 | return -EEXIST; | |
5615 | } | |
5616 | ||
5617 | map->pin_path = strdup(path); | |
5618 | if (!map->pin_path) { | |
5619 | err = -errno; | |
5620 | goto out_err; | |
5621 | } | |
b6989f35 JS |
5622 | } |
5623 | ||
196f8487 THJ |
5624 | err = make_parent_dir(map->pin_path); |
5625 | if (err) | |
5626 | return err; | |
5627 | ||
4580b25f THJ |
5628 | err = check_path(map->pin_path); |
5629 | if (err) | |
5630 | return err; | |
5631 | ||
5632 | if (bpf_obj_pin(map->fd, map->pin_path)) { | |
5633 | err = -errno; | |
5634 | goto out_err; | |
5635 | } | |
5636 | ||
5637 | map->pinned = true; | |
5638 | pr_debug("pinned map '%s'\n", map->pin_path); | |
0c19a9fb | 5639 | |
b6989f35 | 5640 | return 0; |
4580b25f THJ |
5641 | |
5642 | out_err: | |
5643 | cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); | |
5644 | pr_warn("failed to pin map: %s\n", cp); | |
5645 | return err; | |
b6989f35 JS |
5646 | } |
5647 | ||
0c19a9fb SF |
5648 | int bpf_map__unpin(struct bpf_map *map, const char *path) |
5649 | { | |
5650 | int err; | |
5651 | ||
0c19a9fb | 5652 | if (map == NULL) { |
be18010e | 5653 | pr_warn("invalid map pointer\n"); |
0c19a9fb SF |
5654 | return -EINVAL; |
5655 | } | |
5656 | ||
4580b25f THJ |
5657 | if (map->pin_path) { |
5658 | if (path && strcmp(path, map->pin_path)) { | |
5659 | pr_warn("map '%s' already has pin path '%s' different from '%s'\n", | |
5660 | bpf_map__name(map), map->pin_path, path); | |
5661 | return -EINVAL; | |
5662 | } | |
5663 | path = map->pin_path; | |
5664 | } else if (!path) { | |
5665 | pr_warn("no path to unpin map '%s' from\n", | |
5666 | bpf_map__name(map)); | |
5667 | return -EINVAL; | |
5668 | } | |
5669 | ||
5670 | err = check_path(path); | |
5671 | if (err) | |
5672 | return err; | |
5673 | ||
0c19a9fb SF |
5674 | err = unlink(path); |
5675 | if (err != 0) | |
5676 | return -errno; | |
4580b25f THJ |
5677 | |
5678 | map->pinned = false; | |
5679 | pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); | |
0c19a9fb SF |
5680 | |
5681 | return 0; | |
5682 | } | |
5683 | ||
4580b25f THJ |
5684 | int bpf_map__set_pin_path(struct bpf_map *map, const char *path) |
5685 | { | |
5686 | char *new = NULL; | |
5687 | ||
5688 | if (path) { | |
5689 | new = strdup(path); | |
5690 | if (!new) | |
5691 | return -errno; | |
5692 | } | |
5693 | ||
5694 | free(map->pin_path); | |
5695 | map->pin_path = new; | |
5696 | return 0; | |
5697 | } | |
5698 | ||
5699 | const char *bpf_map__get_pin_path(const struct bpf_map *map) | |
5700 | { | |
5701 | return map->pin_path; | |
5702 | } | |
5703 | ||
5704 | bool bpf_map__is_pinned(const struct bpf_map *map) | |
5705 | { | |
5706 | return map->pinned; | |
5707 | } | |
5708 | ||
0c19a9fb | 5709 | int bpf_object__pin_maps(struct bpf_object *obj, const char *path) |
d5148d85 | 5710 | { |
d5148d85 JS |
5711 | struct bpf_map *map; |
5712 | int err; | |
5713 | ||
5714 | if (!obj) | |
5715 | return -ENOENT; | |
5716 | ||
5717 | if (!obj->loaded) { | |
be18010e | 5718 | pr_warn("object not yet loaded; load it first\n"); |
d5148d85 JS |
5719 | return -ENOENT; |
5720 | } | |
5721 | ||
f74a53d9 | 5722 | bpf_object__for_each_map(map, obj) { |
4580b25f | 5723 | char *pin_path = NULL; |
0c19a9fb | 5724 | char buf[PATH_MAX]; |
0c19a9fb | 5725 | |
4580b25f THJ |
5726 | if (path) { |
5727 | int len; | |
5728 | ||
5729 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
5730 | bpf_map__name(map)); | |
5731 | if (len < 0) { | |
5732 | err = -EINVAL; | |
5733 | goto err_unpin_maps; | |
5734 | } else if (len >= PATH_MAX) { | |
5735 | err = -ENAMETOOLONG; | |
5736 | goto err_unpin_maps; | |
5737 | } | |
5738 | pin_path = buf; | |
5739 | } else if (!map->pin_path) { | |
5740 | continue; | |
0c19a9fb SF |
5741 | } |
5742 | ||
4580b25f | 5743 | err = bpf_map__pin(map, pin_path); |
0c19a9fb SF |
5744 | if (err) |
5745 | goto err_unpin_maps; | |
5746 | } | |
5747 | ||
5748 | return 0; | |
5749 | ||
5750 | err_unpin_maps: | |
5751 | while ((map = bpf_map__prev(map, obj))) { | |
4580b25f | 5752 | if (!map->pin_path) |
0c19a9fb SF |
5753 | continue; |
5754 | ||
4580b25f | 5755 | bpf_map__unpin(map, NULL); |
0c19a9fb SF |
5756 | } |
5757 | ||
5758 | return err; | |
5759 | } | |
5760 | ||
5761 | int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) | |
5762 | { | |
5763 | struct bpf_map *map; | |
5764 | int err; | |
5765 | ||
5766 | if (!obj) | |
5767 | return -ENOENT; | |
5768 | ||
f74a53d9 | 5769 | bpf_object__for_each_map(map, obj) { |
4580b25f | 5770 | char *pin_path = NULL; |
d5148d85 | 5771 | char buf[PATH_MAX]; |
d5148d85 | 5772 | |
4580b25f THJ |
5773 | if (path) { |
5774 | int len; | |
5775 | ||
5776 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
5777 | bpf_map__name(map)); | |
5778 | if (len < 0) | |
5779 | return -EINVAL; | |
5780 | else if (len >= PATH_MAX) | |
5781 | return -ENAMETOOLONG; | |
5782 | pin_path = buf; | |
5783 | } else if (!map->pin_path) { | |
5784 | continue; | |
5785 | } | |
d5148d85 | 5786 | |
4580b25f | 5787 | err = bpf_map__unpin(map, pin_path); |
d5148d85 JS |
5788 | if (err) |
5789 | return err; | |
5790 | } | |
5791 | ||
0c19a9fb SF |
5792 | return 0; |
5793 | } | |
5794 | ||
5795 | int bpf_object__pin_programs(struct bpf_object *obj, const char *path) | |
5796 | { | |
5797 | struct bpf_program *prog; | |
5798 | int err; | |
5799 | ||
5800 | if (!obj) | |
5801 | return -ENOENT; | |
5802 | ||
5803 | if (!obj->loaded) { | |
be18010e | 5804 | pr_warn("object not yet loaded; load it first\n"); |
0c19a9fb SF |
5805 | return -ENOENT; |
5806 | } | |
5807 | ||
0c19a9fb SF |
5808 | bpf_object__for_each_program(prog, obj) { |
5809 | char buf[PATH_MAX]; | |
5810 | int len; | |
5811 | ||
5812 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 5813 | prog->pin_name); |
0c19a9fb SF |
5814 | if (len < 0) { |
5815 | err = -EINVAL; | |
5816 | goto err_unpin_programs; | |
5817 | } else if (len >= PATH_MAX) { | |
5818 | err = -ENAMETOOLONG; | |
5819 | goto err_unpin_programs; | |
5820 | } | |
5821 | ||
5822 | err = bpf_program__pin(prog, buf); | |
5823 | if (err) | |
5824 | goto err_unpin_programs; | |
5825 | } | |
5826 | ||
5827 | return 0; | |
5828 | ||
5829 | err_unpin_programs: | |
5830 | while ((prog = bpf_program__prev(prog, obj))) { | |
5831 | char buf[PATH_MAX]; | |
5832 | int len; | |
5833 | ||
5834 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 5835 | prog->pin_name); |
0c19a9fb SF |
5836 | if (len < 0) |
5837 | continue; | |
5838 | else if (len >= PATH_MAX) | |
5839 | continue; | |
5840 | ||
5841 | bpf_program__unpin(prog, buf); | |
5842 | } | |
5843 | ||
5844 | return err; | |
5845 | } | |
5846 | ||
5847 | int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) | |
5848 | { | |
5849 | struct bpf_program *prog; | |
5850 | int err; | |
5851 | ||
5852 | if (!obj) | |
5853 | return -ENOENT; | |
5854 | ||
d5148d85 JS |
5855 | bpf_object__for_each_program(prog, obj) { |
5856 | char buf[PATH_MAX]; | |
5857 | int len; | |
5858 | ||
5859 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
33a2c75c | 5860 | prog->pin_name); |
d5148d85 JS |
5861 | if (len < 0) |
5862 | return -EINVAL; | |
5863 | else if (len >= PATH_MAX) | |
5864 | return -ENAMETOOLONG; | |
5865 | ||
0c19a9fb | 5866 | err = bpf_program__unpin(prog, buf); |
d5148d85 JS |
5867 | if (err) |
5868 | return err; | |
5869 | } | |
5870 | ||
5871 | return 0; | |
5872 | } | |
5873 | ||
0c19a9fb SF |
5874 | int bpf_object__pin(struct bpf_object *obj, const char *path) |
5875 | { | |
5876 | int err; | |
5877 | ||
5878 | err = bpf_object__pin_maps(obj, path); | |
5879 | if (err) | |
5880 | return err; | |
5881 | ||
5882 | err = bpf_object__pin_programs(obj, path); | |
5883 | if (err) { | |
5884 | bpf_object__unpin_maps(obj, path); | |
5885 | return err; | |
5886 | } | |
5887 | ||
5888 | return 0; | |
5889 | } | |
5890 | ||
1a5e3fb1 WN |
5891 | void bpf_object__close(struct bpf_object *obj) |
5892 | { | |
a5b8bd47 WN |
5893 | size_t i; |
5894 | ||
1a5e3fb1 WN |
5895 | if (!obj) |
5896 | return; | |
5897 | ||
10931d24 WN |
5898 | if (obj->clear_priv) |
5899 | obj->clear_priv(obj, obj->priv); | |
5900 | ||
1a5e3fb1 | 5901 | bpf_object__elf_finish(obj); |
52d3352e | 5902 | bpf_object__unload(obj); |
8a138aed | 5903 | btf__free(obj->btf); |
2993e051 | 5904 | btf_ext__free(obj->btf_ext); |
1a5e3fb1 | 5905 | |
9d759a9b | 5906 | for (i = 0; i < obj->nr_maps; i++) { |
eba9c5f4 AN |
5907 | struct bpf_map *map = &obj->maps[i]; |
5908 | ||
5909 | if (map->clear_priv) | |
5910 | map->clear_priv(map, map->priv); | |
5911 | map->priv = NULL; | |
5912 | map->clear_priv = NULL; | |
5913 | ||
5914 | if (map->mmaped) { | |
5915 | munmap(map->mmaped, bpf_map_mmap_sz(map)); | |
5916 | map->mmaped = NULL; | |
5917 | } | |
5918 | ||
590a0088 MKL |
5919 | if (map->st_ops) { |
5920 | zfree(&map->st_ops->data); | |
5921 | zfree(&map->st_ops->progs); | |
5922 | zfree(&map->st_ops->kern_func_off); | |
5923 | zfree(&map->st_ops); | |
5924 | } | |
5925 | ||
eba9c5f4 AN |
5926 | zfree(&map->name); |
5927 | zfree(&map->pin_path); | |
9d759a9b | 5928 | } |
d859900c | 5929 | |
8601fd42 | 5930 | zfree(&obj->kconfig); |
166750bc AN |
5931 | zfree(&obj->externs); |
5932 | obj->nr_extern = 0; | |
5933 | ||
9d759a9b WN |
5934 | zfree(&obj->maps); |
5935 | obj->nr_maps = 0; | |
a5b8bd47 WN |
5936 | |
5937 | if (obj->programs && obj->nr_programs) { | |
5938 | for (i = 0; i < obj->nr_programs; i++) | |
5939 | bpf_program__exit(&obj->programs[i]); | |
5940 | } | |
5941 | zfree(&obj->programs); | |
5942 | ||
9a208eff | 5943 | list_del(&obj->list); |
1a5e3fb1 WN |
5944 | free(obj); |
5945 | } | |
aa9b1ac3 | 5946 | |
9a208eff WN |
5947 | struct bpf_object * |
5948 | bpf_object__next(struct bpf_object *prev) | |
5949 | { | |
5950 | struct bpf_object *next; | |
5951 | ||
5952 | if (!prev) | |
5953 | next = list_first_entry(&bpf_objects_list, | |
5954 | struct bpf_object, | |
5955 | list); | |
5956 | else | |
5957 | next = list_next_entry(prev, list); | |
5958 | ||
5959 | /* Empty list is noticed here so don't need checking on entry. */ | |
5960 | if (&next->list == &bpf_objects_list) | |
5961 | return NULL; | |
5962 | ||
5963 | return next; | |
5964 | } | |
5965 | ||
a324aae3 | 5966 | const char *bpf_object__name(const struct bpf_object *obj) |
acf860ae | 5967 | { |
c9e4c301 | 5968 | return obj ? obj->name : ERR_PTR(-EINVAL); |
acf860ae WN |
5969 | } |
5970 | ||
a324aae3 | 5971 | unsigned int bpf_object__kversion(const struct bpf_object *obj) |
45825d8a | 5972 | { |
a7fe0450 | 5973 | return obj ? obj->kern_version : 0; |
45825d8a WN |
5974 | } |
5975 | ||
a324aae3 | 5976 | struct btf *bpf_object__btf(const struct bpf_object *obj) |
789f6bab AI |
5977 | { |
5978 | return obj ? obj->btf : NULL; | |
5979 | } | |
5980 | ||
8a138aed MKL |
5981 | int bpf_object__btf_fd(const struct bpf_object *obj) |
5982 | { | |
5983 | return obj->btf ? btf__fd(obj->btf) : -1; | |
5984 | } | |
5985 | ||
10931d24 WN |
5986 | int bpf_object__set_priv(struct bpf_object *obj, void *priv, |
5987 | bpf_object_clear_priv_t clear_priv) | |
5988 | { | |
5989 | if (obj->priv && obj->clear_priv) | |
5990 | obj->clear_priv(obj, obj->priv); | |
5991 | ||
5992 | obj->priv = priv; | |
5993 | obj->clear_priv = clear_priv; | |
5994 | return 0; | |
5995 | } | |
5996 | ||
a324aae3 | 5997 | void *bpf_object__priv(const struct bpf_object *obj) |
10931d24 WN |
5998 | { |
5999 | return obj ? obj->priv : ERR_PTR(-EINVAL); | |
6000 | } | |
6001 | ||
eac7d845 | 6002 | static struct bpf_program * |
a324aae3 AN |
6003 | __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, |
6004 | bool forward) | |
aa9b1ac3 | 6005 | { |
a83d6e76 | 6006 | size_t nr_programs = obj->nr_programs; |
0c19a9fb | 6007 | ssize_t idx; |
aa9b1ac3 | 6008 | |
a83d6e76 | 6009 | if (!nr_programs) |
aa9b1ac3 | 6010 | return NULL; |
aa9b1ac3 | 6011 | |
a83d6e76 MKL |
6012 | if (!p) |
6013 | /* Iter from the beginning */ | |
6014 | return forward ? &obj->programs[0] : | |
6015 | &obj->programs[nr_programs - 1]; | |
6016 | ||
0c19a9fb | 6017 | if (p->obj != obj) { |
be18010e | 6018 | pr_warn("error: program handler doesn't match object\n"); |
aa9b1ac3 WN |
6019 | return NULL; |
6020 | } | |
6021 | ||
a83d6e76 | 6022 | idx = (p - obj->programs) + (forward ? 1 : -1); |
0c19a9fb | 6023 | if (idx >= obj->nr_programs || idx < 0) |
aa9b1ac3 WN |
6024 | return NULL; |
6025 | return &obj->programs[idx]; | |
6026 | } | |
6027 | ||
eac7d845 | 6028 | struct bpf_program * |
a324aae3 | 6029 | bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) |
eac7d845 JK |
6030 | { |
6031 | struct bpf_program *prog = prev; | |
6032 | ||
6033 | do { | |
a83d6e76 | 6034 | prog = __bpf_program__iter(prog, obj, true); |
0c19a9fb SF |
6035 | } while (prog && bpf_program__is_function_storage(prog, obj)); |
6036 | ||
6037 | return prog; | |
6038 | } | |
6039 | ||
6040 | struct bpf_program * | |
a324aae3 | 6041 | bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) |
0c19a9fb SF |
6042 | { |
6043 | struct bpf_program *prog = next; | |
6044 | ||
0c19a9fb | 6045 | do { |
a83d6e76 | 6046 | prog = __bpf_program__iter(prog, obj, false); |
eac7d845 JK |
6047 | } while (prog && bpf_program__is_function_storage(prog, obj)); |
6048 | ||
6049 | return prog; | |
6050 | } | |
6051 | ||
edb13ed4 ACM |
6052 | int bpf_program__set_priv(struct bpf_program *prog, void *priv, |
6053 | bpf_program_clear_priv_t clear_priv) | |
aa9b1ac3 WN |
6054 | { |
6055 | if (prog->priv && prog->clear_priv) | |
6056 | prog->clear_priv(prog, prog->priv); | |
6057 | ||
6058 | prog->priv = priv; | |
6059 | prog->clear_priv = clear_priv; | |
6060 | return 0; | |
6061 | } | |
6062 | ||
a324aae3 | 6063 | void *bpf_program__priv(const struct bpf_program *prog) |
aa9b1ac3 | 6064 | { |
be834ffb | 6065 | return prog ? prog->priv : ERR_PTR(-EINVAL); |
aa9b1ac3 WN |
6066 | } |
6067 | ||
9aba3613 JK |
6068 | void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) |
6069 | { | |
6070 | prog->prog_ifindex = ifindex; | |
6071 | } | |
6072 | ||
01af3bf0 AN |
6073 | const char *bpf_program__name(const struct bpf_program *prog) |
6074 | { | |
6075 | return prog->name; | |
6076 | } | |
6077 | ||
a324aae3 | 6078 | const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
6079 | { |
6080 | const char *title; | |
6081 | ||
6082 | title = prog->section_name; | |
715f8db9 | 6083 | if (needs_copy) { |
aa9b1ac3 WN |
6084 | title = strdup(title); |
6085 | if (!title) { | |
be18010e | 6086 | pr_warn("failed to strdup program title\n"); |
6371ca3b | 6087 | return ERR_PTR(-ENOMEM); |
aa9b1ac3 WN |
6088 | } |
6089 | } | |
6090 | ||
6091 | return title; | |
6092 | } | |
6093 | ||
a324aae3 | 6094 | int bpf_program__fd(const struct bpf_program *prog) |
aa9b1ac3 | 6095 | { |
b580563e WN |
6096 | return bpf_program__nth_fd(prog, 0); |
6097 | } | |
6098 | ||
1a734efe THJ |
6099 | size_t bpf_program__size(const struct bpf_program *prog) |
6100 | { | |
6101 | return prog->insns_cnt * sizeof(struct bpf_insn); | |
6102 | } | |
6103 | ||
b580563e WN |
6104 | int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, |
6105 | bpf_program_prep_t prep) | |
6106 | { | |
6107 | int *instances_fds; | |
6108 | ||
6109 | if (nr_instances <= 0 || !prep) | |
6110 | return -EINVAL; | |
6111 | ||
6112 | if (prog->instances.nr > 0 || prog->instances.fds) { | |
be18010e | 6113 | pr_warn("Can't set pre-processor after loading\n"); |
b580563e WN |
6114 | return -EINVAL; |
6115 | } | |
6116 | ||
6117 | instances_fds = malloc(sizeof(int) * nr_instances); | |
6118 | if (!instances_fds) { | |
be18010e | 6119 | pr_warn("alloc memory failed for fds\n"); |
b580563e WN |
6120 | return -ENOMEM; |
6121 | } | |
6122 | ||
6123 | /* fill all fd with -1 */ | |
6124 | memset(instances_fds, -1, sizeof(int) * nr_instances); | |
6125 | ||
6126 | prog->instances.nr = nr_instances; | |
6127 | prog->instances.fds = instances_fds; | |
6128 | prog->preprocessor = prep; | |
6129 | return 0; | |
6130 | } | |
6131 | ||
a324aae3 | 6132 | int bpf_program__nth_fd(const struct bpf_program *prog, int n) |
b580563e WN |
6133 | { |
6134 | int fd; | |
6135 | ||
1e960043 JK |
6136 | if (!prog) |
6137 | return -EINVAL; | |
6138 | ||
b580563e | 6139 | if (n >= prog->instances.nr || n < 0) { |
be18010e KW |
6140 | pr_warn("Can't get the %dth fd from program %s: only %d instances\n", |
6141 | n, prog->section_name, prog->instances.nr); | |
b580563e WN |
6142 | return -EINVAL; |
6143 | } | |
6144 | ||
6145 | fd = prog->instances.fds[n]; | |
6146 | if (fd < 0) { | |
be18010e KW |
6147 | pr_warn("%dth instance of program '%s' is invalid\n", |
6148 | n, prog->section_name); | |
b580563e WN |
6149 | return -ENOENT; |
6150 | } | |
6151 | ||
6152 | return fd; | |
aa9b1ac3 | 6153 | } |
9d759a9b | 6154 | |
f1eead9e AN |
6155 | enum bpf_prog_type bpf_program__get_type(struct bpf_program *prog) |
6156 | { | |
6157 | return prog->type; | |
6158 | } | |
6159 | ||
dd26b7f5 | 6160 | void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) |
5f44e4c8 WN |
6161 | { |
6162 | prog->type = type; | |
6163 | } | |
6164 | ||
a324aae3 | 6165 | static bool bpf_program__is_type(const struct bpf_program *prog, |
5f44e4c8 WN |
6166 | enum bpf_prog_type type) |
6167 | { | |
6168 | return prog ? (prog->type == type) : false; | |
6169 | } | |
6170 | ||
a324aae3 AN |
6171 | #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ |
6172 | int bpf_program__set_##NAME(struct bpf_program *prog) \ | |
6173 | { \ | |
6174 | if (!prog) \ | |
6175 | return -EINVAL; \ | |
6176 | bpf_program__set_type(prog, TYPE); \ | |
6177 | return 0; \ | |
6178 | } \ | |
6179 | \ | |
6180 | bool bpf_program__is_##NAME(const struct bpf_program *prog) \ | |
6181 | { \ | |
6182 | return bpf_program__is_type(prog, TYPE); \ | |
6183 | } \ | |
ed794073 | 6184 | |
7803ba73 | 6185 | BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); |
ed794073 | 6186 | BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); |
7803ba73 JS |
6187 | BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); |
6188 | BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); | |
ed794073 | 6189 | BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); |
e14c93fd | 6190 | BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); |
7803ba73 JS |
6191 | BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); |
6192 | BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); | |
12a8654b | 6193 | BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING); |
590a0088 | 6194 | BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS); |
2db6eab1 | 6195 | BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); |
5f44e4c8 | 6196 | |
f1eead9e AN |
6197 | enum bpf_attach_type |
6198 | bpf_program__get_expected_attach_type(struct bpf_program *prog) | |
6199 | { | |
6200 | return prog->expected_attach_type; | |
6201 | } | |
6202 | ||
16962b24 JF |
6203 | void bpf_program__set_expected_attach_type(struct bpf_program *prog, |
6204 | enum bpf_attach_type type) | |
d7be143b AI |
6205 | { |
6206 | prog->expected_attach_type = type; | |
6207 | } | |
6208 | ||
f75a697e AS |
6209 | #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, btf, atype) \ |
6210 | { string, sizeof(string) - 1, ptype, eatype, is_attachable, btf, atype } | |
d7be143b | 6211 | |
956b620f | 6212 | /* Programs that can NOT be attached. */ |
f75a697e | 6213 | #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0, 0) |
d7be143b | 6214 | |
956b620f AI |
6215 | /* Programs that can be attached. */ |
6216 | #define BPF_APROG_SEC(string, ptype, atype) \ | |
f75a697e | 6217 | BPF_PROG_SEC_IMPL(string, ptype, 0, 1, 0, atype) |
81efee75 | 6218 | |
956b620f AI |
6219 | /* Programs that must specify expected attach type at load time. */ |
6220 | #define BPF_EAPROG_SEC(string, ptype, eatype) \ | |
f75a697e AS |
6221 | BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, 0, eatype) |
6222 | ||
6223 | /* Programs that use BTF to identify attach point */ | |
12a8654b AS |
6224 | #define BPF_PROG_BTF(string, ptype, eatype) \ |
6225 | BPF_PROG_SEC_IMPL(string, ptype, eatype, 0, 1, 0) | |
956b620f AI |
6226 | |
6227 | /* Programs that can be attached but attach type can't be identified by section | |
6228 | * name. Kept for backward compatibility. | |
6229 | */ | |
6230 | #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) | |
e50b0a6f | 6231 | |
d7a18ea7 AN |
6232 | #define SEC_DEF(sec_pfx, ptype, ...) { \ |
6233 | .sec = sec_pfx, \ | |
6234 | .len = sizeof(sec_pfx) - 1, \ | |
6235 | .prog_type = BPF_PROG_TYPE_##ptype, \ | |
6236 | __VA_ARGS__ \ | |
6237 | } | |
6238 | ||
6239 | struct bpf_sec_def; | |
6240 | ||
6241 | typedef struct bpf_link *(*attach_fn_t)(const struct bpf_sec_def *sec, | |
6242 | struct bpf_program *prog); | |
6243 | ||
6244 | static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, | |
6245 | struct bpf_program *prog); | |
6246 | static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, | |
6247 | struct bpf_program *prog); | |
6248 | static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, | |
6249 | struct bpf_program *prog); | |
6250 | static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, | |
6251 | struct bpf_program *prog); | |
6252 | ||
6253 | struct bpf_sec_def { | |
583c9009 RG |
6254 | const char *sec; |
6255 | size_t len; | |
6256 | enum bpf_prog_type prog_type; | |
d7be143b | 6257 | enum bpf_attach_type expected_attach_type; |
f75a697e AS |
6258 | bool is_attachable; |
6259 | bool is_attach_btf; | |
956b620f | 6260 | enum bpf_attach_type attach_type; |
d7a18ea7 AN |
6261 | attach_fn_t attach_fn; |
6262 | }; | |
6263 | ||
6264 | static const struct bpf_sec_def section_defs[] = { | |
956b620f | 6265 | BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), |
67d69ccd | 6266 | BPF_PROG_SEC("sk_reuseport", BPF_PROG_TYPE_SK_REUSEPORT), |
d7a18ea7 AN |
6267 | SEC_DEF("kprobe/", KPROBE, |
6268 | .attach_fn = attach_kprobe), | |
32dff6db | 6269 | BPF_PROG_SEC("uprobe/", BPF_PROG_TYPE_KPROBE), |
d7a18ea7 AN |
6270 | SEC_DEF("kretprobe/", KPROBE, |
6271 | .attach_fn = attach_kprobe), | |
32dff6db | 6272 | BPF_PROG_SEC("uretprobe/", BPF_PROG_TYPE_KPROBE), |
956b620f AI |
6273 | BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), |
6274 | BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), | |
d7a18ea7 AN |
6275 | SEC_DEF("tracepoint/", TRACEPOINT, |
6276 | .attach_fn = attach_tp), | |
6277 | SEC_DEF("tp/", TRACEPOINT, | |
6278 | .attach_fn = attach_tp), | |
6279 | SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT, | |
6280 | .attach_fn = attach_raw_tp), | |
6281 | SEC_DEF("raw_tp/", RAW_TRACEPOINT, | |
6282 | .attach_fn = attach_raw_tp), | |
6283 | SEC_DEF("tp_btf/", TRACING, | |
6284 | .expected_attach_type = BPF_TRACE_RAW_TP, | |
6285 | .is_attach_btf = true, | |
6286 | .attach_fn = attach_trace), | |
6287 | SEC_DEF("fentry/", TRACING, | |
6288 | .expected_attach_type = BPF_TRACE_FENTRY, | |
6289 | .is_attach_btf = true, | |
6290 | .attach_fn = attach_trace), | |
6291 | SEC_DEF("fexit/", TRACING, | |
6292 | .expected_attach_type = BPF_TRACE_FEXIT, | |
6293 | .is_attach_btf = true, | |
6294 | .attach_fn = attach_trace), | |
2db6eab1 AS |
6295 | SEC_DEF("freplace/", EXT, |
6296 | .is_attach_btf = true, | |
6297 | .attach_fn = attach_trace), | |
956b620f AI |
6298 | BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), |
6299 | BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), | |
6300 | BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), | |
6301 | BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), | |
6302 | BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), | |
6303 | BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), | |
bafa7afe AI |
6304 | BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, |
6305 | BPF_CGROUP_INET_INGRESS), | |
6306 | BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, | |
6307 | BPF_CGROUP_INET_EGRESS), | |
956b620f AI |
6308 | BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), |
6309 | BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, | |
6310 | BPF_CGROUP_INET_SOCK_CREATE), | |
6311 | BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, | |
6312 | BPF_CGROUP_INET4_POST_BIND), | |
6313 | BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, | |
6314 | BPF_CGROUP_INET6_POST_BIND), | |
6315 | BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, | |
6316 | BPF_CGROUP_DEVICE), | |
6317 | BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, | |
6318 | BPF_CGROUP_SOCK_OPS), | |
c6f6851b AI |
6319 | BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, |
6320 | BPF_SK_SKB_STREAM_PARSER), | |
6321 | BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, | |
6322 | BPF_SK_SKB_STREAM_VERDICT), | |
956b620f AI |
6323 | BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), |
6324 | BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, | |
6325 | BPF_SK_MSG_VERDICT), | |
6326 | BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, | |
6327 | BPF_LIRC_MODE2), | |
6328 | BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, | |
6329 | BPF_FLOW_DISSECTOR), | |
6330 | BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
6331 | BPF_CGROUP_INET4_BIND), | |
6332 | BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
6333 | BPF_CGROUP_INET6_BIND), | |
6334 | BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
6335 | BPF_CGROUP_INET4_CONNECT), | |
6336 | BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
6337 | BPF_CGROUP_INET6_CONNECT), | |
6338 | BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
6339 | BPF_CGROUP_UDP4_SENDMSG), | |
6340 | BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
6341 | BPF_CGROUP_UDP6_SENDMSG), | |
9bb59ac1 DB |
6342 | BPF_EAPROG_SEC("cgroup/recvmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, |
6343 | BPF_CGROUP_UDP4_RECVMSG), | |
6344 | BPF_EAPROG_SEC("cgroup/recvmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
6345 | BPF_CGROUP_UDP6_RECVMSG), | |
063cc9f0 AI |
6346 | BPF_EAPROG_SEC("cgroup/sysctl", BPF_PROG_TYPE_CGROUP_SYSCTL, |
6347 | BPF_CGROUP_SYSCTL), | |
4cdbfb59 SF |
6348 | BPF_EAPROG_SEC("cgroup/getsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, |
6349 | BPF_CGROUP_GETSOCKOPT), | |
6350 | BPF_EAPROG_SEC("cgroup/setsockopt", BPF_PROG_TYPE_CGROUP_SOCKOPT, | |
6351 | BPF_CGROUP_SETSOCKOPT), | |
590a0088 | 6352 | BPF_PROG_SEC("struct_ops", BPF_PROG_TYPE_STRUCT_OPS), |
583c9009 | 6353 | }; |
d7be143b | 6354 | |
956b620f | 6355 | #undef BPF_PROG_SEC_IMPL |
583c9009 | 6356 | #undef BPF_PROG_SEC |
956b620f AI |
6357 | #undef BPF_APROG_SEC |
6358 | #undef BPF_EAPROG_SEC | |
6359 | #undef BPF_APROG_COMPAT | |
d7a18ea7 | 6360 | #undef SEC_DEF |
583c9009 | 6361 | |
c76e4c22 TS |
6362 | #define MAX_TYPE_NAME_SIZE 32 |
6363 | ||
d7a18ea7 AN |
6364 | static const struct bpf_sec_def *find_sec_def(const char *sec_name) |
6365 | { | |
6366 | int i, n = ARRAY_SIZE(section_defs); | |
6367 | ||
6368 | for (i = 0; i < n; i++) { | |
6369 | if (strncmp(sec_name, | |
6370 | section_defs[i].sec, section_defs[i].len)) | |
6371 | continue; | |
6372 | return §ion_defs[i]; | |
6373 | } | |
6374 | return NULL; | |
6375 | } | |
6376 | ||
c76e4c22 TS |
6377 | static char *libbpf_get_type_names(bool attach_type) |
6378 | { | |
d7a18ea7 | 6379 | int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; |
c76e4c22 TS |
6380 | char *buf; |
6381 | ||
6382 | buf = malloc(len); | |
6383 | if (!buf) | |
6384 | return NULL; | |
6385 | ||
6386 | buf[0] = '\0'; | |
6387 | /* Forge string buf with all available names */ | |
d7a18ea7 AN |
6388 | for (i = 0; i < ARRAY_SIZE(section_defs); i++) { |
6389 | if (attach_type && !section_defs[i].is_attachable) | |
c76e4c22 TS |
6390 | continue; |
6391 | ||
d7a18ea7 | 6392 | if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { |
c76e4c22 TS |
6393 | free(buf); |
6394 | return NULL; | |
6395 | } | |
6396 | strcat(buf, " "); | |
d7a18ea7 | 6397 | strcat(buf, section_defs[i].sec); |
c76e4c22 TS |
6398 | } |
6399 | ||
6400 | return buf; | |
6401 | } | |
6402 | ||
b60df2a0 JK |
6403 | int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, |
6404 | enum bpf_attach_type *expected_attach_type) | |
583c9009 | 6405 | { |
d7a18ea7 | 6406 | const struct bpf_sec_def *sec_def; |
c76e4c22 | 6407 | char *type_names; |
583c9009 | 6408 | |
b60df2a0 JK |
6409 | if (!name) |
6410 | return -EINVAL; | |
583c9009 | 6411 | |
d7a18ea7 AN |
6412 | sec_def = find_sec_def(name); |
6413 | if (sec_def) { | |
6414 | *prog_type = sec_def->prog_type; | |
6415 | *expected_attach_type = sec_def->expected_attach_type; | |
b60df2a0 JK |
6416 | return 0; |
6417 | } | |
d7a18ea7 | 6418 | |
4a3d6c6a | 6419 | pr_debug("failed to guess program type from ELF section '%s'\n", name); |
c76e4c22 TS |
6420 | type_names = libbpf_get_type_names(false); |
6421 | if (type_names != NULL) { | |
3f519353 | 6422 | pr_debug("supported section(type) names are:%s\n", type_names); |
c76e4c22 TS |
6423 | free(type_names); |
6424 | } | |
6425 | ||
dd4436bb | 6426 | return -ESRCH; |
b60df2a0 | 6427 | } |
583c9009 | 6428 | |
590a0088 MKL |
6429 | static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, |
6430 | size_t offset) | |
6431 | { | |
6432 | struct bpf_map *map; | |
6433 | size_t i; | |
6434 | ||
6435 | for (i = 0; i < obj->nr_maps; i++) { | |
6436 | map = &obj->maps[i]; | |
6437 | if (!bpf_map__is_struct_ops(map)) | |
6438 | continue; | |
6439 | if (map->sec_offset <= offset && | |
6440 | offset - map->sec_offset < map->def.value_size) | |
6441 | return map; | |
6442 | } | |
6443 | ||
6444 | return NULL; | |
6445 | } | |
6446 | ||
6447 | /* Collect the reloc from ELF and populate the st_ops->progs[] */ | |
6448 | static int bpf_object__collect_struct_ops_map_reloc(struct bpf_object *obj, | |
6449 | GElf_Shdr *shdr, | |
6450 | Elf_Data *data) | |
6451 | { | |
6452 | const struct btf_member *member; | |
6453 | struct bpf_struct_ops *st_ops; | |
6454 | struct bpf_program *prog; | |
6455 | unsigned int shdr_idx; | |
6456 | const struct btf *btf; | |
6457 | struct bpf_map *map; | |
6458 | Elf_Data *symbols; | |
6459 | unsigned int moff; | |
6460 | const char *name; | |
1d1a3bcf | 6461 | __u32 member_idx; |
590a0088 MKL |
6462 | GElf_Sym sym; |
6463 | GElf_Rel rel; | |
6464 | int i, nrels; | |
6465 | ||
6466 | symbols = obj->efile.symbols; | |
6467 | btf = obj->btf; | |
6468 | nrels = shdr->sh_size / shdr->sh_entsize; | |
6469 | for (i = 0; i < nrels; i++) { | |
6470 | if (!gelf_getrel(data, i, &rel)) { | |
6471 | pr_warn("struct_ops reloc: failed to get %d reloc\n", i); | |
6472 | return -LIBBPF_ERRNO__FORMAT; | |
6473 | } | |
6474 | ||
6475 | if (!gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym)) { | |
6476 | pr_warn("struct_ops reloc: symbol %zx not found\n", | |
6477 | (size_t)GELF_R_SYM(rel.r_info)); | |
6478 | return -LIBBPF_ERRNO__FORMAT; | |
6479 | } | |
6480 | ||
6481 | name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, | |
6482 | sym.st_name) ? : "<?>"; | |
6483 | map = find_struct_ops_map_by_offset(obj, rel.r_offset); | |
6484 | if (!map) { | |
6485 | pr_warn("struct_ops reloc: cannot find map at rel.r_offset %zu\n", | |
6486 | (size_t)rel.r_offset); | |
6487 | return -EINVAL; | |
6488 | } | |
6489 | ||
6490 | moff = rel.r_offset - map->sec_offset; | |
6491 | shdr_idx = sym.st_shndx; | |
6492 | st_ops = map->st_ops; | |
6493 | pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel.r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", | |
6494 | map->name, | |
6495 | (long long)(rel.r_info >> 32), | |
6496 | (long long)sym.st_value, | |
6497 | shdr_idx, (size_t)rel.r_offset, | |
6498 | map->sec_offset, sym.st_name, name); | |
6499 | ||
6500 | if (shdr_idx >= SHN_LORESERVE) { | |
6501 | pr_warn("struct_ops reloc %s: rel.r_offset %zu shdr_idx %u unsupported non-static function\n", | |
6502 | map->name, (size_t)rel.r_offset, shdr_idx); | |
6503 | return -LIBBPF_ERRNO__RELOC; | |
6504 | } | |
6505 | ||
6506 | member = find_member_by_offset(st_ops->type, moff * 8); | |
6507 | if (!member) { | |
6508 | pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", | |
6509 | map->name, moff); | |
6510 | return -EINVAL; | |
6511 | } | |
6512 | member_idx = member - btf_members(st_ops->type); | |
6513 | name = btf__name_by_offset(btf, member->name_off); | |
6514 | ||
6515 | if (!resolve_func_ptr(btf, member->type, NULL)) { | |
6516 | pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", | |
6517 | map->name, name); | |
6518 | return -EINVAL; | |
6519 | } | |
6520 | ||
6521 | prog = bpf_object__find_prog_by_idx(obj, shdr_idx); | |
6522 | if (!prog) { | |
6523 | pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", | |
6524 | map->name, shdr_idx, name); | |
6525 | return -EINVAL; | |
6526 | } | |
6527 | ||
6528 | if (prog->type == BPF_PROG_TYPE_UNSPEC) { | |
6529 | const struct bpf_sec_def *sec_def; | |
6530 | ||
6531 | sec_def = find_sec_def(prog->section_name); | |
6532 | if (sec_def && | |
6533 | sec_def->prog_type != BPF_PROG_TYPE_STRUCT_OPS) { | |
6534 | /* for pr_warn */ | |
6535 | prog->type = sec_def->prog_type; | |
6536 | goto invalid_prog; | |
6537 | } | |
6538 | ||
6539 | prog->type = BPF_PROG_TYPE_STRUCT_OPS; | |
6540 | prog->attach_btf_id = st_ops->type_id; | |
6541 | prog->expected_attach_type = member_idx; | |
6542 | } else if (prog->type != BPF_PROG_TYPE_STRUCT_OPS || | |
6543 | prog->attach_btf_id != st_ops->type_id || | |
6544 | prog->expected_attach_type != member_idx) { | |
6545 | goto invalid_prog; | |
6546 | } | |
6547 | st_ops->progs[member_idx] = prog; | |
6548 | } | |
6549 | ||
6550 | return 0; | |
6551 | ||
6552 | invalid_prog: | |
6553 | pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", | |
6554 | map->name, prog->name, prog->section_name, prog->type, | |
6555 | prog->attach_btf_id, prog->expected_attach_type, name); | |
6556 | return -EINVAL; | |
6557 | } | |
6558 | ||
a6ed02ca KS |
6559 | #define BTF_TRACE_PREFIX "btf_trace_" |
6560 | #define BTF_MAX_NAME_SIZE 128 | |
6561 | ||
6562 | static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, | |
6563 | const char *name, __u32 kind) | |
6564 | { | |
6565 | char btf_type_name[BTF_MAX_NAME_SIZE]; | |
6566 | int ret; | |
6567 | ||
6568 | ret = snprintf(btf_type_name, sizeof(btf_type_name), | |
6569 | "%s%s", prefix, name); | |
6570 | /* snprintf returns the number of characters written excluding the | |
6571 | * the terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it | |
6572 | * indicates truncation. | |
6573 | */ | |
6574 | if (ret < 0 || ret >= sizeof(btf_type_name)) | |
6575 | return -ENAMETOOLONG; | |
6576 | return btf__find_by_name_kind(btf, btf_type_name, kind); | |
6577 | } | |
6578 | ||
6579 | static inline int __find_vmlinux_btf_id(struct btf *btf, const char *name, | |
6580 | enum bpf_attach_type attach_type) | |
6581 | { | |
6582 | int err; | |
6583 | ||
6584 | if (attach_type == BPF_TRACE_RAW_TP) | |
6585 | err = find_btf_by_prefix_kind(btf, BTF_TRACE_PREFIX, name, | |
6586 | BTF_KIND_TYPEDEF); | |
6587 | else | |
6588 | err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); | |
6589 | ||
ff26ce5c EC |
6590 | if (err <= 0) |
6591 | pr_warn("%s is not found in vmlinux BTF\n", name); | |
6592 | ||
a6ed02ca KS |
6593 | return err; |
6594 | } | |
6595 | ||
b8c54ea4 AS |
6596 | int libbpf_find_vmlinux_btf_id(const char *name, |
6597 | enum bpf_attach_type attach_type) | |
12a8654b | 6598 | { |
a6ed02ca | 6599 | struct btf *btf; |
12a8654b | 6600 | |
a6ed02ca | 6601 | btf = libbpf_find_kernel_btf(); |
12a8654b AS |
6602 | if (IS_ERR(btf)) { |
6603 | pr_warn("vmlinux BTF is not found\n"); | |
6604 | return -EINVAL; | |
6605 | } | |
6606 | ||
a6ed02ca | 6607 | return __find_vmlinux_btf_id(btf, name, attach_type); |
b8c54ea4 AS |
6608 | } |
6609 | ||
e7bf94db AS |
6610 | static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) |
6611 | { | |
6612 | struct bpf_prog_info_linear *info_linear; | |
6613 | struct bpf_prog_info *info; | |
6614 | struct btf *btf = NULL; | |
6615 | int err = -EINVAL; | |
6616 | ||
6617 | info_linear = bpf_program__get_prog_info_linear(attach_prog_fd, 0); | |
6618 | if (IS_ERR_OR_NULL(info_linear)) { | |
6619 | pr_warn("failed get_prog_info_linear for FD %d\n", | |
6620 | attach_prog_fd); | |
6621 | return -EINVAL; | |
6622 | } | |
6623 | info = &info_linear->info; | |
6624 | if (!info->btf_id) { | |
6625 | pr_warn("The target program doesn't have BTF\n"); | |
6626 | goto out; | |
6627 | } | |
6628 | if (btf__get_from_id(info->btf_id, &btf)) { | |
6629 | pr_warn("Failed to get BTF of the program\n"); | |
6630 | goto out; | |
6631 | } | |
6632 | err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); | |
6633 | btf__free(btf); | |
6634 | if (err <= 0) { | |
6635 | pr_warn("%s is not found in prog's BTF\n", name); | |
6636 | goto out; | |
6637 | } | |
6638 | out: | |
6639 | free(info_linear); | |
6640 | return err; | |
6641 | } | |
6642 | ||
a6ed02ca | 6643 | static int libbpf_find_attach_btf_id(struct bpf_program *prog) |
b8c54ea4 | 6644 | { |
a6ed02ca KS |
6645 | enum bpf_attach_type attach_type = prog->expected_attach_type; |
6646 | __u32 attach_prog_fd = prog->attach_prog_fd; | |
6647 | const char *name = prog->section_name; | |
b8c54ea4 AS |
6648 | int i, err; |
6649 | ||
12a8654b | 6650 | if (!name) |
b8c54ea4 | 6651 | return -EINVAL; |
12a8654b | 6652 | |
d7a18ea7 AN |
6653 | for (i = 0; i < ARRAY_SIZE(section_defs); i++) { |
6654 | if (!section_defs[i].is_attach_btf) | |
12a8654b | 6655 | continue; |
d7a18ea7 | 6656 | if (strncmp(name, section_defs[i].sec, section_defs[i].len)) |
12a8654b | 6657 | continue; |
e7bf94db | 6658 | if (attach_prog_fd) |
d7a18ea7 | 6659 | err = libbpf_find_prog_btf_id(name + section_defs[i].len, |
e7bf94db AS |
6660 | attach_prog_fd); |
6661 | else | |
a6ed02ca KS |
6662 | err = __find_vmlinux_btf_id(prog->obj->btf_vmlinux, |
6663 | name + section_defs[i].len, | |
6664 | attach_type); | |
b8c54ea4 | 6665 | return err; |
12a8654b AS |
6666 | } |
6667 | pr_warn("failed to identify btf_id based on ELF section name '%s'\n", name); | |
b8c54ea4 | 6668 | return -ESRCH; |
12a8654b AS |
6669 | } |
6670 | ||
956b620f AI |
6671 | int libbpf_attach_type_by_name(const char *name, |
6672 | enum bpf_attach_type *attach_type) | |
6673 | { | |
c76e4c22 | 6674 | char *type_names; |
956b620f AI |
6675 | int i; |
6676 | ||
6677 | if (!name) | |
6678 | return -EINVAL; | |
6679 | ||
d7a18ea7 AN |
6680 | for (i = 0; i < ARRAY_SIZE(section_defs); i++) { |
6681 | if (strncmp(name, section_defs[i].sec, section_defs[i].len)) | |
956b620f | 6682 | continue; |
d7a18ea7 | 6683 | if (!section_defs[i].is_attachable) |
956b620f | 6684 | return -EINVAL; |
d7a18ea7 | 6685 | *attach_type = section_defs[i].attach_type; |
956b620f AI |
6686 | return 0; |
6687 | } | |
4a3d6c6a | 6688 | pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); |
c76e4c22 TS |
6689 | type_names = libbpf_get_type_names(true); |
6690 | if (type_names != NULL) { | |
4a3d6c6a | 6691 | pr_debug("attachable section(type) names are:%s\n", type_names); |
c76e4c22 TS |
6692 | free(type_names); |
6693 | } | |
6694 | ||
956b620f AI |
6695 | return -EINVAL; |
6696 | } | |
6697 | ||
a324aae3 | 6698 | int bpf_map__fd(const struct bpf_map *map) |
9d759a9b | 6699 | { |
6e009e65 | 6700 | return map ? map->fd : -EINVAL; |
9d759a9b WN |
6701 | } |
6702 | ||
a324aae3 | 6703 | const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) |
9d759a9b | 6704 | { |
53897a78 | 6705 | return map ? &map->def : ERR_PTR(-EINVAL); |
9d759a9b WN |
6706 | } |
6707 | ||
a324aae3 | 6708 | const char *bpf_map__name(const struct bpf_map *map) |
561bbcca | 6709 | { |
009ad5d5 | 6710 | return map ? map->name : NULL; |
561bbcca WN |
6711 | } |
6712 | ||
5b891af7 | 6713 | __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) |
8a138aed | 6714 | { |
61746dbe | 6715 | return map ? map->btf_key_type_id : 0; |
8a138aed MKL |
6716 | } |
6717 | ||
5b891af7 | 6718 | __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) |
8a138aed | 6719 | { |
61746dbe | 6720 | return map ? map->btf_value_type_id : 0; |
8a138aed MKL |
6721 | } |
6722 | ||
edb13ed4 ACM |
6723 | int bpf_map__set_priv(struct bpf_map *map, void *priv, |
6724 | bpf_map_clear_priv_t clear_priv) | |
9d759a9b WN |
6725 | { |
6726 | if (!map) | |
6727 | return -EINVAL; | |
6728 | ||
6729 | if (map->priv) { | |
6730 | if (map->clear_priv) | |
6731 | map->clear_priv(map, map->priv); | |
6732 | } | |
6733 | ||
6734 | map->priv = priv; | |
6735 | map->clear_priv = clear_priv; | |
6736 | return 0; | |
6737 | } | |
6738 | ||
a324aae3 | 6739 | void *bpf_map__priv(const struct bpf_map *map) |
9d759a9b | 6740 | { |
b4cbfa56 | 6741 | return map ? map->priv : ERR_PTR(-EINVAL); |
9d759a9b WN |
6742 | } |
6743 | ||
a324aae3 | 6744 | bool bpf_map__is_offload_neutral(const struct bpf_map *map) |
f83fb22c JK |
6745 | { |
6746 | return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; | |
6747 | } | |
6748 | ||
a324aae3 | 6749 | bool bpf_map__is_internal(const struct bpf_map *map) |
d859900c DB |
6750 | { |
6751 | return map->libbpf_type != LIBBPF_MAP_UNSPEC; | |
6752 | } | |
6753 | ||
9aba3613 JK |
6754 | void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) |
6755 | { | |
6756 | map->map_ifindex = ifindex; | |
6757 | } | |
6758 | ||
addb9fc9 NS |
6759 | int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) |
6760 | { | |
6761 | if (!bpf_map_type__is_map_in_map(map->def.type)) { | |
be18010e | 6762 | pr_warn("error: unsupported map type\n"); |
addb9fc9 NS |
6763 | return -EINVAL; |
6764 | } | |
6765 | if (map->inner_map_fd != -1) { | |
be18010e | 6766 | pr_warn("error: inner_map_fd already specified\n"); |
addb9fc9 NS |
6767 | return -EINVAL; |
6768 | } | |
6769 | map->inner_map_fd = fd; | |
6770 | return 0; | |
6771 | } | |
6772 | ||
0c19a9fb | 6773 | static struct bpf_map * |
a324aae3 | 6774 | __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) |
9d759a9b | 6775 | { |
0c19a9fb | 6776 | ssize_t idx; |
9d759a9b WN |
6777 | struct bpf_map *s, *e; |
6778 | ||
6779 | if (!obj || !obj->maps) | |
6780 | return NULL; | |
6781 | ||
6782 | s = obj->maps; | |
6783 | e = obj->maps + obj->nr_maps; | |
6784 | ||
0c19a9fb | 6785 | if ((m < s) || (m >= e)) { |
be18010e KW |
6786 | pr_warn("error in %s: map handler doesn't belong to object\n", |
6787 | __func__); | |
9d759a9b WN |
6788 | return NULL; |
6789 | } | |
6790 | ||
0c19a9fb SF |
6791 | idx = (m - obj->maps) + i; |
6792 | if (idx >= obj->nr_maps || idx < 0) | |
9d759a9b WN |
6793 | return NULL; |
6794 | return &obj->maps[idx]; | |
6795 | } | |
561bbcca | 6796 | |
0c19a9fb | 6797 | struct bpf_map * |
a324aae3 | 6798 | bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) |
0c19a9fb SF |
6799 | { |
6800 | if (prev == NULL) | |
6801 | return obj->maps; | |
6802 | ||
6803 | return __bpf_map__iter(prev, obj, 1); | |
6804 | } | |
6805 | ||
6806 | struct bpf_map * | |
a324aae3 | 6807 | bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) |
0c19a9fb SF |
6808 | { |
6809 | if (next == NULL) { | |
6810 | if (!obj->nr_maps) | |
6811 | return NULL; | |
6812 | return obj->maps + obj->nr_maps - 1; | |
6813 | } | |
6814 | ||
6815 | return __bpf_map__iter(next, obj, -1); | |
6816 | } | |
6817 | ||
561bbcca | 6818 | struct bpf_map * |
a324aae3 | 6819 | bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) |
561bbcca WN |
6820 | { |
6821 | struct bpf_map *pos; | |
6822 | ||
f74a53d9 | 6823 | bpf_object__for_each_map(pos, obj) { |
973170e6 | 6824 | if (pos->name && !strcmp(pos->name, name)) |
561bbcca WN |
6825 | return pos; |
6826 | } | |
6827 | return NULL; | |
6828 | } | |
5a6acad1 | 6829 | |
f3cea32d | 6830 | int |
a324aae3 | 6831 | bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) |
f3cea32d MF |
6832 | { |
6833 | return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); | |
6834 | } | |
6835 | ||
5a6acad1 WN |
6836 | struct bpf_map * |
6837 | bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) | |
6838 | { | |
db48814b | 6839 | return ERR_PTR(-ENOTSUP); |
5a6acad1 | 6840 | } |
e28ff1a8 JS |
6841 | |
6842 | long libbpf_get_error(const void *ptr) | |
6843 | { | |
d98363b5 | 6844 | return PTR_ERR_OR_ZERO(ptr); |
e28ff1a8 | 6845 | } |
6f6d33f3 JF |
6846 | |
6847 | int bpf_prog_load(const char *file, enum bpf_prog_type type, | |
6848 | struct bpf_object **pobj, int *prog_fd) | |
d7be143b AI |
6849 | { |
6850 | struct bpf_prog_load_attr attr; | |
6851 | ||
6852 | memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); | |
6853 | attr.file = file; | |
6854 | attr.prog_type = type; | |
6855 | attr.expected_attach_type = 0; | |
6856 | ||
6857 | return bpf_prog_load_xattr(&attr, pobj, prog_fd); | |
6858 | } | |
6859 | ||
6860 | int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, | |
6861 | struct bpf_object **pobj, int *prog_fd) | |
6f6d33f3 | 6862 | { |
33bae185 | 6863 | struct bpf_object_open_attr open_attr = {}; |
48cca7e4 | 6864 | struct bpf_program *prog, *first_prog = NULL; |
6f6d33f3 | 6865 | struct bpf_object *obj; |
f0307a7e | 6866 | struct bpf_map *map; |
6f6d33f3 JF |
6867 | int err; |
6868 | ||
d7be143b AI |
6869 | if (!attr) |
6870 | return -EINVAL; | |
17387dd5 JK |
6871 | if (!attr->file) |
6872 | return -EINVAL; | |
d7be143b | 6873 | |
33bae185 LY |
6874 | open_attr.file = attr->file; |
6875 | open_attr.prog_type = attr->prog_type; | |
6876 | ||
07f2d4ea | 6877 | obj = bpf_object__open_xattr(&open_attr); |
3597683c | 6878 | if (IS_ERR_OR_NULL(obj)) |
6f6d33f3 JF |
6879 | return -ENOENT; |
6880 | ||
48cca7e4 | 6881 | bpf_object__for_each_program(prog, obj) { |
dd4436bb | 6882 | enum bpf_attach_type attach_type = attr->expected_attach_type; |
48cca7e4 | 6883 | /* |
dd4436bb AN |
6884 | * to preserve backwards compatibility, bpf_prog_load treats |
6885 | * attr->prog_type, if specified, as an override to whatever | |
6886 | * bpf_object__open guessed | |
48cca7e4 | 6887 | */ |
dd4436bb AN |
6888 | if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) { |
6889 | bpf_program__set_type(prog, attr->prog_type); | |
6890 | bpf_program__set_expected_attach_type(prog, | |
6891 | attach_type); | |
6892 | } | |
6893 | if (bpf_program__get_type(prog) == BPF_PROG_TYPE_UNSPEC) { | |
6894 | /* | |
6895 | * we haven't guessed from section name and user | |
6896 | * didn't provide a fallback type, too bad... | |
6897 | */ | |
6898 | bpf_object__close(obj); | |
6899 | return -EINVAL; | |
583c9009 | 6900 | } |
48cca7e4 | 6901 | |
dd4436bb | 6902 | prog->prog_ifindex = attr->ifindex; |
da11b417 | 6903 | prog->log_level = attr->log_level; |
04656198 | 6904 | prog->prog_flags = attr->prog_flags; |
69495d2a | 6905 | if (!first_prog) |
48cca7e4 AS |
6906 | first_prog = prog; |
6907 | } | |
6908 | ||
f74a53d9 | 6909 | bpf_object__for_each_map(map, obj) { |
f83fb22c JK |
6910 | if (!bpf_map__is_offload_neutral(map)) |
6911 | map->map_ifindex = attr->ifindex; | |
f0307a7e DB |
6912 | } |
6913 | ||
48cca7e4 | 6914 | if (!first_prog) { |
be18010e | 6915 | pr_warn("object file doesn't contain bpf program\n"); |
48cca7e4 AS |
6916 | bpf_object__close(obj); |
6917 | return -ENOENT; | |
583c9009 RG |
6918 | } |
6919 | ||
6f6d33f3 JF |
6920 | err = bpf_object__load(obj); |
6921 | if (err) { | |
6922 | bpf_object__close(obj); | |
6923 | return -EINVAL; | |
6924 | } | |
6925 | ||
6926 | *pobj = obj; | |
48cca7e4 | 6927 | *prog_fd = bpf_program__fd(first_prog); |
6f6d33f3 JF |
6928 | return 0; |
6929 | } | |
d0cabbb0 | 6930 | |
1c2e9efc | 6931 | struct bpf_link { |
d6958706 | 6932 | int (*detach)(struct bpf_link *link); |
1c2e9efc | 6933 | int (*destroy)(struct bpf_link *link); |
c016b68e AN |
6934 | char *pin_path; /* NULL, if not pinned */ |
6935 | int fd; /* hook FD, -1 if not applicable */ | |
d6958706 | 6936 | bool disconnected; |
1c2e9efc AN |
6937 | }; |
6938 | ||
d6958706 AN |
6939 | /* Release "ownership" of underlying BPF resource (typically, BPF program |
6940 | * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected | |
6941 | * link, when destructed through bpf_link__destroy() call won't attempt to | |
6942 | * detach/unregisted that BPF resource. This is useful in situations where, | |
6943 | * say, attached BPF program has to outlive userspace program that attached it | |
6944 | * in the system. Depending on type of BPF program, though, there might be | |
6945 | * additional steps (like pinning BPF program in BPF FS) necessary to ensure | |
6946 | * exit of userspace program doesn't trigger automatic detachment and clean up | |
6947 | * inside the kernel. | |
6948 | */ | |
6949 | void bpf_link__disconnect(struct bpf_link *link) | |
6950 | { | |
6951 | link->disconnected = true; | |
6952 | } | |
6953 | ||
1c2e9efc AN |
6954 | int bpf_link__destroy(struct bpf_link *link) |
6955 | { | |
d6958706 | 6956 | int err = 0; |
1c2e9efc AN |
6957 | |
6958 | if (!link) | |
6959 | return 0; | |
6960 | ||
d6958706 AN |
6961 | if (!link->disconnected && link->detach) |
6962 | err = link->detach(link); | |
6963 | if (link->destroy) | |
6964 | link->destroy(link); | |
c016b68e AN |
6965 | if (link->pin_path) |
6966 | free(link->pin_path); | |
1c2e9efc AN |
6967 | free(link); |
6968 | ||
6969 | return err; | |
6970 | } | |
6971 | ||
c016b68e AN |
6972 | int bpf_link__fd(const struct bpf_link *link) |
6973 | { | |
6974 | return link->fd; | |
6975 | } | |
6976 | ||
6977 | const char *bpf_link__pin_path(const struct bpf_link *link) | |
6978 | { | |
6979 | return link->pin_path; | |
6980 | } | |
6981 | ||
6982 | static int bpf_link__detach_fd(struct bpf_link *link) | |
6983 | { | |
6984 | return close(link->fd); | |
6985 | } | |
6986 | ||
6987 | struct bpf_link *bpf_link__open(const char *path) | |
6988 | { | |
6989 | struct bpf_link *link; | |
6990 | int fd; | |
6991 | ||
6992 | fd = bpf_obj_get(path); | |
6993 | if (fd < 0) { | |
6994 | fd = -errno; | |
6995 | pr_warn("failed to open link at %s: %d\n", path, fd); | |
6996 | return ERR_PTR(fd); | |
6997 | } | |
6998 | ||
6999 | link = calloc(1, sizeof(*link)); | |
7000 | if (!link) { | |
7001 | close(fd); | |
7002 | return ERR_PTR(-ENOMEM); | |
7003 | } | |
7004 | link->detach = &bpf_link__detach_fd; | |
7005 | link->fd = fd; | |
7006 | ||
7007 | link->pin_path = strdup(path); | |
7008 | if (!link->pin_path) { | |
7009 | bpf_link__destroy(link); | |
7010 | return ERR_PTR(-ENOMEM); | |
7011 | } | |
7012 | ||
7013 | return link; | |
7014 | } | |
7015 | ||
7016 | int bpf_link__pin(struct bpf_link *link, const char *path) | |
7017 | { | |
7018 | int err; | |
7019 | ||
7020 | if (link->pin_path) | |
7021 | return -EBUSY; | |
7022 | err = make_parent_dir(path); | |
7023 | if (err) | |
7024 | return err; | |
7025 | err = check_path(path); | |
7026 | if (err) | |
7027 | return err; | |
7028 | ||
7029 | link->pin_path = strdup(path); | |
7030 | if (!link->pin_path) | |
7031 | return -ENOMEM; | |
7032 | ||
7033 | if (bpf_obj_pin(link->fd, link->pin_path)) { | |
7034 | err = -errno; | |
7035 | zfree(&link->pin_path); | |
7036 | return err; | |
7037 | } | |
7038 | ||
7039 | pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); | |
7040 | return 0; | |
7041 | } | |
7042 | ||
7043 | int bpf_link__unpin(struct bpf_link *link) | |
7044 | { | |
7045 | int err; | |
7046 | ||
7047 | if (!link->pin_path) | |
7048 | return -EINVAL; | |
7049 | ||
7050 | err = unlink(link->pin_path); | |
7051 | if (err != 0) | |
7052 | return -errno; | |
7053 | ||
7054 | pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); | |
7055 | zfree(&link->pin_path); | |
7056 | return 0; | |
7057 | } | |
63f2f5ee | 7058 | |
d6958706 | 7059 | static int bpf_link__detach_perf_event(struct bpf_link *link) |
63f2f5ee | 7060 | { |
63f2f5ee AN |
7061 | int err; |
7062 | ||
c016b68e | 7063 | err = ioctl(link->fd, PERF_EVENT_IOC_DISABLE, 0); |
63f2f5ee AN |
7064 | if (err) |
7065 | err = -errno; | |
7066 | ||
c016b68e | 7067 | close(link->fd); |
63f2f5ee AN |
7068 | return err; |
7069 | } | |
7070 | ||
7071 | struct bpf_link *bpf_program__attach_perf_event(struct bpf_program *prog, | |
7072 | int pfd) | |
7073 | { | |
7074 | char errmsg[STRERR_BUFSIZE]; | |
c016b68e | 7075 | struct bpf_link *link; |
63f2f5ee AN |
7076 | int prog_fd, err; |
7077 | ||
7078 | if (pfd < 0) { | |
be18010e KW |
7079 | pr_warn("program '%s': invalid perf event FD %d\n", |
7080 | bpf_program__title(prog, false), pfd); | |
63f2f5ee AN |
7081 | return ERR_PTR(-EINVAL); |
7082 | } | |
7083 | prog_fd = bpf_program__fd(prog); | |
7084 | if (prog_fd < 0) { | |
be18010e KW |
7085 | pr_warn("program '%s': can't attach BPF program w/o FD (did you load it?)\n", |
7086 | bpf_program__title(prog, false)); | |
63f2f5ee AN |
7087 | return ERR_PTR(-EINVAL); |
7088 | } | |
7089 | ||
d6958706 | 7090 | link = calloc(1, sizeof(*link)); |
63f2f5ee AN |
7091 | if (!link) |
7092 | return ERR_PTR(-ENOMEM); | |
c016b68e | 7093 | link->detach = &bpf_link__detach_perf_event; |
63f2f5ee AN |
7094 | link->fd = pfd; |
7095 | ||
7096 | if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { | |
7097 | err = -errno; | |
7098 | free(link); | |
be18010e KW |
7099 | pr_warn("program '%s': failed to attach to pfd %d: %s\n", |
7100 | bpf_program__title(prog, false), pfd, | |
63f2f5ee AN |
7101 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); |
7102 | return ERR_PTR(err); | |
7103 | } | |
7104 | if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { | |
7105 | err = -errno; | |
7106 | free(link); | |
be18010e KW |
7107 | pr_warn("program '%s': failed to enable pfd %d: %s\n", |
7108 | bpf_program__title(prog, false), pfd, | |
63f2f5ee AN |
7109 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); |
7110 | return ERR_PTR(err); | |
7111 | } | |
c016b68e | 7112 | return link; |
63f2f5ee AN |
7113 | } |
7114 | ||
b2650027 AN |
7115 | /* |
7116 | * this function is expected to parse integer in the range of [0, 2^31-1] from | |
7117 | * given file using scanf format string fmt. If actual parsed value is | |
7118 | * negative, the result might be indistinguishable from error | |
7119 | */ | |
7120 | static int parse_uint_from_file(const char *file, const char *fmt) | |
7121 | { | |
7122 | char buf[STRERR_BUFSIZE]; | |
7123 | int err, ret; | |
7124 | FILE *f; | |
7125 | ||
7126 | f = fopen(file, "r"); | |
7127 | if (!f) { | |
7128 | err = -errno; | |
7129 | pr_debug("failed to open '%s': %s\n", file, | |
7130 | libbpf_strerror_r(err, buf, sizeof(buf))); | |
7131 | return err; | |
7132 | } | |
7133 | err = fscanf(f, fmt, &ret); | |
7134 | if (err != 1) { | |
7135 | err = err == EOF ? -EIO : -errno; | |
7136 | pr_debug("failed to parse '%s': %s\n", file, | |
7137 | libbpf_strerror_r(err, buf, sizeof(buf))); | |
7138 | fclose(f); | |
7139 | return err; | |
7140 | } | |
7141 | fclose(f); | |
7142 | return ret; | |
7143 | } | |
7144 | ||
7145 | static int determine_kprobe_perf_type(void) | |
7146 | { | |
7147 | const char *file = "/sys/bus/event_source/devices/kprobe/type"; | |
7148 | ||
7149 | return parse_uint_from_file(file, "%d\n"); | |
7150 | } | |
7151 | ||
7152 | static int determine_uprobe_perf_type(void) | |
7153 | { | |
7154 | const char *file = "/sys/bus/event_source/devices/uprobe/type"; | |
7155 | ||
7156 | return parse_uint_from_file(file, "%d\n"); | |
7157 | } | |
7158 | ||
7159 | static int determine_kprobe_retprobe_bit(void) | |
7160 | { | |
7161 | const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; | |
7162 | ||
7163 | return parse_uint_from_file(file, "config:%d\n"); | |
7164 | } | |
7165 | ||
7166 | static int determine_uprobe_retprobe_bit(void) | |
7167 | { | |
7168 | const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; | |
7169 | ||
7170 | return parse_uint_from_file(file, "config:%d\n"); | |
7171 | } | |
7172 | ||
7173 | static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, | |
7174 | uint64_t offset, int pid) | |
7175 | { | |
7176 | struct perf_event_attr attr = {}; | |
7177 | char errmsg[STRERR_BUFSIZE]; | |
7178 | int type, pfd, err; | |
7179 | ||
7180 | type = uprobe ? determine_uprobe_perf_type() | |
7181 | : determine_kprobe_perf_type(); | |
7182 | if (type < 0) { | |
be18010e KW |
7183 | pr_warn("failed to determine %s perf type: %s\n", |
7184 | uprobe ? "uprobe" : "kprobe", | |
7185 | libbpf_strerror_r(type, errmsg, sizeof(errmsg))); | |
b2650027 AN |
7186 | return type; |
7187 | } | |
7188 | if (retprobe) { | |
7189 | int bit = uprobe ? determine_uprobe_retprobe_bit() | |
7190 | : determine_kprobe_retprobe_bit(); | |
7191 | ||
7192 | if (bit < 0) { | |
be18010e KW |
7193 | pr_warn("failed to determine %s retprobe bit: %s\n", |
7194 | uprobe ? "uprobe" : "kprobe", | |
7195 | libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); | |
b2650027 AN |
7196 | return bit; |
7197 | } | |
7198 | attr.config |= 1 << bit; | |
7199 | } | |
7200 | attr.size = sizeof(attr); | |
7201 | attr.type = type; | |
36db2a94 AN |
7202 | attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ |
7203 | attr.config2 = offset; /* kprobe_addr or probe_offset */ | |
b2650027 AN |
7204 | |
7205 | /* pid filter is meaningful only for uprobes */ | |
7206 | pfd = syscall(__NR_perf_event_open, &attr, | |
7207 | pid < 0 ? -1 : pid /* pid */, | |
7208 | pid == -1 ? 0 : -1 /* cpu */, | |
7209 | -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); | |
7210 | if (pfd < 0) { | |
7211 | err = -errno; | |
be18010e KW |
7212 | pr_warn("%s perf_event_open() failed: %s\n", |
7213 | uprobe ? "uprobe" : "kprobe", | |
7214 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
b2650027 AN |
7215 | return err; |
7216 | } | |
7217 | return pfd; | |
7218 | } | |
7219 | ||
7220 | struct bpf_link *bpf_program__attach_kprobe(struct bpf_program *prog, | |
7221 | bool retprobe, | |
7222 | const char *func_name) | |
7223 | { | |
7224 | char errmsg[STRERR_BUFSIZE]; | |
7225 | struct bpf_link *link; | |
7226 | int pfd, err; | |
7227 | ||
7228 | pfd = perf_event_open_probe(false /* uprobe */, retprobe, func_name, | |
7229 | 0 /* offset */, -1 /* pid */); | |
7230 | if (pfd < 0) { | |
be18010e KW |
7231 | pr_warn("program '%s': failed to create %s '%s' perf event: %s\n", |
7232 | bpf_program__title(prog, false), | |
7233 | retprobe ? "kretprobe" : "kprobe", func_name, | |
7234 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
b2650027 AN |
7235 | return ERR_PTR(pfd); |
7236 | } | |
7237 | link = bpf_program__attach_perf_event(prog, pfd); | |
7238 | if (IS_ERR(link)) { | |
7239 | close(pfd); | |
7240 | err = PTR_ERR(link); | |
be18010e KW |
7241 | pr_warn("program '%s': failed to attach to %s '%s': %s\n", |
7242 | bpf_program__title(prog, false), | |
7243 | retprobe ? "kretprobe" : "kprobe", func_name, | |
7244 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
b2650027 AN |
7245 | return link; |
7246 | } | |
7247 | return link; | |
7248 | } | |
7249 | ||
d7a18ea7 AN |
7250 | static struct bpf_link *attach_kprobe(const struct bpf_sec_def *sec, |
7251 | struct bpf_program *prog) | |
7252 | { | |
7253 | const char *func_name; | |
7254 | bool retprobe; | |
7255 | ||
7256 | func_name = bpf_program__title(prog, false) + sec->len; | |
7257 | retprobe = strcmp(sec->sec, "kretprobe/") == 0; | |
7258 | ||
7259 | return bpf_program__attach_kprobe(prog, retprobe, func_name); | |
7260 | } | |
7261 | ||
b2650027 AN |
7262 | struct bpf_link *bpf_program__attach_uprobe(struct bpf_program *prog, |
7263 | bool retprobe, pid_t pid, | |
7264 | const char *binary_path, | |
7265 | size_t func_offset) | |
7266 | { | |
7267 | char errmsg[STRERR_BUFSIZE]; | |
7268 | struct bpf_link *link; | |
7269 | int pfd, err; | |
7270 | ||
7271 | pfd = perf_event_open_probe(true /* uprobe */, retprobe, | |
7272 | binary_path, func_offset, pid); | |
7273 | if (pfd < 0) { | |
be18010e KW |
7274 | pr_warn("program '%s': failed to create %s '%s:0x%zx' perf event: %s\n", |
7275 | bpf_program__title(prog, false), | |
7276 | retprobe ? "uretprobe" : "uprobe", | |
7277 | binary_path, func_offset, | |
7278 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
b2650027 AN |
7279 | return ERR_PTR(pfd); |
7280 | } | |
7281 | link = bpf_program__attach_perf_event(prog, pfd); | |
7282 | if (IS_ERR(link)) { | |
7283 | close(pfd); | |
7284 | err = PTR_ERR(link); | |
be18010e KW |
7285 | pr_warn("program '%s': failed to attach to %s '%s:0x%zx': %s\n", |
7286 | bpf_program__title(prog, false), | |
7287 | retprobe ? "uretprobe" : "uprobe", | |
7288 | binary_path, func_offset, | |
7289 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
b2650027 AN |
7290 | return link; |
7291 | } | |
7292 | return link; | |
7293 | } | |
7294 | ||
f6de59c1 AN |
7295 | static int determine_tracepoint_id(const char *tp_category, |
7296 | const char *tp_name) | |
7297 | { | |
7298 | char file[PATH_MAX]; | |
7299 | int ret; | |
7300 | ||
7301 | ret = snprintf(file, sizeof(file), | |
7302 | "/sys/kernel/debug/tracing/events/%s/%s/id", | |
7303 | tp_category, tp_name); | |
7304 | if (ret < 0) | |
7305 | return -errno; | |
7306 | if (ret >= sizeof(file)) { | |
7307 | pr_debug("tracepoint %s/%s path is too long\n", | |
7308 | tp_category, tp_name); | |
7309 | return -E2BIG; | |
7310 | } | |
7311 | return parse_uint_from_file(file, "%d\n"); | |
7312 | } | |
7313 | ||
7314 | static int perf_event_open_tracepoint(const char *tp_category, | |
7315 | const char *tp_name) | |
7316 | { | |
7317 | struct perf_event_attr attr = {}; | |
7318 | char errmsg[STRERR_BUFSIZE]; | |
7319 | int tp_id, pfd, err; | |
7320 | ||
7321 | tp_id = determine_tracepoint_id(tp_category, tp_name); | |
7322 | if (tp_id < 0) { | |
be18010e KW |
7323 | pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", |
7324 | tp_category, tp_name, | |
7325 | libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); | |
f6de59c1 AN |
7326 | return tp_id; |
7327 | } | |
7328 | ||
7329 | attr.type = PERF_TYPE_TRACEPOINT; | |
7330 | attr.size = sizeof(attr); | |
7331 | attr.config = tp_id; | |
7332 | ||
7333 | pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, | |
7334 | -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); | |
7335 | if (pfd < 0) { | |
7336 | err = -errno; | |
be18010e KW |
7337 | pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", |
7338 | tp_category, tp_name, | |
7339 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
f6de59c1 AN |
7340 | return err; |
7341 | } | |
7342 | return pfd; | |
7343 | } | |
7344 | ||
7345 | struct bpf_link *bpf_program__attach_tracepoint(struct bpf_program *prog, | |
7346 | const char *tp_category, | |
7347 | const char *tp_name) | |
7348 | { | |
7349 | char errmsg[STRERR_BUFSIZE]; | |
7350 | struct bpf_link *link; | |
7351 | int pfd, err; | |
7352 | ||
7353 | pfd = perf_event_open_tracepoint(tp_category, tp_name); | |
7354 | if (pfd < 0) { | |
be18010e KW |
7355 | pr_warn("program '%s': failed to create tracepoint '%s/%s' perf event: %s\n", |
7356 | bpf_program__title(prog, false), | |
7357 | tp_category, tp_name, | |
7358 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
f6de59c1 AN |
7359 | return ERR_PTR(pfd); |
7360 | } | |
7361 | link = bpf_program__attach_perf_event(prog, pfd); | |
7362 | if (IS_ERR(link)) { | |
7363 | close(pfd); | |
7364 | err = PTR_ERR(link); | |
be18010e KW |
7365 | pr_warn("program '%s': failed to attach to tracepoint '%s/%s': %s\n", |
7366 | bpf_program__title(prog, false), | |
7367 | tp_category, tp_name, | |
7368 | libbpf_strerror_r(err, errmsg, sizeof(errmsg))); | |
f6de59c1 AN |
7369 | return link; |
7370 | } | |
7371 | return link; | |
7372 | } | |
7373 | ||
d7a18ea7 AN |
7374 | static struct bpf_link *attach_tp(const struct bpf_sec_def *sec, |
7375 | struct bpf_program *prog) | |
7376 | { | |
7377 | char *sec_name, *tp_cat, *tp_name; | |
7378 | struct bpf_link *link; | |
7379 | ||
7380 | sec_name = strdup(bpf_program__title(prog, false)); | |
7381 | if (!sec_name) | |
7382 | return ERR_PTR(-ENOMEM); | |
7383 | ||
7384 | /* extract "tp/<category>/<name>" */ | |
7385 | tp_cat = sec_name + sec->len; | |
7386 | tp_name = strchr(tp_cat, '/'); | |
7387 | if (!tp_name) { | |
7388 | link = ERR_PTR(-EINVAL); | |
7389 | goto out; | |
7390 | } | |
7391 | *tp_name = '\0'; | |
7392 | tp_name++; | |
7393 | ||
7394 | link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); | |
7395 | out: | |
7396 | free(sec_name); | |
7397 | return link; | |
7398 | } | |
7399 | ||
84bf5e1f AN |
7400 | struct bpf_link *bpf_program__attach_raw_tracepoint(struct bpf_program *prog, |
7401 | const char *tp_name) | |
7402 | { | |
7403 | char errmsg[STRERR_BUFSIZE]; | |
c016b68e | 7404 | struct bpf_link *link; |
84bf5e1f AN |
7405 | int prog_fd, pfd; |
7406 | ||
7407 | prog_fd = bpf_program__fd(prog); | |
7408 | if (prog_fd < 0) { | |
be18010e KW |
7409 | pr_warn("program '%s': can't attach before loaded\n", |
7410 | bpf_program__title(prog, false)); | |
84bf5e1f AN |
7411 | return ERR_PTR(-EINVAL); |
7412 | } | |
7413 | ||
d6958706 | 7414 | link = calloc(1, sizeof(*link)); |
84bf5e1f AN |
7415 | if (!link) |
7416 | return ERR_PTR(-ENOMEM); | |
c016b68e | 7417 | link->detach = &bpf_link__detach_fd; |
84bf5e1f AN |
7418 | |
7419 | pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); | |
7420 | if (pfd < 0) { | |
7421 | pfd = -errno; | |
7422 | free(link); | |
be18010e KW |
7423 | pr_warn("program '%s': failed to attach to raw tracepoint '%s': %s\n", |
7424 | bpf_program__title(prog, false), tp_name, | |
7425 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
84bf5e1f AN |
7426 | return ERR_PTR(pfd); |
7427 | } | |
7428 | link->fd = pfd; | |
c016b68e | 7429 | return link; |
84bf5e1f AN |
7430 | } |
7431 | ||
d7a18ea7 AN |
7432 | static struct bpf_link *attach_raw_tp(const struct bpf_sec_def *sec, |
7433 | struct bpf_program *prog) | |
7434 | { | |
7435 | const char *tp_name = bpf_program__title(prog, false) + sec->len; | |
7436 | ||
7437 | return bpf_program__attach_raw_tracepoint(prog, tp_name); | |
7438 | } | |
7439 | ||
b8c54ea4 AS |
7440 | struct bpf_link *bpf_program__attach_trace(struct bpf_program *prog) |
7441 | { | |
7442 | char errmsg[STRERR_BUFSIZE]; | |
c016b68e | 7443 | struct bpf_link *link; |
b8c54ea4 AS |
7444 | int prog_fd, pfd; |
7445 | ||
7446 | prog_fd = bpf_program__fd(prog); | |
7447 | if (prog_fd < 0) { | |
7448 | pr_warn("program '%s': can't attach before loaded\n", | |
7449 | bpf_program__title(prog, false)); | |
7450 | return ERR_PTR(-EINVAL); | |
7451 | } | |
7452 | ||
d6958706 | 7453 | link = calloc(1, sizeof(*link)); |
b8c54ea4 AS |
7454 | if (!link) |
7455 | return ERR_PTR(-ENOMEM); | |
c016b68e | 7456 | link->detach = &bpf_link__detach_fd; |
b8c54ea4 AS |
7457 | |
7458 | pfd = bpf_raw_tracepoint_open(NULL, prog_fd); | |
7459 | if (pfd < 0) { | |
7460 | pfd = -errno; | |
7461 | free(link); | |
7462 | pr_warn("program '%s': failed to attach to trace: %s\n", | |
7463 | bpf_program__title(prog, false), | |
7464 | libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); | |
7465 | return ERR_PTR(pfd); | |
7466 | } | |
7467 | link->fd = pfd; | |
7468 | return (struct bpf_link *)link; | |
7469 | } | |
7470 | ||
d7a18ea7 AN |
7471 | static struct bpf_link *attach_trace(const struct bpf_sec_def *sec, |
7472 | struct bpf_program *prog) | |
7473 | { | |
7474 | return bpf_program__attach_trace(prog); | |
7475 | } | |
7476 | ||
7477 | struct bpf_link *bpf_program__attach(struct bpf_program *prog) | |
7478 | { | |
7479 | const struct bpf_sec_def *sec_def; | |
7480 | ||
7481 | sec_def = find_sec_def(bpf_program__title(prog, false)); | |
7482 | if (!sec_def || !sec_def->attach_fn) | |
7483 | return ERR_PTR(-ESRCH); | |
7484 | ||
7485 | return sec_def->attach_fn(sec_def, prog); | |
7486 | } | |
7487 | ||
590a0088 MKL |
7488 | static int bpf_link__detach_struct_ops(struct bpf_link *link) |
7489 | { | |
590a0088 MKL |
7490 | __u32 zero = 0; |
7491 | ||
c016b68e | 7492 | if (bpf_map_delete_elem(link->fd, &zero)) |
590a0088 MKL |
7493 | return -errno; |
7494 | ||
7495 | return 0; | |
7496 | } | |
7497 | ||
7498 | struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map) | |
7499 | { | |
7500 | struct bpf_struct_ops *st_ops; | |
c016b68e | 7501 | struct bpf_link *link; |
590a0088 MKL |
7502 | __u32 i, zero = 0; |
7503 | int err; | |
7504 | ||
7505 | if (!bpf_map__is_struct_ops(map) || map->fd == -1) | |
7506 | return ERR_PTR(-EINVAL); | |
7507 | ||
7508 | link = calloc(1, sizeof(*link)); | |
7509 | if (!link) | |
7510 | return ERR_PTR(-EINVAL); | |
7511 | ||
7512 | st_ops = map->st_ops; | |
7513 | for (i = 0; i < btf_vlen(st_ops->type); i++) { | |
7514 | struct bpf_program *prog = st_ops->progs[i]; | |
7515 | void *kern_data; | |
7516 | int prog_fd; | |
7517 | ||
7518 | if (!prog) | |
7519 | continue; | |
7520 | ||
7521 | prog_fd = bpf_program__fd(prog); | |
7522 | kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; | |
7523 | *(unsigned long *)kern_data = prog_fd; | |
7524 | } | |
7525 | ||
7526 | err = bpf_map_update_elem(map->fd, &zero, st_ops->kern_vdata, 0); | |
7527 | if (err) { | |
7528 | err = -errno; | |
7529 | free(link); | |
7530 | return ERR_PTR(err); | |
7531 | } | |
7532 | ||
c016b68e | 7533 | link->detach = bpf_link__detach_struct_ops; |
590a0088 MKL |
7534 | link->fd = map->fd; |
7535 | ||
c016b68e | 7536 | return link; |
590a0088 MKL |
7537 | } |
7538 | ||
d0cabbb0 | 7539 | enum bpf_perf_event_ret |
3dca2115 DB |
7540 | bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, |
7541 | void **copy_mem, size_t *copy_size, | |
7542 | bpf_perf_event_print_t fn, void *private_data) | |
d0cabbb0 | 7543 | { |
3dca2115 | 7544 | struct perf_event_mmap_page *header = mmap_mem; |
a64af0ef | 7545 | __u64 data_head = ring_buffer_read_head(header); |
d0cabbb0 | 7546 | __u64 data_tail = header->data_tail; |
3dca2115 DB |
7547 | void *base = ((__u8 *)header) + page_size; |
7548 | int ret = LIBBPF_PERF_EVENT_CONT; | |
7549 | struct perf_event_header *ehdr; | |
7550 | size_t ehdr_size; | |
7551 | ||
7552 | while (data_head != data_tail) { | |
7553 | ehdr = base + (data_tail & (mmap_size - 1)); | |
7554 | ehdr_size = ehdr->size; | |
7555 | ||
7556 | if (((void *)ehdr) + ehdr_size > base + mmap_size) { | |
7557 | void *copy_start = ehdr; | |
7558 | size_t len_first = base + mmap_size - copy_start; | |
7559 | size_t len_secnd = ehdr_size - len_first; | |
7560 | ||
7561 | if (*copy_size < ehdr_size) { | |
7562 | free(*copy_mem); | |
7563 | *copy_mem = malloc(ehdr_size); | |
7564 | if (!*copy_mem) { | |
7565 | *copy_size = 0; | |
d0cabbb0 JK |
7566 | ret = LIBBPF_PERF_EVENT_ERROR; |
7567 | break; | |
7568 | } | |
3dca2115 | 7569 | *copy_size = ehdr_size; |
d0cabbb0 JK |
7570 | } |
7571 | ||
3dca2115 DB |
7572 | memcpy(*copy_mem, copy_start, len_first); |
7573 | memcpy(*copy_mem + len_first, base, len_secnd); | |
7574 | ehdr = *copy_mem; | |
d0cabbb0 JK |
7575 | } |
7576 | ||
3dca2115 DB |
7577 | ret = fn(ehdr, private_data); |
7578 | data_tail += ehdr_size; | |
d0cabbb0 JK |
7579 | if (ret != LIBBPF_PERF_EVENT_CONT) |
7580 | break; | |
d0cabbb0 JK |
7581 | } |
7582 | ||
a64af0ef | 7583 | ring_buffer_write_tail(header, data_tail); |
d0cabbb0 JK |
7584 | return ret; |
7585 | } | |
34be1646 | 7586 | |
fb84b822 AN |
7587 | struct perf_buffer; |
7588 | ||
7589 | struct perf_buffer_params { | |
7590 | struct perf_event_attr *attr; | |
7591 | /* if event_cb is specified, it takes precendence */ | |
7592 | perf_buffer_event_fn event_cb; | |
7593 | /* sample_cb and lost_cb are higher-level common-case callbacks */ | |
7594 | perf_buffer_sample_fn sample_cb; | |
7595 | perf_buffer_lost_fn lost_cb; | |
7596 | void *ctx; | |
7597 | int cpu_cnt; | |
7598 | int *cpus; | |
7599 | int *map_keys; | |
7600 | }; | |
7601 | ||
7602 | struct perf_cpu_buf { | |
7603 | struct perf_buffer *pb; | |
7604 | void *base; /* mmap()'ed memory */ | |
7605 | void *buf; /* for reconstructing segmented data */ | |
7606 | size_t buf_size; | |
7607 | int fd; | |
7608 | int cpu; | |
7609 | int map_key; | |
7610 | }; | |
7611 | ||
7612 | struct perf_buffer { | |
7613 | perf_buffer_event_fn event_cb; | |
7614 | perf_buffer_sample_fn sample_cb; | |
7615 | perf_buffer_lost_fn lost_cb; | |
7616 | void *ctx; /* passed into callbacks */ | |
7617 | ||
7618 | size_t page_size; | |
7619 | size_t mmap_size; | |
7620 | struct perf_cpu_buf **cpu_bufs; | |
7621 | struct epoll_event *events; | |
783b8f01 | 7622 | int cpu_cnt; /* number of allocated CPU buffers */ |
fb84b822 AN |
7623 | int epoll_fd; /* perf event FD */ |
7624 | int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ | |
7625 | }; | |
7626 | ||
7627 | static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, | |
7628 | struct perf_cpu_buf *cpu_buf) | |
7629 | { | |
7630 | if (!cpu_buf) | |
7631 | return; | |
7632 | if (cpu_buf->base && | |
7633 | munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) | |
be18010e | 7634 | pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); |
fb84b822 AN |
7635 | if (cpu_buf->fd >= 0) { |
7636 | ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); | |
7637 | close(cpu_buf->fd); | |
7638 | } | |
7639 | free(cpu_buf->buf); | |
7640 | free(cpu_buf); | |
7641 | } | |
7642 | ||
7643 | void perf_buffer__free(struct perf_buffer *pb) | |
7644 | { | |
7645 | int i; | |
7646 | ||
7647 | if (!pb) | |
7648 | return; | |
7649 | if (pb->cpu_bufs) { | |
7650 | for (i = 0; i < pb->cpu_cnt && pb->cpu_bufs[i]; i++) { | |
7651 | struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; | |
7652 | ||
7653 | bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); | |
7654 | perf_buffer__free_cpu_buf(pb, cpu_buf); | |
7655 | } | |
7656 | free(pb->cpu_bufs); | |
7657 | } | |
7658 | if (pb->epoll_fd >= 0) | |
7659 | close(pb->epoll_fd); | |
7660 | free(pb->events); | |
7661 | free(pb); | |
7662 | } | |
7663 | ||
7664 | static struct perf_cpu_buf * | |
7665 | perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, | |
7666 | int cpu, int map_key) | |
7667 | { | |
7668 | struct perf_cpu_buf *cpu_buf; | |
7669 | char msg[STRERR_BUFSIZE]; | |
7670 | int err; | |
7671 | ||
7672 | cpu_buf = calloc(1, sizeof(*cpu_buf)); | |
7673 | if (!cpu_buf) | |
7674 | return ERR_PTR(-ENOMEM); | |
7675 | ||
7676 | cpu_buf->pb = pb; | |
7677 | cpu_buf->cpu = cpu; | |
7678 | cpu_buf->map_key = map_key; | |
7679 | ||
7680 | cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, | |
7681 | -1, PERF_FLAG_FD_CLOEXEC); | |
7682 | if (cpu_buf->fd < 0) { | |
7683 | err = -errno; | |
be18010e KW |
7684 | pr_warn("failed to open perf buffer event on cpu #%d: %s\n", |
7685 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
7686 | goto error; |
7687 | } | |
7688 | ||
7689 | cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, | |
7690 | PROT_READ | PROT_WRITE, MAP_SHARED, | |
7691 | cpu_buf->fd, 0); | |
7692 | if (cpu_buf->base == MAP_FAILED) { | |
7693 | cpu_buf->base = NULL; | |
7694 | err = -errno; | |
be18010e KW |
7695 | pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", |
7696 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
7697 | goto error; |
7698 | } | |
7699 | ||
7700 | if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { | |
7701 | err = -errno; | |
be18010e KW |
7702 | pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", |
7703 | cpu, libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
7704 | goto error; |
7705 | } | |
7706 | ||
7707 | return cpu_buf; | |
7708 | ||
7709 | error: | |
7710 | perf_buffer__free_cpu_buf(pb, cpu_buf); | |
7711 | return (struct perf_cpu_buf *)ERR_PTR(err); | |
7712 | } | |
7713 | ||
7714 | static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, | |
7715 | struct perf_buffer_params *p); | |
7716 | ||
7717 | struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, | |
7718 | const struct perf_buffer_opts *opts) | |
7719 | { | |
7720 | struct perf_buffer_params p = {}; | |
4be6e05c ACM |
7721 | struct perf_event_attr attr = { 0, }; |
7722 | ||
7723 | attr.config = PERF_COUNT_SW_BPF_OUTPUT, | |
7724 | attr.type = PERF_TYPE_SOFTWARE; | |
7725 | attr.sample_type = PERF_SAMPLE_RAW; | |
7726 | attr.sample_period = 1; | |
7727 | attr.wakeup_events = 1; | |
fb84b822 AN |
7728 | |
7729 | p.attr = &attr; | |
7730 | p.sample_cb = opts ? opts->sample_cb : NULL; | |
7731 | p.lost_cb = opts ? opts->lost_cb : NULL; | |
7732 | p.ctx = opts ? opts->ctx : NULL; | |
7733 | ||
7734 | return __perf_buffer__new(map_fd, page_cnt, &p); | |
7735 | } | |
7736 | ||
7737 | struct perf_buffer * | |
7738 | perf_buffer__new_raw(int map_fd, size_t page_cnt, | |
7739 | const struct perf_buffer_raw_opts *opts) | |
7740 | { | |
7741 | struct perf_buffer_params p = {}; | |
7742 | ||
7743 | p.attr = opts->attr; | |
7744 | p.event_cb = opts->event_cb; | |
7745 | p.ctx = opts->ctx; | |
7746 | p.cpu_cnt = opts->cpu_cnt; | |
7747 | p.cpus = opts->cpus; | |
7748 | p.map_keys = opts->map_keys; | |
7749 | ||
7750 | return __perf_buffer__new(map_fd, page_cnt, &p); | |
7751 | } | |
7752 | ||
7753 | static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, | |
7754 | struct perf_buffer_params *p) | |
7755 | { | |
783b8f01 | 7756 | const char *online_cpus_file = "/sys/devices/system/cpu/online"; |
fb84b822 AN |
7757 | struct bpf_map_info map = {}; |
7758 | char msg[STRERR_BUFSIZE]; | |
7759 | struct perf_buffer *pb; | |
783b8f01 | 7760 | bool *online = NULL; |
fb84b822 | 7761 | __u32 map_info_len; |
783b8f01 | 7762 | int err, i, j, n; |
fb84b822 AN |
7763 | |
7764 | if (page_cnt & (page_cnt - 1)) { | |
be18010e KW |
7765 | pr_warn("page count should be power of two, but is %zu\n", |
7766 | page_cnt); | |
fb84b822 AN |
7767 | return ERR_PTR(-EINVAL); |
7768 | } | |
7769 | ||
7770 | map_info_len = sizeof(map); | |
7771 | err = bpf_obj_get_info_by_fd(map_fd, &map, &map_info_len); | |
7772 | if (err) { | |
7773 | err = -errno; | |
be18010e KW |
7774 | pr_warn("failed to get map info for map FD %d: %s\n", |
7775 | map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
7776 | return ERR_PTR(err); |
7777 | } | |
7778 | ||
7779 | if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { | |
be18010e KW |
7780 | pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", |
7781 | map.name); | |
fb84b822 AN |
7782 | return ERR_PTR(-EINVAL); |
7783 | } | |
7784 | ||
7785 | pb = calloc(1, sizeof(*pb)); | |
7786 | if (!pb) | |
7787 | return ERR_PTR(-ENOMEM); | |
7788 | ||
7789 | pb->event_cb = p->event_cb; | |
7790 | pb->sample_cb = p->sample_cb; | |
7791 | pb->lost_cb = p->lost_cb; | |
7792 | pb->ctx = p->ctx; | |
7793 | ||
7794 | pb->page_size = getpagesize(); | |
7795 | pb->mmap_size = pb->page_size * page_cnt; | |
7796 | pb->map_fd = map_fd; | |
7797 | ||
7798 | pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); | |
7799 | if (pb->epoll_fd < 0) { | |
7800 | err = -errno; | |
be18010e KW |
7801 | pr_warn("failed to create epoll instance: %s\n", |
7802 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
7803 | goto error; |
7804 | } | |
7805 | ||
7806 | if (p->cpu_cnt > 0) { | |
7807 | pb->cpu_cnt = p->cpu_cnt; | |
7808 | } else { | |
7809 | pb->cpu_cnt = libbpf_num_possible_cpus(); | |
7810 | if (pb->cpu_cnt < 0) { | |
7811 | err = pb->cpu_cnt; | |
7812 | goto error; | |
7813 | } | |
7814 | if (map.max_entries < pb->cpu_cnt) | |
7815 | pb->cpu_cnt = map.max_entries; | |
7816 | } | |
7817 | ||
7818 | pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); | |
7819 | if (!pb->events) { | |
7820 | err = -ENOMEM; | |
be18010e | 7821 | pr_warn("failed to allocate events: out of memory\n"); |
fb84b822 AN |
7822 | goto error; |
7823 | } | |
7824 | pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); | |
7825 | if (!pb->cpu_bufs) { | |
7826 | err = -ENOMEM; | |
be18010e | 7827 | pr_warn("failed to allocate buffers: out of memory\n"); |
fb84b822 AN |
7828 | goto error; |
7829 | } | |
7830 | ||
783b8f01 AN |
7831 | err = parse_cpu_mask_file(online_cpus_file, &online, &n); |
7832 | if (err) { | |
7833 | pr_warn("failed to get online CPU mask: %d\n", err); | |
7834 | goto error; | |
7835 | } | |
7836 | ||
7837 | for (i = 0, j = 0; i < pb->cpu_cnt; i++) { | |
fb84b822 AN |
7838 | struct perf_cpu_buf *cpu_buf; |
7839 | int cpu, map_key; | |
7840 | ||
7841 | cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; | |
7842 | map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; | |
7843 | ||
783b8f01 AN |
7844 | /* in case user didn't explicitly requested particular CPUs to |
7845 | * be attached to, skip offline/not present CPUs | |
7846 | */ | |
7847 | if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) | |
7848 | continue; | |
7849 | ||
fb84b822 AN |
7850 | cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); |
7851 | if (IS_ERR(cpu_buf)) { | |
7852 | err = PTR_ERR(cpu_buf); | |
7853 | goto error; | |
7854 | } | |
7855 | ||
783b8f01 | 7856 | pb->cpu_bufs[j] = cpu_buf; |
fb84b822 AN |
7857 | |
7858 | err = bpf_map_update_elem(pb->map_fd, &map_key, | |
7859 | &cpu_buf->fd, 0); | |
7860 | if (err) { | |
7861 | err = -errno; | |
be18010e KW |
7862 | pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", |
7863 | cpu, map_key, cpu_buf->fd, | |
7864 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
7865 | goto error; |
7866 | } | |
7867 | ||
783b8f01 AN |
7868 | pb->events[j].events = EPOLLIN; |
7869 | pb->events[j].data.ptr = cpu_buf; | |
fb84b822 | 7870 | if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, |
783b8f01 | 7871 | &pb->events[j]) < 0) { |
fb84b822 | 7872 | err = -errno; |
be18010e KW |
7873 | pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", |
7874 | cpu, cpu_buf->fd, | |
7875 | libbpf_strerror_r(err, msg, sizeof(msg))); | |
fb84b822 AN |
7876 | goto error; |
7877 | } | |
783b8f01 | 7878 | j++; |
fb84b822 | 7879 | } |
783b8f01 AN |
7880 | pb->cpu_cnt = j; |
7881 | free(online); | |
fb84b822 AN |
7882 | |
7883 | return pb; | |
7884 | ||
7885 | error: | |
783b8f01 | 7886 | free(online); |
fb84b822 AN |
7887 | if (pb) |
7888 | perf_buffer__free(pb); | |
7889 | return ERR_PTR(err); | |
7890 | } | |
7891 | ||
7892 | struct perf_sample_raw { | |
7893 | struct perf_event_header header; | |
7894 | uint32_t size; | |
7895 | char data[0]; | |
7896 | }; | |
7897 | ||
7898 | struct perf_sample_lost { | |
7899 | struct perf_event_header header; | |
7900 | uint64_t id; | |
7901 | uint64_t lost; | |
7902 | uint64_t sample_id; | |
7903 | }; | |
7904 | ||
7905 | static enum bpf_perf_event_ret | |
7906 | perf_buffer__process_record(struct perf_event_header *e, void *ctx) | |
7907 | { | |
7908 | struct perf_cpu_buf *cpu_buf = ctx; | |
7909 | struct perf_buffer *pb = cpu_buf->pb; | |
7910 | void *data = e; | |
7911 | ||
7912 | /* user wants full control over parsing perf event */ | |
7913 | if (pb->event_cb) | |
7914 | return pb->event_cb(pb->ctx, cpu_buf->cpu, e); | |
7915 | ||
7916 | switch (e->type) { | |
7917 | case PERF_RECORD_SAMPLE: { | |
7918 | struct perf_sample_raw *s = data; | |
7919 | ||
7920 | if (pb->sample_cb) | |
7921 | pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); | |
7922 | break; | |
7923 | } | |
7924 | case PERF_RECORD_LOST: { | |
7925 | struct perf_sample_lost *s = data; | |
7926 | ||
7927 | if (pb->lost_cb) | |
7928 | pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); | |
7929 | break; | |
7930 | } | |
7931 | default: | |
be18010e | 7932 | pr_warn("unknown perf sample type %d\n", e->type); |
fb84b822 AN |
7933 | return LIBBPF_PERF_EVENT_ERROR; |
7934 | } | |
7935 | return LIBBPF_PERF_EVENT_CONT; | |
7936 | } | |
7937 | ||
7938 | static int perf_buffer__process_records(struct perf_buffer *pb, | |
7939 | struct perf_cpu_buf *cpu_buf) | |
7940 | { | |
7941 | enum bpf_perf_event_ret ret; | |
7942 | ||
7943 | ret = bpf_perf_event_read_simple(cpu_buf->base, pb->mmap_size, | |
7944 | pb->page_size, &cpu_buf->buf, | |
7945 | &cpu_buf->buf_size, | |
7946 | perf_buffer__process_record, cpu_buf); | |
7947 | if (ret != LIBBPF_PERF_EVENT_CONT) | |
7948 | return ret; | |
7949 | return 0; | |
7950 | } | |
7951 | ||
7952 | int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) | |
7953 | { | |
7954 | int i, cnt, err; | |
7955 | ||
7956 | cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); | |
7957 | for (i = 0; i < cnt; i++) { | |
7958 | struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; | |
7959 | ||
7960 | err = perf_buffer__process_records(pb, cpu_buf); | |
7961 | if (err) { | |
be18010e | 7962 | pr_warn("error while processing records: %d\n", err); |
fb84b822 AN |
7963 | return err; |
7964 | } | |
7965 | } | |
7966 | return cnt < 0 ? -errno : cnt; | |
7967 | } | |
7968 | ||
34be1646 SL |
7969 | struct bpf_prog_info_array_desc { |
7970 | int array_offset; /* e.g. offset of jited_prog_insns */ | |
7971 | int count_offset; /* e.g. offset of jited_prog_len */ | |
7972 | int size_offset; /* > 0: offset of rec size, | |
7973 | * < 0: fix size of -size_offset | |
7974 | */ | |
7975 | }; | |
7976 | ||
7977 | static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { | |
7978 | [BPF_PROG_INFO_JITED_INSNS] = { | |
7979 | offsetof(struct bpf_prog_info, jited_prog_insns), | |
7980 | offsetof(struct bpf_prog_info, jited_prog_len), | |
7981 | -1, | |
7982 | }, | |
7983 | [BPF_PROG_INFO_XLATED_INSNS] = { | |
7984 | offsetof(struct bpf_prog_info, xlated_prog_insns), | |
7985 | offsetof(struct bpf_prog_info, xlated_prog_len), | |
7986 | -1, | |
7987 | }, | |
7988 | [BPF_PROG_INFO_MAP_IDS] = { | |
7989 | offsetof(struct bpf_prog_info, map_ids), | |
7990 | offsetof(struct bpf_prog_info, nr_map_ids), | |
7991 | -(int)sizeof(__u32), | |
7992 | }, | |
7993 | [BPF_PROG_INFO_JITED_KSYMS] = { | |
7994 | offsetof(struct bpf_prog_info, jited_ksyms), | |
7995 | offsetof(struct bpf_prog_info, nr_jited_ksyms), | |
7996 | -(int)sizeof(__u64), | |
7997 | }, | |
7998 | [BPF_PROG_INFO_JITED_FUNC_LENS] = { | |
7999 | offsetof(struct bpf_prog_info, jited_func_lens), | |
8000 | offsetof(struct bpf_prog_info, nr_jited_func_lens), | |
8001 | -(int)sizeof(__u32), | |
8002 | }, | |
8003 | [BPF_PROG_INFO_FUNC_INFO] = { | |
8004 | offsetof(struct bpf_prog_info, func_info), | |
8005 | offsetof(struct bpf_prog_info, nr_func_info), | |
8006 | offsetof(struct bpf_prog_info, func_info_rec_size), | |
8007 | }, | |
8008 | [BPF_PROG_INFO_LINE_INFO] = { | |
8009 | offsetof(struct bpf_prog_info, line_info), | |
8010 | offsetof(struct bpf_prog_info, nr_line_info), | |
8011 | offsetof(struct bpf_prog_info, line_info_rec_size), | |
8012 | }, | |
8013 | [BPF_PROG_INFO_JITED_LINE_INFO] = { | |
8014 | offsetof(struct bpf_prog_info, jited_line_info), | |
8015 | offsetof(struct bpf_prog_info, nr_jited_line_info), | |
8016 | offsetof(struct bpf_prog_info, jited_line_info_rec_size), | |
8017 | }, | |
8018 | [BPF_PROG_INFO_PROG_TAGS] = { | |
8019 | offsetof(struct bpf_prog_info, prog_tags), | |
8020 | offsetof(struct bpf_prog_info, nr_prog_tags), | |
8021 | -(int)sizeof(__u8) * BPF_TAG_SIZE, | |
8022 | }, | |
8023 | ||
8024 | }; | |
8025 | ||
8983b731 AN |
8026 | static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, |
8027 | int offset) | |
34be1646 SL |
8028 | { |
8029 | __u32 *array = (__u32 *)info; | |
8030 | ||
8031 | if (offset >= 0) | |
8032 | return array[offset / sizeof(__u32)]; | |
8033 | return -(int)offset; | |
8034 | } | |
8035 | ||
8983b731 AN |
8036 | static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, |
8037 | int offset) | |
34be1646 SL |
8038 | { |
8039 | __u64 *array = (__u64 *)info; | |
8040 | ||
8041 | if (offset >= 0) | |
8042 | return array[offset / sizeof(__u64)]; | |
8043 | return -(int)offset; | |
8044 | } | |
8045 | ||
8046 | static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, | |
8047 | __u32 val) | |
8048 | { | |
8049 | __u32 *array = (__u32 *)info; | |
8050 | ||
8051 | if (offset >= 0) | |
8052 | array[offset / sizeof(__u32)] = val; | |
8053 | } | |
8054 | ||
8055 | static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, | |
8056 | __u64 val) | |
8057 | { | |
8058 | __u64 *array = (__u64 *)info; | |
8059 | ||
8060 | if (offset >= 0) | |
8061 | array[offset / sizeof(__u64)] = val; | |
8062 | } | |
8063 | ||
8064 | struct bpf_prog_info_linear * | |
8065 | bpf_program__get_prog_info_linear(int fd, __u64 arrays) | |
8066 | { | |
8067 | struct bpf_prog_info_linear *info_linear; | |
8068 | struct bpf_prog_info info = {}; | |
8069 | __u32 info_len = sizeof(info); | |
8070 | __u32 data_len = 0; | |
8071 | int i, err; | |
8072 | void *ptr; | |
8073 | ||
8074 | if (arrays >> BPF_PROG_INFO_LAST_ARRAY) | |
8075 | return ERR_PTR(-EINVAL); | |
8076 | ||
8077 | /* step 1: get array dimensions */ | |
8078 | err = bpf_obj_get_info_by_fd(fd, &info, &info_len); | |
8079 | if (err) { | |
8080 | pr_debug("can't get prog info: %s", strerror(errno)); | |
8081 | return ERR_PTR(-EFAULT); | |
8082 | } | |
8083 | ||
8084 | /* step 2: calculate total size of all arrays */ | |
8085 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
8086 | bool include_array = (arrays & (1UL << i)) > 0; | |
8087 | struct bpf_prog_info_array_desc *desc; | |
8088 | __u32 count, size; | |
8089 | ||
8090 | desc = bpf_prog_info_array_desc + i; | |
8091 | ||
8092 | /* kernel is too old to support this field */ | |
8093 | if (info_len < desc->array_offset + sizeof(__u32) || | |
8094 | info_len < desc->count_offset + sizeof(__u32) || | |
8095 | (desc->size_offset > 0 && info_len < desc->size_offset)) | |
8096 | include_array = false; | |
8097 | ||
8098 | if (!include_array) { | |
8099 | arrays &= ~(1UL << i); /* clear the bit */ | |
8100 | continue; | |
8101 | } | |
8102 | ||
8103 | count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
8104 | size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
8105 | ||
8106 | data_len += count * size; | |
8107 | } | |
8108 | ||
8109 | /* step 3: allocate continuous memory */ | |
8110 | data_len = roundup(data_len, sizeof(__u64)); | |
8111 | info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); | |
8112 | if (!info_linear) | |
8113 | return ERR_PTR(-ENOMEM); | |
8114 | ||
8115 | /* step 4: fill data to info_linear->info */ | |
8116 | info_linear->arrays = arrays; | |
8117 | memset(&info_linear->info, 0, sizeof(info)); | |
8118 | ptr = info_linear->data; | |
8119 | ||
8120 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
8121 | struct bpf_prog_info_array_desc *desc; | |
8122 | __u32 count, size; | |
8123 | ||
8124 | if ((arrays & (1UL << i)) == 0) | |
8125 | continue; | |
8126 | ||
8127 | desc = bpf_prog_info_array_desc + i; | |
8128 | count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
8129 | size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
8130 | bpf_prog_info_set_offset_u32(&info_linear->info, | |
8131 | desc->count_offset, count); | |
8132 | bpf_prog_info_set_offset_u32(&info_linear->info, | |
8133 | desc->size_offset, size); | |
8134 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
8135 | desc->array_offset, | |
8136 | ptr_to_u64(ptr)); | |
8137 | ptr += count * size; | |
8138 | } | |
8139 | ||
8140 | /* step 5: call syscall again to get required arrays */ | |
8141 | err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); | |
8142 | if (err) { | |
8143 | pr_debug("can't get prog info: %s", strerror(errno)); | |
8144 | free(info_linear); | |
8145 | return ERR_PTR(-EFAULT); | |
8146 | } | |
8147 | ||
8148 | /* step 6: verify the data */ | |
8149 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
8150 | struct bpf_prog_info_array_desc *desc; | |
8151 | __u32 v1, v2; | |
8152 | ||
8153 | if ((arrays & (1UL << i)) == 0) | |
8154 | continue; | |
8155 | ||
8156 | desc = bpf_prog_info_array_desc + i; | |
8157 | v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); | |
8158 | v2 = bpf_prog_info_read_offset_u32(&info_linear->info, | |
8159 | desc->count_offset); | |
8160 | if (v1 != v2) | |
be18010e | 8161 | pr_warn("%s: mismatch in element count\n", __func__); |
34be1646 SL |
8162 | |
8163 | v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); | |
8164 | v2 = bpf_prog_info_read_offset_u32(&info_linear->info, | |
8165 | desc->size_offset); | |
8166 | if (v1 != v2) | |
be18010e | 8167 | pr_warn("%s: mismatch in rec size\n", __func__); |
34be1646 SL |
8168 | } |
8169 | ||
8170 | /* step 7: update info_len and data_len */ | |
8171 | info_linear->info_len = sizeof(struct bpf_prog_info); | |
8172 | info_linear->data_len = data_len; | |
8173 | ||
8174 | return info_linear; | |
8175 | } | |
8176 | ||
8177 | void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) | |
8178 | { | |
8179 | int i; | |
8180 | ||
8181 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
8182 | struct bpf_prog_info_array_desc *desc; | |
8183 | __u64 addr, offs; | |
8184 | ||
8185 | if ((info_linear->arrays & (1UL << i)) == 0) | |
8186 | continue; | |
8187 | ||
8188 | desc = bpf_prog_info_array_desc + i; | |
8189 | addr = bpf_prog_info_read_offset_u64(&info_linear->info, | |
8190 | desc->array_offset); | |
8191 | offs = addr - ptr_to_u64(info_linear->data); | |
8192 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
8193 | desc->array_offset, offs); | |
8194 | } | |
8195 | } | |
8196 | ||
8197 | void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) | |
8198 | { | |
8199 | int i; | |
8200 | ||
8201 | for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { | |
8202 | struct bpf_prog_info_array_desc *desc; | |
8203 | __u64 addr, offs; | |
8204 | ||
8205 | if ((info_linear->arrays & (1UL << i)) == 0) | |
8206 | continue; | |
8207 | ||
8208 | desc = bpf_prog_info_array_desc + i; | |
8209 | offs = bpf_prog_info_read_offset_u64(&info_linear->info, | |
8210 | desc->array_offset); | |
8211 | addr = offs + ptr_to_u64(info_linear->data); | |
8212 | bpf_prog_info_set_offset_u64(&info_linear->info, | |
8213 | desc->array_offset, addr); | |
8214 | } | |
8215 | } | |
6446b315 | 8216 | |
ff26ce5c EC |
8217 | int bpf_program__set_attach_target(struct bpf_program *prog, |
8218 | int attach_prog_fd, | |
8219 | const char *attach_func_name) | |
8220 | { | |
8221 | int btf_id; | |
8222 | ||
8223 | if (!prog || attach_prog_fd < 0 || !attach_func_name) | |
8224 | return -EINVAL; | |
8225 | ||
8226 | if (attach_prog_fd) | |
8227 | btf_id = libbpf_find_prog_btf_id(attach_func_name, | |
8228 | attach_prog_fd); | |
8229 | else | |
8230 | btf_id = __find_vmlinux_btf_id(prog->obj->btf_vmlinux, | |
8231 | attach_func_name, | |
8232 | prog->expected_attach_type); | |
8233 | ||
8234 | if (btf_id < 0) | |
8235 | return btf_id; | |
8236 | ||
8237 | prog->attach_btf_id = btf_id; | |
8238 | prog->attach_prog_fd = attach_prog_fd; | |
8239 | return 0; | |
8240 | } | |
8241 | ||
6803ee25 | 8242 | int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) |
6446b315 | 8243 | { |
6803ee25 AN |
8244 | int err = 0, n, len, start, end = -1; |
8245 | bool *tmp; | |
6446b315 | 8246 | |
6803ee25 AN |
8247 | *mask = NULL; |
8248 | *mask_sz = 0; | |
8249 | ||
8250 | /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ | |
8251 | while (*s) { | |
8252 | if (*s == ',' || *s == '\n') { | |
8253 | s++; | |
8254 | continue; | |
8255 | } | |
8256 | n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); | |
8257 | if (n <= 0 || n > 2) { | |
8258 | pr_warn("Failed to get CPU range %s: %d\n", s, n); | |
8259 | err = -EINVAL; | |
8260 | goto cleanup; | |
8261 | } else if (n == 1) { | |
8262 | end = start; | |
8263 | } | |
8264 | if (start < 0 || start > end) { | |
8265 | pr_warn("Invalid CPU range [%d,%d] in %s\n", | |
8266 | start, end, s); | |
8267 | err = -EINVAL; | |
8268 | goto cleanup; | |
8269 | } | |
8270 | tmp = realloc(*mask, end + 1); | |
8271 | if (!tmp) { | |
8272 | err = -ENOMEM; | |
8273 | goto cleanup; | |
8274 | } | |
8275 | *mask = tmp; | |
8276 | memset(tmp + *mask_sz, 0, start - *mask_sz); | |
8277 | memset(tmp + start, 1, end - start + 1); | |
8278 | *mask_sz = end + 1; | |
8279 | s += len; | |
8280 | } | |
8281 | if (!*mask_sz) { | |
8282 | pr_warn("Empty CPU range\n"); | |
8283 | return -EINVAL; | |
8284 | } | |
8285 | return 0; | |
8286 | cleanup: | |
8287 | free(*mask); | |
8288 | *mask = NULL; | |
8289 | return err; | |
8290 | } | |
8291 | ||
8292 | int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) | |
8293 | { | |
8294 | int fd, err = 0, len; | |
8295 | char buf[128]; | |
6446b315 HL |
8296 | |
8297 | fd = open(fcpu, O_RDONLY); | |
8298 | if (fd < 0) { | |
6803ee25 AN |
8299 | err = -errno; |
8300 | pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); | |
8301 | return err; | |
6446b315 HL |
8302 | } |
8303 | len = read(fd, buf, sizeof(buf)); | |
8304 | close(fd); | |
8305 | if (len <= 0) { | |
6803ee25 AN |
8306 | err = len ? -errno : -EINVAL; |
8307 | pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); | |
8308 | return err; | |
6446b315 | 8309 | } |
6803ee25 AN |
8310 | if (len >= sizeof(buf)) { |
8311 | pr_warn("CPU mask is too big in file %s\n", fcpu); | |
8312 | return -E2BIG; | |
6446b315 HL |
8313 | } |
8314 | buf[len] = '\0'; | |
8315 | ||
6803ee25 AN |
8316 | return parse_cpu_mask_str(buf, mask, mask_sz); |
8317 | } | |
8318 | ||
8319 | int libbpf_num_possible_cpus(void) | |
8320 | { | |
8321 | static const char *fcpu = "/sys/devices/system/cpu/possible"; | |
8322 | static int cpus; | |
8323 | int err, n, i, tmp_cpus; | |
8324 | bool *mask; | |
8325 | ||
8326 | tmp_cpus = READ_ONCE(cpus); | |
8327 | if (tmp_cpus > 0) | |
8328 | return tmp_cpus; | |
8329 | ||
8330 | err = parse_cpu_mask_file(fcpu, &mask, &n); | |
8331 | if (err) | |
8332 | return err; | |
8333 | ||
8334 | tmp_cpus = 0; | |
8335 | for (i = 0; i < n; i++) { | |
8336 | if (mask[i]) | |
8337 | tmp_cpus++; | |
6446b315 | 8338 | } |
6803ee25 | 8339 | free(mask); |
56fbc241 TC |
8340 | |
8341 | WRITE_ONCE(cpus, tmp_cpus); | |
8342 | return tmp_cpus; | |
6446b315 | 8343 | } |
d66562fb AN |
8344 | |
8345 | int bpf_object__open_skeleton(struct bpf_object_skeleton *s, | |
8346 | const struct bpf_object_open_opts *opts) | |
8347 | { | |
8348 | DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, | |
8349 | .object_name = s->name, | |
8350 | ); | |
8351 | struct bpf_object *obj; | |
8352 | int i; | |
8353 | ||
8354 | /* Attempt to preserve opts->object_name, unless overriden by user | |
8355 | * explicitly. Overwriting object name for skeletons is discouraged, | |
8356 | * as it breaks global data maps, because they contain object name | |
8357 | * prefix as their own map name prefix. When skeleton is generated, | |
8358 | * bpftool is making an assumption that this name will stay the same. | |
8359 | */ | |
8360 | if (opts) { | |
8361 | memcpy(&skel_opts, opts, sizeof(*opts)); | |
8362 | if (!opts->object_name) | |
8363 | skel_opts.object_name = s->name; | |
8364 | } | |
8365 | ||
8366 | obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); | |
8367 | if (IS_ERR(obj)) { | |
8368 | pr_warn("failed to initialize skeleton BPF object '%s': %ld\n", | |
8369 | s->name, PTR_ERR(obj)); | |
8370 | return PTR_ERR(obj); | |
8371 | } | |
8372 | ||
8373 | *s->obj = obj; | |
8374 | ||
8375 | for (i = 0; i < s->map_cnt; i++) { | |
8376 | struct bpf_map **map = s->maps[i].map; | |
8377 | const char *name = s->maps[i].name; | |
8378 | void **mmaped = s->maps[i].mmaped; | |
8379 | ||
8380 | *map = bpf_object__find_map_by_name(obj, name); | |
8381 | if (!*map) { | |
8382 | pr_warn("failed to find skeleton map '%s'\n", name); | |
8383 | return -ESRCH; | |
8384 | } | |
8385 | ||
2ad97d47 | 8386 | /* externs shouldn't be pre-setup from user code */ |
81bfdd08 | 8387 | if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) |
d66562fb AN |
8388 | *mmaped = (*map)->mmaped; |
8389 | } | |
8390 | ||
8391 | for (i = 0; i < s->prog_cnt; i++) { | |
8392 | struct bpf_program **prog = s->progs[i].prog; | |
8393 | const char *name = s->progs[i].name; | |
8394 | ||
8395 | *prog = bpf_object__find_program_by_name(obj, name); | |
8396 | if (!*prog) { | |
8397 | pr_warn("failed to find skeleton program '%s'\n", name); | |
8398 | return -ESRCH; | |
8399 | } | |
8400 | } | |
8401 | ||
8402 | return 0; | |
8403 | } | |
8404 | ||
8405 | int bpf_object__load_skeleton(struct bpf_object_skeleton *s) | |
8406 | { | |
8407 | int i, err; | |
8408 | ||
8409 | err = bpf_object__load(*s->obj); | |
8410 | if (err) { | |
8411 | pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); | |
8412 | return err; | |
8413 | } | |
8414 | ||
8415 | for (i = 0; i < s->map_cnt; i++) { | |
8416 | struct bpf_map *map = *s->maps[i].map; | |
8417 | size_t mmap_sz = bpf_map_mmap_sz(map); | |
8418 | int prot, map_fd = bpf_map__fd(map); | |
8419 | void **mmaped = s->maps[i].mmaped; | |
d66562fb AN |
8420 | |
8421 | if (!mmaped) | |
8422 | continue; | |
8423 | ||
8424 | if (!(map->def.map_flags & BPF_F_MMAPABLE)) { | |
8425 | *mmaped = NULL; | |
8426 | continue; | |
8427 | } | |
8428 | ||
8429 | if (map->def.map_flags & BPF_F_RDONLY_PROG) | |
8430 | prot = PROT_READ; | |
8431 | else | |
8432 | prot = PROT_READ | PROT_WRITE; | |
8433 | ||
8434 | /* Remap anonymous mmap()-ed "map initialization image" as | |
8435 | * a BPF map-backed mmap()-ed memory, but preserving the same | |
8436 | * memory address. This will cause kernel to change process' | |
8437 | * page table to point to a different piece of kernel memory, | |
8438 | * but from userspace point of view memory address (and its | |
8439 | * contents, being identical at this point) will stay the | |
8440 | * same. This mapping will be released by bpf_object__close() | |
8441 | * as per normal clean up procedure, so we don't need to worry | |
8442 | * about it from skeleton's clean up perspective. | |
8443 | */ | |
2ad97d47 AN |
8444 | *mmaped = mmap(map->mmaped, mmap_sz, prot, |
8445 | MAP_SHARED | MAP_FIXED, map_fd, 0); | |
8446 | if (*mmaped == MAP_FAILED) { | |
d66562fb AN |
8447 | err = -errno; |
8448 | *mmaped = NULL; | |
8449 | pr_warn("failed to re-mmap() map '%s': %d\n", | |
8450 | bpf_map__name(map), err); | |
8451 | return err; | |
8452 | } | |
8453 | } | |
8454 | ||
8455 | return 0; | |
8456 | } | |
8457 | ||
8458 | int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) | |
8459 | { | |
8460 | int i; | |
8461 | ||
8462 | for (i = 0; i < s->prog_cnt; i++) { | |
8463 | struct bpf_program *prog = *s->progs[i].prog; | |
8464 | struct bpf_link **link = s->progs[i].link; | |
8465 | const struct bpf_sec_def *sec_def; | |
8466 | const char *sec_name = bpf_program__title(prog, false); | |
8467 | ||
8468 | sec_def = find_sec_def(sec_name); | |
8469 | if (!sec_def || !sec_def->attach_fn) | |
8470 | continue; | |
8471 | ||
8472 | *link = sec_def->attach_fn(sec_def, prog); | |
8473 | if (IS_ERR(*link)) { | |
8474 | pr_warn("failed to auto-attach program '%s': %ld\n", | |
8475 | bpf_program__name(prog), PTR_ERR(*link)); | |
8476 | return PTR_ERR(*link); | |
8477 | } | |
8478 | } | |
8479 | ||
8480 | return 0; | |
8481 | } | |
8482 | ||
8483 | void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) | |
8484 | { | |
8485 | int i; | |
8486 | ||
8487 | for (i = 0; i < s->prog_cnt; i++) { | |
8488 | struct bpf_link **link = s->progs[i].link; | |
8489 | ||
8490 | if (!IS_ERR_OR_NULL(*link)) | |
8491 | bpf_link__destroy(*link); | |
8492 | *link = NULL; | |
8493 | } | |
8494 | } | |
8495 | ||
8496 | void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) | |
8497 | { | |
8498 | if (s->progs) | |
8499 | bpf_object__detach_skeleton(s); | |
8500 | if (s->obj) | |
8501 | bpf_object__close(*s->obj); | |
8502 | free(s->maps); | |
8503 | free(s->progs); | |
8504 | free(s); | |
8505 | } |