Commit | Line | Data |
---|---|---|
1bc38b8f | 1 | // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) |
6061a3d6 | 2 | |
1b76c13e WN |
3 | /* |
4 | * Common eBPF ELF object loading operations. | |
5 | * | |
6 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
7 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
8 | * Copyright (C) 2015 Huawei Inc. | |
f367540c | 9 | * Copyright (C) 2017 Nicira, Inc. |
1b76c13e WN |
10 | */ |
11 | ||
531b014e | 12 | #define _GNU_SOURCE |
1b76c13e | 13 | #include <stdlib.h> |
b3f59d66 WN |
14 | #include <stdio.h> |
15 | #include <stdarg.h> | |
f367540c | 16 | #include <libgen.h> |
34090915 | 17 | #include <inttypes.h> |
b3f59d66 | 18 | #include <string.h> |
1b76c13e | 19 | #include <unistd.h> |
1a5e3fb1 WN |
20 | #include <fcntl.h> |
21 | #include <errno.h> | |
1b76c13e | 22 | #include <asm/unistd.h> |
e28ff1a8 | 23 | #include <linux/err.h> |
cb1e5e96 | 24 | #include <linux/kernel.h> |
1b76c13e | 25 | #include <linux/bpf.h> |
38d5d3b3 | 26 | #include <linux/btf.h> |
9a208eff | 27 | #include <linux/list.h> |
f367540c | 28 | #include <linux/limits.h> |
438363c0 | 29 | #include <linux/perf_event.h> |
a64af0ef | 30 | #include <linux/ring_buffer.h> |
f367540c JS |
31 | #include <sys/stat.h> |
32 | #include <sys/types.h> | |
33 | #include <sys/vfs.h> | |
531b014e | 34 | #include <tools/libc_compat.h> |
1a5e3fb1 WN |
35 | #include <libelf.h> |
36 | #include <gelf.h> | |
1b76c13e WN |
37 | |
38 | #include "libbpf.h" | |
52d3352e | 39 | #include "bpf.h" |
8a138aed | 40 | #include "btf.h" |
6d41907c | 41 | #include "str_error.h" |
b3f59d66 | 42 | |
9b16137a WN |
43 | #ifndef EM_BPF |
44 | #define EM_BPF 247 | |
45 | #endif | |
46 | ||
f367540c JS |
47 | #ifndef BPF_FS_MAGIC |
48 | #define BPF_FS_MAGIC 0xcafe4a11 | |
49 | #endif | |
50 | ||
b3f59d66 WN |
51 | #define __printf(a, b) __attribute__((format(printf, a, b))) |
52 | ||
53 | __printf(1, 2) | |
54 | static int __base_pr(const char *format, ...) | |
55 | { | |
56 | va_list args; | |
57 | int err; | |
58 | ||
59 | va_start(args, format); | |
60 | err = vfprintf(stderr, format, args); | |
61 | va_end(args); | |
62 | return err; | |
63 | } | |
64 | ||
65 | static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; | |
66 | static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; | |
67 | static __printf(1, 2) libbpf_print_fn_t __pr_debug; | |
68 | ||
69 | #define __pr(func, fmt, ...) \ | |
70 | do { \ | |
71 | if ((func)) \ | |
72 | (func)("libbpf: " fmt, ##__VA_ARGS__); \ | |
73 | } while (0) | |
74 | ||
75 | #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) | |
76 | #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) | |
77 | #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) | |
78 | ||
79 | void libbpf_set_print(libbpf_print_fn_t warn, | |
80 | libbpf_print_fn_t info, | |
81 | libbpf_print_fn_t debug) | |
82 | { | |
83 | __pr_warning = warn; | |
84 | __pr_info = info; | |
85 | __pr_debug = debug; | |
86 | } | |
1a5e3fb1 | 87 | |
6371ca3b WN |
88 | #define STRERR_BUFSIZE 128 |
89 | ||
6371ca3b WN |
90 | #define CHECK_ERR(action, err, out) do { \ |
91 | err = action; \ | |
92 | if (err) \ | |
93 | goto out; \ | |
94 | } while(0) | |
95 | ||
96 | ||
1a5e3fb1 WN |
97 | /* Copied from tools/perf/util/util.h */ |
98 | #ifndef zfree | |
99 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
100 | #endif | |
101 | ||
102 | #ifndef zclose | |
103 | # define zclose(fd) ({ \ | |
104 | int ___err = 0; \ | |
105 | if ((fd) >= 0) \ | |
106 | ___err = close((fd)); \ | |
107 | fd = -1; \ | |
108 | ___err; }) | |
109 | #endif | |
110 | ||
111 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
112 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
113 | #else | |
114 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
115 | #endif | |
116 | ||
a5b8bd47 WN |
117 | /* |
118 | * bpf_prog should be a better name but it has been used in | |
119 | * linux/filter.h. | |
120 | */ | |
121 | struct bpf_program { | |
122 | /* Index in elf obj file, for relocation use. */ | |
123 | int idx; | |
88cda1c9 | 124 | char *name; |
f0307a7e | 125 | int prog_ifindex; |
a5b8bd47 WN |
126 | char *section_name; |
127 | struct bpf_insn *insns; | |
48cca7e4 | 128 | size_t insns_cnt, main_prog_cnt; |
5f44e4c8 | 129 | enum bpf_prog_type type; |
34090915 | 130 | |
48cca7e4 AS |
131 | struct reloc_desc { |
132 | enum { | |
133 | RELO_LD64, | |
134 | RELO_CALL, | |
135 | } type; | |
34090915 | 136 | int insn_idx; |
48cca7e4 AS |
137 | union { |
138 | int map_idx; | |
139 | int text_off; | |
140 | }; | |
34090915 WN |
141 | } *reloc_desc; |
142 | int nr_reloc; | |
55cffde2 | 143 | |
b580563e WN |
144 | struct { |
145 | int nr; | |
146 | int *fds; | |
147 | } instances; | |
148 | bpf_program_prep_t preprocessor; | |
aa9b1ac3 WN |
149 | |
150 | struct bpf_object *obj; | |
151 | void *priv; | |
152 | bpf_program_clear_priv_t clear_priv; | |
d7be143b AI |
153 | |
154 | enum bpf_attach_type expected_attach_type; | |
a5b8bd47 WN |
155 | }; |
156 | ||
9d759a9b WN |
157 | struct bpf_map { |
158 | int fd; | |
561bbcca | 159 | char *name; |
4708bbda | 160 | size_t offset; |
f0307a7e | 161 | int map_ifindex; |
9d759a9b | 162 | struct bpf_map_def def; |
5b891af7 MKL |
163 | __u32 btf_key_type_id; |
164 | __u32 btf_value_type_id; | |
9d759a9b WN |
165 | void *priv; |
166 | bpf_map_clear_priv_t clear_priv; | |
167 | }; | |
168 | ||
9a208eff WN |
169 | static LIST_HEAD(bpf_objects_list); |
170 | ||
1a5e3fb1 | 171 | struct bpf_object { |
cb1e5e96 | 172 | char license[64]; |
438363c0 | 173 | __u32 kern_version; |
0b3d1efa | 174 | |
a5b8bd47 WN |
175 | struct bpf_program *programs; |
176 | size_t nr_programs; | |
9d759a9b WN |
177 | struct bpf_map *maps; |
178 | size_t nr_maps; | |
179 | ||
52d3352e | 180 | bool loaded; |
9a94f277 | 181 | bool has_pseudo_calls; |
a5b8bd47 | 182 | |
1a5e3fb1 WN |
183 | /* |
184 | * Information when doing elf related work. Only valid if fd | |
185 | * is valid. | |
186 | */ | |
187 | struct { | |
188 | int fd; | |
6c956392 WN |
189 | void *obj_buf; |
190 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
191 | Elf *elf; |
192 | GElf_Ehdr ehdr; | |
bec7d68c | 193 | Elf_Data *symbols; |
77ba9a5b | 194 | size_t strtabidx; |
b62f06e8 WN |
195 | struct { |
196 | GElf_Shdr shdr; | |
197 | Elf_Data *data; | |
198 | } *reloc; | |
199 | int nr_reloc; | |
666810e8 | 200 | int maps_shndx; |
48cca7e4 | 201 | int text_shndx; |
1a5e3fb1 | 202 | } efile; |
9a208eff WN |
203 | /* |
204 | * All loaded bpf_object is linked in a list, which is | |
205 | * hidden to caller. bpf_objects__<func> handlers deal with | |
206 | * all objects. | |
207 | */ | |
208 | struct list_head list; | |
10931d24 | 209 | |
8a138aed MKL |
210 | struct btf *btf; |
211 | ||
10931d24 WN |
212 | void *priv; |
213 | bpf_object_clear_priv_t clear_priv; | |
214 | ||
1a5e3fb1 WN |
215 | char path[]; |
216 | }; | |
217 | #define obj_elf_valid(o) ((o)->efile.elf) | |
218 | ||
29cd77f4 | 219 | void bpf_program__unload(struct bpf_program *prog) |
55cffde2 | 220 | { |
b580563e WN |
221 | int i; |
222 | ||
55cffde2 WN |
223 | if (!prog) |
224 | return; | |
225 | ||
b580563e WN |
226 | /* |
227 | * If the object is opened but the program was never loaded, | |
228 | * it is possible that prog->instances.nr == -1. | |
229 | */ | |
230 | if (prog->instances.nr > 0) { | |
231 | for (i = 0; i < prog->instances.nr; i++) | |
232 | zclose(prog->instances.fds[i]); | |
233 | } else if (prog->instances.nr != -1) { | |
234 | pr_warning("Internal error: instances.nr is %d\n", | |
235 | prog->instances.nr); | |
236 | } | |
237 | ||
238 | prog->instances.nr = -1; | |
239 | zfree(&prog->instances.fds); | |
55cffde2 WN |
240 | } |
241 | ||
a5b8bd47 WN |
242 | static void bpf_program__exit(struct bpf_program *prog) |
243 | { | |
244 | if (!prog) | |
245 | return; | |
246 | ||
aa9b1ac3 WN |
247 | if (prog->clear_priv) |
248 | prog->clear_priv(prog, prog->priv); | |
249 | ||
250 | prog->priv = NULL; | |
251 | prog->clear_priv = NULL; | |
252 | ||
55cffde2 | 253 | bpf_program__unload(prog); |
88cda1c9 | 254 | zfree(&prog->name); |
a5b8bd47 WN |
255 | zfree(&prog->section_name); |
256 | zfree(&prog->insns); | |
34090915 WN |
257 | zfree(&prog->reloc_desc); |
258 | ||
259 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
260 | prog->insns_cnt = 0; |
261 | prog->idx = -1; | |
262 | } | |
263 | ||
264 | static int | |
88cda1c9 MKL |
265 | bpf_program__init(void *data, size_t size, char *section_name, int idx, |
266 | struct bpf_program *prog) | |
a5b8bd47 WN |
267 | { |
268 | if (size < sizeof(struct bpf_insn)) { | |
88cda1c9 | 269 | pr_warning("corrupted section '%s'\n", section_name); |
a5b8bd47 WN |
270 | return -EINVAL; |
271 | } | |
272 | ||
273 | bzero(prog, sizeof(*prog)); | |
274 | ||
88cda1c9 | 275 | prog->section_name = strdup(section_name); |
a5b8bd47 | 276 | if (!prog->section_name) { |
077c066a JDB |
277 | pr_warning("failed to alloc name for prog under section(%d) %s\n", |
278 | idx, section_name); | |
a5b8bd47 WN |
279 | goto errout; |
280 | } | |
281 | ||
282 | prog->insns = malloc(size); | |
283 | if (!prog->insns) { | |
88cda1c9 MKL |
284 | pr_warning("failed to alloc insns for prog under section %s\n", |
285 | section_name); | |
a5b8bd47 WN |
286 | goto errout; |
287 | } | |
288 | prog->insns_cnt = size / sizeof(struct bpf_insn); | |
289 | memcpy(prog->insns, data, | |
290 | prog->insns_cnt * sizeof(struct bpf_insn)); | |
291 | prog->idx = idx; | |
b580563e WN |
292 | prog->instances.fds = NULL; |
293 | prog->instances.nr = -1; | |
5f44e4c8 | 294 | prog->type = BPF_PROG_TYPE_KPROBE; |
a5b8bd47 WN |
295 | |
296 | return 0; | |
297 | errout: | |
298 | bpf_program__exit(prog); | |
299 | return -ENOMEM; | |
300 | } | |
301 | ||
302 | static int | |
303 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
88cda1c9 | 304 | char *section_name, int idx) |
a5b8bd47 WN |
305 | { |
306 | struct bpf_program prog, *progs; | |
307 | int nr_progs, err; | |
308 | ||
88cda1c9 | 309 | err = bpf_program__init(data, size, section_name, idx, &prog); |
a5b8bd47 WN |
310 | if (err) |
311 | return err; | |
312 | ||
313 | progs = obj->programs; | |
314 | nr_progs = obj->nr_programs; | |
315 | ||
531b014e | 316 | progs = reallocarray(progs, nr_progs + 1, sizeof(progs[0])); |
a5b8bd47 WN |
317 | if (!progs) { |
318 | /* | |
319 | * In this case the original obj->programs | |
320 | * is still valid, so don't need special treat for | |
321 | * bpf_close_object(). | |
322 | */ | |
88cda1c9 MKL |
323 | pr_warning("failed to alloc a new program under section '%s'\n", |
324 | section_name); | |
a5b8bd47 WN |
325 | bpf_program__exit(&prog); |
326 | return -ENOMEM; | |
327 | } | |
328 | ||
329 | pr_debug("found program %s\n", prog.section_name); | |
330 | obj->programs = progs; | |
331 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 332 | prog.obj = obj; |
a5b8bd47 WN |
333 | progs[nr_progs] = prog; |
334 | return 0; | |
335 | } | |
336 | ||
88cda1c9 MKL |
337 | static int |
338 | bpf_object__init_prog_names(struct bpf_object *obj) | |
339 | { | |
340 | Elf_Data *symbols = obj->efile.symbols; | |
341 | struct bpf_program *prog; | |
342 | size_t pi, si; | |
343 | ||
344 | for (pi = 0; pi < obj->nr_programs; pi++) { | |
48cca7e4 | 345 | const char *name = NULL; |
88cda1c9 MKL |
346 | |
347 | prog = &obj->programs[pi]; | |
348 | ||
349 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; | |
350 | si++) { | |
351 | GElf_Sym sym; | |
352 | ||
353 | if (!gelf_getsym(symbols, si, &sym)) | |
354 | continue; | |
355 | if (sym.st_shndx != prog->idx) | |
356 | continue; | |
fe4d44b2 RG |
357 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) |
358 | continue; | |
88cda1c9 MKL |
359 | |
360 | name = elf_strptr(obj->efile.elf, | |
361 | obj->efile.strtabidx, | |
362 | sym.st_name); | |
363 | if (!name) { | |
364 | pr_warning("failed to get sym name string for prog %s\n", | |
365 | prog->section_name); | |
366 | return -LIBBPF_ERRNO__LIBELF; | |
367 | } | |
368 | } | |
369 | ||
9a94f277 JK |
370 | if (!name && prog->idx == obj->efile.text_shndx) |
371 | name = ".text"; | |
372 | ||
88cda1c9 MKL |
373 | if (!name) { |
374 | pr_warning("failed to find sym for prog %s\n", | |
375 | prog->section_name); | |
376 | return -EINVAL; | |
377 | } | |
9a94f277 | 378 | |
88cda1c9 MKL |
379 | prog->name = strdup(name); |
380 | if (!prog->name) { | |
381 | pr_warning("failed to allocate memory for prog sym %s\n", | |
382 | name); | |
383 | return -ENOMEM; | |
384 | } | |
385 | } | |
386 | ||
387 | return 0; | |
388 | } | |
389 | ||
6c956392 WN |
390 | static struct bpf_object *bpf_object__new(const char *path, |
391 | void *obj_buf, | |
392 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
393 | { |
394 | struct bpf_object *obj; | |
395 | ||
396 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
397 | if (!obj) { | |
398 | pr_warning("alloc memory failed for %s\n", path); | |
6371ca3b | 399 | return ERR_PTR(-ENOMEM); |
1a5e3fb1 WN |
400 | } |
401 | ||
402 | strcpy(obj->path, path); | |
403 | obj->efile.fd = -1; | |
6c956392 WN |
404 | |
405 | /* | |
406 | * Caller of this function should also calls | |
407 | * bpf_object__elf_finish() after data collection to return | |
408 | * obj_buf to user. If not, we should duplicate the buffer to | |
409 | * avoid user freeing them before elf finish. | |
410 | */ | |
411 | obj->efile.obj_buf = obj_buf; | |
412 | obj->efile.obj_buf_sz = obj_buf_sz; | |
666810e8 | 413 | obj->efile.maps_shndx = -1; |
6c956392 | 414 | |
52d3352e | 415 | obj->loaded = false; |
9a208eff WN |
416 | |
417 | INIT_LIST_HEAD(&obj->list); | |
418 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
419 | return obj; |
420 | } | |
421 | ||
422 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
423 | { | |
424 | if (!obj_elf_valid(obj)) | |
425 | return; | |
426 | ||
427 | if (obj->efile.elf) { | |
428 | elf_end(obj->efile.elf); | |
429 | obj->efile.elf = NULL; | |
430 | } | |
bec7d68c | 431 | obj->efile.symbols = NULL; |
b62f06e8 WN |
432 | |
433 | zfree(&obj->efile.reloc); | |
434 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 435 | zclose(obj->efile.fd); |
6c956392 WN |
436 | obj->efile.obj_buf = NULL; |
437 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
438 | } |
439 | ||
440 | static int bpf_object__elf_init(struct bpf_object *obj) | |
441 | { | |
442 | int err = 0; | |
443 | GElf_Ehdr *ep; | |
444 | ||
445 | if (obj_elf_valid(obj)) { | |
446 | pr_warning("elf init: internal error\n"); | |
6371ca3b | 447 | return -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
448 | } |
449 | ||
6c956392 WN |
450 | if (obj->efile.obj_buf_sz > 0) { |
451 | /* | |
452 | * obj_buf should have been validated by | |
453 | * bpf_object__open_buffer(). | |
454 | */ | |
455 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
456 | obj->efile.obj_buf_sz); | |
457 | } else { | |
458 | obj->efile.fd = open(obj->path, O_RDONLY); | |
459 | if (obj->efile.fd < 0) { | |
1ce6a9fc | 460 | char errmsg[STRERR_BUFSIZE]; |
24d6a808 AI |
461 | char *cp = libbpf_strerror_r(errno, errmsg, |
462 | sizeof(errmsg)); | |
1ce6a9fc TR |
463 | |
464 | pr_warning("failed to open %s: %s\n", obj->path, cp); | |
6c956392 WN |
465 | return -errno; |
466 | } | |
467 | ||
468 | obj->efile.elf = elf_begin(obj->efile.fd, | |
469 | LIBBPF_ELF_C_READ_MMAP, | |
470 | NULL); | |
1a5e3fb1 WN |
471 | } |
472 | ||
1a5e3fb1 WN |
473 | if (!obj->efile.elf) { |
474 | pr_warning("failed to open %s as ELF file\n", | |
475 | obj->path); | |
6371ca3b | 476 | err = -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
477 | goto errout; |
478 | } | |
479 | ||
480 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
481 | pr_warning("failed to get EHDR from %s\n", | |
482 | obj->path); | |
6371ca3b | 483 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
484 | goto errout; |
485 | } | |
486 | ep = &obj->efile.ehdr; | |
487 | ||
9b16137a WN |
488 | /* Old LLVM set e_machine to EM_NONE */ |
489 | if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { | |
1a5e3fb1 WN |
490 | pr_warning("%s is not an eBPF object file\n", |
491 | obj->path); | |
6371ca3b | 492 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
493 | goto errout; |
494 | } | |
495 | ||
496 | return 0; | |
497 | errout: | |
498 | bpf_object__elf_finish(obj); | |
499 | return err; | |
500 | } | |
501 | ||
cc4228d5 WN |
502 | static int |
503 | bpf_object__check_endianness(struct bpf_object *obj) | |
504 | { | |
505 | static unsigned int const endian = 1; | |
506 | ||
507 | switch (obj->efile.ehdr.e_ident[EI_DATA]) { | |
508 | case ELFDATA2LSB: | |
509 | /* We are big endian, BPF obj is little endian. */ | |
510 | if (*(unsigned char const *)&endian != 1) | |
511 | goto mismatch; | |
512 | break; | |
513 | ||
514 | case ELFDATA2MSB: | |
515 | /* We are little endian, BPF obj is big endian. */ | |
516 | if (*(unsigned char const *)&endian != 0) | |
517 | goto mismatch; | |
518 | break; | |
519 | default: | |
6371ca3b | 520 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
521 | } |
522 | ||
523 | return 0; | |
524 | ||
525 | mismatch: | |
526 | pr_warning("Error: endianness mismatch.\n"); | |
6371ca3b | 527 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
528 | } |
529 | ||
cb1e5e96 WN |
530 | static int |
531 | bpf_object__init_license(struct bpf_object *obj, | |
532 | void *data, size_t size) | |
533 | { | |
534 | memcpy(obj->license, data, | |
535 | min(size, sizeof(obj->license) - 1)); | |
536 | pr_debug("license of %s is %s\n", obj->path, obj->license); | |
537 | return 0; | |
538 | } | |
539 | ||
540 | static int | |
541 | bpf_object__init_kversion(struct bpf_object *obj, | |
542 | void *data, size_t size) | |
543 | { | |
438363c0 | 544 | __u32 kver; |
cb1e5e96 WN |
545 | |
546 | if (size != sizeof(kver)) { | |
547 | pr_warning("invalid kver section in %s\n", obj->path); | |
6371ca3b | 548 | return -LIBBPF_ERRNO__FORMAT; |
cb1e5e96 WN |
549 | } |
550 | memcpy(&kver, data, sizeof(kver)); | |
551 | obj->kern_version = kver; | |
552 | pr_debug("kernel version of %s is %x\n", obj->path, | |
553 | obj->kern_version); | |
554 | return 0; | |
555 | } | |
556 | ||
4708bbda EL |
557 | static int compare_bpf_map(const void *_a, const void *_b) |
558 | { | |
559 | const struct bpf_map *a = _a; | |
560 | const struct bpf_map *b = _b; | |
9d759a9b | 561 | |
4708bbda | 562 | return a->offset - b->offset; |
0b3d1efa WN |
563 | } |
564 | ||
973170e6 | 565 | static int |
c034a177 | 566 | bpf_object__init_maps(struct bpf_object *obj, int flags) |
561bbcca | 567 | { |
c034a177 | 568 | bool strict = !(flags & MAPS_RELAX_COMPAT); |
b13c5c14 | 569 | int i, map_idx, map_def_sz, nr_maps = 0; |
4708bbda EL |
570 | Elf_Scn *scn; |
571 | Elf_Data *data; | |
561bbcca WN |
572 | Elf_Data *symbols = obj->efile.symbols; |
573 | ||
4708bbda EL |
574 | if (obj->efile.maps_shndx < 0) |
575 | return -EINVAL; | |
576 | if (!symbols) | |
577 | return -EINVAL; | |
578 | ||
579 | scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); | |
580 | if (scn) | |
581 | data = elf_getdata(scn, NULL); | |
582 | if (!scn || !data) { | |
583 | pr_warning("failed to get Elf_Data from map section %d\n", | |
584 | obj->efile.maps_shndx); | |
973170e6 | 585 | return -EINVAL; |
4708bbda | 586 | } |
561bbcca | 587 | |
4708bbda EL |
588 | /* |
589 | * Count number of maps. Each map has a name. | |
590 | * Array of maps is not supported: only the first element is | |
591 | * considered. | |
592 | * | |
593 | * TODO: Detect array of map and report error. | |
594 | */ | |
561bbcca WN |
595 | for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { |
596 | GElf_Sym sym; | |
4708bbda EL |
597 | |
598 | if (!gelf_getsym(symbols, i, &sym)) | |
599 | continue; | |
600 | if (sym.st_shndx != obj->efile.maps_shndx) | |
601 | continue; | |
602 | nr_maps++; | |
603 | } | |
604 | ||
605 | /* Alloc obj->maps and fill nr_maps. */ | |
606 | pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, | |
607 | nr_maps, data->d_size); | |
608 | ||
609 | if (!nr_maps) | |
610 | return 0; | |
611 | ||
b13c5c14 CG |
612 | /* Assume equally sized map definitions */ |
613 | map_def_sz = data->d_size / nr_maps; | |
614 | if (!data->d_size || (data->d_size % nr_maps) != 0) { | |
615 | pr_warning("unable to determine map definition size " | |
616 | "section %s, %d maps in %zd bytes\n", | |
617 | obj->path, nr_maps, data->d_size); | |
618 | return -EINVAL; | |
619 | } | |
620 | ||
4708bbda EL |
621 | obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); |
622 | if (!obj->maps) { | |
623 | pr_warning("alloc maps for object failed\n"); | |
624 | return -ENOMEM; | |
625 | } | |
626 | obj->nr_maps = nr_maps; | |
627 | ||
628 | /* | |
629 | * fill all fd with -1 so won't close incorrect | |
630 | * fd (fd=0 is stdin) when failure (zclose won't close | |
631 | * negative fd)). | |
632 | */ | |
633 | for (i = 0; i < nr_maps; i++) | |
634 | obj->maps[i].fd = -1; | |
635 | ||
636 | /* | |
637 | * Fill obj->maps using data in "maps" section. | |
638 | */ | |
639 | for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { | |
640 | GElf_Sym sym; | |
561bbcca | 641 | const char *map_name; |
4708bbda | 642 | struct bpf_map_def *def; |
561bbcca WN |
643 | |
644 | if (!gelf_getsym(symbols, i, &sym)) | |
645 | continue; | |
666810e8 | 646 | if (sym.st_shndx != obj->efile.maps_shndx) |
561bbcca WN |
647 | continue; |
648 | ||
649 | map_name = elf_strptr(obj->efile.elf, | |
77ba9a5b | 650 | obj->efile.strtabidx, |
561bbcca | 651 | sym.st_name); |
4708bbda | 652 | obj->maps[map_idx].offset = sym.st_value; |
b13c5c14 | 653 | if (sym.st_value + map_def_sz > data->d_size) { |
4708bbda EL |
654 | pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", |
655 | obj->path, map_name); | |
656 | return -EINVAL; | |
561bbcca | 657 | } |
4708bbda | 658 | |
561bbcca | 659 | obj->maps[map_idx].name = strdup(map_name); |
973170e6 WN |
660 | if (!obj->maps[map_idx].name) { |
661 | pr_warning("failed to alloc map name\n"); | |
662 | return -ENOMEM; | |
663 | } | |
4708bbda | 664 | pr_debug("map %d is \"%s\"\n", map_idx, |
561bbcca | 665 | obj->maps[map_idx].name); |
4708bbda | 666 | def = (struct bpf_map_def *)(data->d_buf + sym.st_value); |
b13c5c14 CG |
667 | /* |
668 | * If the definition of the map in the object file fits in | |
669 | * bpf_map_def, copy it. Any extra fields in our version | |
670 | * of bpf_map_def will default to zero as a result of the | |
671 | * calloc above. | |
672 | */ | |
673 | if (map_def_sz <= sizeof(struct bpf_map_def)) { | |
674 | memcpy(&obj->maps[map_idx].def, def, map_def_sz); | |
675 | } else { | |
676 | /* | |
677 | * Here the map structure being read is bigger than what | |
678 | * we expect, truncate if the excess bits are all zero. | |
679 | * If they are not zero, reject this map as | |
680 | * incompatible. | |
681 | */ | |
682 | char *b; | |
683 | for (b = ((char *)def) + sizeof(struct bpf_map_def); | |
684 | b < ((char *)def) + map_def_sz; b++) { | |
685 | if (*b != 0) { | |
686 | pr_warning("maps section in %s: \"%s\" " | |
687 | "has unrecognized, non-zero " | |
688 | "options\n", | |
689 | obj->path, map_name); | |
c034a177 JF |
690 | if (strict) |
691 | return -EINVAL; | |
b13c5c14 CG |
692 | } |
693 | } | |
694 | memcpy(&obj->maps[map_idx].def, def, | |
695 | sizeof(struct bpf_map_def)); | |
696 | } | |
4708bbda | 697 | map_idx++; |
561bbcca | 698 | } |
4708bbda EL |
699 | |
700 | qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); | |
b13c5c14 | 701 | return 0; |
561bbcca WN |
702 | } |
703 | ||
e3d91b0c JDB |
704 | static bool section_have_execinstr(struct bpf_object *obj, int idx) |
705 | { | |
706 | Elf_Scn *scn; | |
707 | GElf_Shdr sh; | |
708 | ||
709 | scn = elf_getscn(obj->efile.elf, idx); | |
710 | if (!scn) | |
711 | return false; | |
712 | ||
713 | if (gelf_getshdr(scn, &sh) != &sh) | |
714 | return false; | |
715 | ||
716 | if (sh.sh_flags & SHF_EXECINSTR) | |
717 | return true; | |
718 | ||
719 | return false; | |
720 | } | |
721 | ||
c034a177 | 722 | static int bpf_object__elf_collect(struct bpf_object *obj, int flags) |
29603665 WN |
723 | { |
724 | Elf *elf = obj->efile.elf; | |
725 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
726 | Elf_Scn *scn = NULL; | |
666810e8 | 727 | int idx = 0, err = 0; |
29603665 WN |
728 | |
729 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
730 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
731 | pr_warning("failed to get e_shstrndx from %s\n", | |
732 | obj->path); | |
6371ca3b | 733 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
734 | } |
735 | ||
736 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
737 | char *name; | |
738 | GElf_Shdr sh; | |
739 | Elf_Data *data; | |
740 | ||
741 | idx++; | |
742 | if (gelf_getshdr(scn, &sh) != &sh) { | |
077c066a JDB |
743 | pr_warning("failed to get section(%d) header from %s\n", |
744 | idx, obj->path); | |
6371ca3b | 745 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
746 | goto out; |
747 | } | |
748 | ||
749 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
750 | if (!name) { | |
077c066a JDB |
751 | pr_warning("failed to get section(%d) name from %s\n", |
752 | idx, obj->path); | |
6371ca3b | 753 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
754 | goto out; |
755 | } | |
756 | ||
757 | data = elf_getdata(scn, 0); | |
758 | if (!data) { | |
077c066a JDB |
759 | pr_warning("failed to get section(%d) data from %s(%s)\n", |
760 | idx, name, obj->path); | |
6371ca3b | 761 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
762 | goto out; |
763 | } | |
077c066a JDB |
764 | pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", |
765 | idx, name, (unsigned long)data->d_size, | |
29603665 WN |
766 | (int)sh.sh_link, (unsigned long)sh.sh_flags, |
767 | (int)sh.sh_type); | |
cb1e5e96 WN |
768 | |
769 | if (strcmp(name, "license") == 0) | |
770 | err = bpf_object__init_license(obj, | |
771 | data->d_buf, | |
772 | data->d_size); | |
773 | else if (strcmp(name, "version") == 0) | |
774 | err = bpf_object__init_kversion(obj, | |
775 | data->d_buf, | |
776 | data->d_size); | |
4708bbda | 777 | else if (strcmp(name, "maps") == 0) |
666810e8 | 778 | obj->efile.maps_shndx = idx; |
8a138aed MKL |
779 | else if (strcmp(name, BTF_ELF_SEC) == 0) { |
780 | obj->btf = btf__new(data->d_buf, data->d_size, | |
781 | __pr_debug); | |
782 | if (IS_ERR(obj->btf)) { | |
783 | pr_warning("Error loading ELF section %s: %ld. Ignored and continue.\n", | |
784 | BTF_ELF_SEC, PTR_ERR(obj->btf)); | |
785 | obj->btf = NULL; | |
786 | } | |
787 | } else if (sh.sh_type == SHT_SYMTAB) { | |
bec7d68c WN |
788 | if (obj->efile.symbols) { |
789 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
790 | obj->path); | |
6371ca3b | 791 | err = -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 792 | } else { |
bec7d68c | 793 | obj->efile.symbols = data; |
77ba9a5b WN |
794 | obj->efile.strtabidx = sh.sh_link; |
795 | } | |
a5b8bd47 WN |
796 | } else if ((sh.sh_type == SHT_PROGBITS) && |
797 | (sh.sh_flags & SHF_EXECINSTR) && | |
798 | (data->d_size > 0)) { | |
48cca7e4 AS |
799 | if (strcmp(name, ".text") == 0) |
800 | obj->efile.text_shndx = idx; | |
a5b8bd47 WN |
801 | err = bpf_object__add_program(obj, data->d_buf, |
802 | data->d_size, name, idx); | |
803 | if (err) { | |
6371ca3b | 804 | char errmsg[STRERR_BUFSIZE]; |
24d6a808 AI |
805 | char *cp = libbpf_strerror_r(-err, errmsg, |
806 | sizeof(errmsg)); | |
6371ca3b | 807 | |
a5b8bd47 | 808 | pr_warning("failed to alloc program %s (%s): %s", |
1ce6a9fc | 809 | name, obj->path, cp); |
a5b8bd47 | 810 | } |
b62f06e8 WN |
811 | } else if (sh.sh_type == SHT_REL) { |
812 | void *reloc = obj->efile.reloc; | |
813 | int nr_reloc = obj->efile.nr_reloc + 1; | |
e3d91b0c JDB |
814 | int sec = sh.sh_info; /* points to other section */ |
815 | ||
816 | /* Only do relo for section with exec instructions */ | |
817 | if (!section_have_execinstr(obj, sec)) { | |
818 | pr_debug("skip relo %s(%d) for section(%d)\n", | |
819 | name, idx, sec); | |
820 | continue; | |
821 | } | |
b62f06e8 | 822 | |
531b014e JK |
823 | reloc = reallocarray(reloc, nr_reloc, |
824 | sizeof(*obj->efile.reloc)); | |
b62f06e8 WN |
825 | if (!reloc) { |
826 | pr_warning("realloc failed\n"); | |
827 | err = -ENOMEM; | |
828 | } else { | |
829 | int n = nr_reloc - 1; | |
830 | ||
831 | obj->efile.reloc = reloc; | |
832 | obj->efile.nr_reloc = nr_reloc; | |
833 | ||
834 | obj->efile.reloc[n].shdr = sh; | |
835 | obj->efile.reloc[n].data = data; | |
836 | } | |
077c066a JDB |
837 | } else { |
838 | pr_debug("skip section(%d) %s\n", idx, name); | |
bec7d68c | 839 | } |
cb1e5e96 WN |
840 | if (err) |
841 | goto out; | |
29603665 | 842 | } |
561bbcca | 843 | |
77ba9a5b WN |
844 | if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { |
845 | pr_warning("Corrupted ELF file: index of strtab invalid\n"); | |
846 | return LIBBPF_ERRNO__FORMAT; | |
847 | } | |
88cda1c9 | 848 | if (obj->efile.maps_shndx >= 0) { |
c034a177 | 849 | err = bpf_object__init_maps(obj, flags); |
88cda1c9 MKL |
850 | if (err) |
851 | goto out; | |
852 | } | |
853 | err = bpf_object__init_prog_names(obj); | |
29603665 WN |
854 | out: |
855 | return err; | |
856 | } | |
857 | ||
34090915 WN |
858 | static struct bpf_program * |
859 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
860 | { | |
861 | struct bpf_program *prog; | |
862 | size_t i; | |
863 | ||
864 | for (i = 0; i < obj->nr_programs; i++) { | |
865 | prog = &obj->programs[i]; | |
866 | if (prog->idx == idx) | |
867 | return prog; | |
868 | } | |
869 | return NULL; | |
870 | } | |
871 | ||
6d4b198b JK |
872 | struct bpf_program * |
873 | bpf_object__find_program_by_title(struct bpf_object *obj, const char *title) | |
874 | { | |
875 | struct bpf_program *pos; | |
876 | ||
877 | bpf_object__for_each_program(pos, obj) { | |
878 | if (pos->section_name && !strcmp(pos->section_name, title)) | |
879 | return pos; | |
880 | } | |
881 | return NULL; | |
882 | } | |
883 | ||
34090915 | 884 | static int |
48cca7e4 AS |
885 | bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, |
886 | Elf_Data *data, struct bpf_object *obj) | |
34090915 | 887 | { |
48cca7e4 AS |
888 | Elf_Data *symbols = obj->efile.symbols; |
889 | int text_shndx = obj->efile.text_shndx; | |
890 | int maps_shndx = obj->efile.maps_shndx; | |
891 | struct bpf_map *maps = obj->maps; | |
892 | size_t nr_maps = obj->nr_maps; | |
34090915 WN |
893 | int i, nrels; |
894 | ||
895 | pr_debug("collecting relocating info for: '%s'\n", | |
896 | prog->section_name); | |
897 | nrels = shdr->sh_size / shdr->sh_entsize; | |
898 | ||
899 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
900 | if (!prog->reloc_desc) { | |
901 | pr_warning("failed to alloc memory in relocation\n"); | |
902 | return -ENOMEM; | |
903 | } | |
904 | prog->nr_reloc = nrels; | |
905 | ||
906 | for (i = 0; i < nrels; i++) { | |
907 | GElf_Sym sym; | |
908 | GElf_Rel rel; | |
909 | unsigned int insn_idx; | |
910 | struct bpf_insn *insns = prog->insns; | |
911 | size_t map_idx; | |
912 | ||
913 | if (!gelf_getrel(data, i, &rel)) { | |
914 | pr_warning("relocation: failed to get %d reloc\n", i); | |
6371ca3b | 915 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
916 | } |
917 | ||
34090915 WN |
918 | if (!gelf_getsym(symbols, |
919 | GELF_R_SYM(rel.r_info), | |
920 | &sym)) { | |
921 | pr_warning("relocation: symbol %"PRIx64" not found\n", | |
922 | GELF_R_SYM(rel.r_info)); | |
6371ca3b | 923 | return -LIBBPF_ERRNO__FORMAT; |
34090915 | 924 | } |
7d9890ef DM |
925 | pr_debug("relo for %lld value %lld name %d\n", |
926 | (long long) (rel.r_info >> 32), | |
927 | (long long) sym.st_value, sym.st_name); | |
34090915 | 928 | |
48cca7e4 | 929 | if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { |
666810e8 WN |
930 | pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", |
931 | prog->section_name, sym.st_shndx); | |
932 | return -LIBBPF_ERRNO__RELOC; | |
933 | } | |
934 | ||
935 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); | |
936 | pr_debug("relocation: insn_idx=%u\n", insn_idx); | |
937 | ||
48cca7e4 AS |
938 | if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { |
939 | if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { | |
940 | pr_warning("incorrect bpf_call opcode\n"); | |
941 | return -LIBBPF_ERRNO__RELOC; | |
942 | } | |
943 | prog->reloc_desc[i].type = RELO_CALL; | |
944 | prog->reloc_desc[i].insn_idx = insn_idx; | |
945 | prog->reloc_desc[i].text_off = sym.st_value; | |
9a94f277 | 946 | obj->has_pseudo_calls = true; |
48cca7e4 AS |
947 | continue; |
948 | } | |
949 | ||
34090915 WN |
950 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { |
951 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
952 | insn_idx, insns[insn_idx].code); | |
6371ca3b | 953 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
954 | } |
955 | ||
94e5adec JS |
956 | /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ |
957 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { | |
958 | if (maps[map_idx].offset == sym.st_value) { | |
959 | pr_debug("relocation: find map %zd (%s) for insn %u\n", | |
960 | map_idx, maps[map_idx].name, insn_idx); | |
961 | break; | |
962 | } | |
963 | } | |
964 | ||
34090915 WN |
965 | if (map_idx >= nr_maps) { |
966 | pr_warning("bpf relocation: map_idx %d large than %d\n", | |
967 | (int)map_idx, (int)nr_maps - 1); | |
6371ca3b | 968 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
969 | } |
970 | ||
48cca7e4 | 971 | prog->reloc_desc[i].type = RELO_LD64; |
34090915 WN |
972 | prog->reloc_desc[i].insn_idx = insn_idx; |
973 | prog->reloc_desc[i].map_idx = map_idx; | |
974 | } | |
975 | return 0; | |
976 | } | |
977 | ||
8a138aed MKL |
978 | static int bpf_map_find_btf_info(struct bpf_map *map, const struct btf *btf) |
979 | { | |
38d5d3b3 MKL |
980 | const struct btf_type *container_type; |
981 | const struct btf_member *key, *value; | |
8a138aed MKL |
982 | struct bpf_map_def *def = &map->def; |
983 | const size_t max_name = 256; | |
38d5d3b3 | 984 | char container_name[max_name]; |
5b891af7 | 985 | __s64 key_size, value_size; |
38d5d3b3 | 986 | __s32 container_id; |
8a138aed | 987 | |
38d5d3b3 MKL |
988 | if (snprintf(container_name, max_name, "____btf_map_%s", map->name) == |
989 | max_name) { | |
990 | pr_warning("map:%s length of '____btf_map_%s' is too long\n", | |
8a138aed MKL |
991 | map->name, map->name); |
992 | return -EINVAL; | |
993 | } | |
994 | ||
38d5d3b3 MKL |
995 | container_id = btf__find_by_name(btf, container_name); |
996 | if (container_id < 0) { | |
997 | pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n", | |
998 | map->name, container_name); | |
999 | return container_id; | |
8a138aed MKL |
1000 | } |
1001 | ||
38d5d3b3 MKL |
1002 | container_type = btf__type_by_id(btf, container_id); |
1003 | if (!container_type) { | |
1004 | pr_warning("map:%s cannot find BTF type for container_id:%u\n", | |
1005 | map->name, container_id); | |
1006 | return -EINVAL; | |
8a138aed MKL |
1007 | } |
1008 | ||
38d5d3b3 MKL |
1009 | if (BTF_INFO_KIND(container_type->info) != BTF_KIND_STRUCT || |
1010 | BTF_INFO_VLEN(container_type->info) < 2) { | |
1011 | pr_warning("map:%s container_name:%s is an invalid container struct\n", | |
1012 | map->name, container_name); | |
8a138aed MKL |
1013 | return -EINVAL; |
1014 | } | |
1015 | ||
38d5d3b3 MKL |
1016 | key = (struct btf_member *)(container_type + 1); |
1017 | value = key + 1; | |
1018 | ||
1019 | key_size = btf__resolve_size(btf, key->type); | |
1020 | if (key_size < 0) { | |
1021 | pr_warning("map:%s invalid BTF key_type_size\n", | |
1022 | map->name); | |
1023 | return key_size; | |
8a138aed MKL |
1024 | } |
1025 | ||
38d5d3b3 MKL |
1026 | if (def->key_size != key_size) { |
1027 | pr_warning("map:%s btf_key_type_size:%u != map_def_key_size:%u\n", | |
1028 | map->name, (__u32)key_size, def->key_size); | |
1029 | return -EINVAL; | |
8a138aed MKL |
1030 | } |
1031 | ||
38d5d3b3 | 1032 | value_size = btf__resolve_size(btf, value->type); |
8a138aed | 1033 | if (value_size < 0) { |
38d5d3b3 | 1034 | pr_warning("map:%s invalid BTF value_type_size\n", map->name); |
8a138aed MKL |
1035 | return value_size; |
1036 | } | |
1037 | ||
1038 | if (def->value_size != value_size) { | |
38d5d3b3 MKL |
1039 | pr_warning("map:%s btf_value_type_size:%u != map_def_value_size:%u\n", |
1040 | map->name, (__u32)value_size, def->value_size); | |
8a138aed MKL |
1041 | return -EINVAL; |
1042 | } | |
1043 | ||
38d5d3b3 MKL |
1044 | map->btf_key_type_id = key->type; |
1045 | map->btf_value_type_id = value->type; | |
8a138aed MKL |
1046 | |
1047 | return 0; | |
1048 | } | |
1049 | ||
26736eb9 JK |
1050 | int bpf_map__reuse_fd(struct bpf_map *map, int fd) |
1051 | { | |
1052 | struct bpf_map_info info = {}; | |
1053 | __u32 len = sizeof(info); | |
1054 | int new_fd, err; | |
1055 | char *new_name; | |
1056 | ||
1057 | err = bpf_obj_get_info_by_fd(fd, &info, &len); | |
1058 | if (err) | |
1059 | return err; | |
1060 | ||
1061 | new_name = strdup(info.name); | |
1062 | if (!new_name) | |
1063 | return -errno; | |
1064 | ||
1065 | new_fd = open("/", O_RDONLY | O_CLOEXEC); | |
1066 | if (new_fd < 0) | |
1067 | goto err_free_new_name; | |
1068 | ||
1069 | new_fd = dup3(fd, new_fd, O_CLOEXEC); | |
1070 | if (new_fd < 0) | |
1071 | goto err_close_new_fd; | |
1072 | ||
1073 | err = zclose(map->fd); | |
1074 | if (err) | |
1075 | goto err_close_new_fd; | |
1076 | free(map->name); | |
1077 | ||
1078 | map->fd = new_fd; | |
1079 | map->name = new_name; | |
1080 | map->def.type = info.type; | |
1081 | map->def.key_size = info.key_size; | |
1082 | map->def.value_size = info.value_size; | |
1083 | map->def.max_entries = info.max_entries; | |
1084 | map->def.map_flags = info.map_flags; | |
1085 | map->btf_key_type_id = info.btf_key_type_id; | |
1086 | map->btf_value_type_id = info.btf_value_type_id; | |
1087 | ||
1088 | return 0; | |
1089 | ||
1090 | err_close_new_fd: | |
1091 | close(new_fd); | |
1092 | err_free_new_name: | |
1093 | free(new_name); | |
1094 | return -errno; | |
1095 | } | |
1096 | ||
52d3352e WN |
1097 | static int |
1098 | bpf_object__create_maps(struct bpf_object *obj) | |
1099 | { | |
8a138aed | 1100 | struct bpf_create_map_attr create_attr = {}; |
52d3352e | 1101 | unsigned int i; |
8a138aed | 1102 | int err; |
52d3352e | 1103 | |
9d759a9b | 1104 | for (i = 0; i < obj->nr_maps; i++) { |
8a138aed MKL |
1105 | struct bpf_map *map = &obj->maps[i]; |
1106 | struct bpf_map_def *def = &map->def; | |
1ce6a9fc | 1107 | char *cp, errmsg[STRERR_BUFSIZE]; |
8a138aed MKL |
1108 | int *pfd = &map->fd; |
1109 | ||
26736eb9 JK |
1110 | if (map->fd >= 0) { |
1111 | pr_debug("skip map create (preset) %s: fd=%d\n", | |
1112 | map->name, map->fd); | |
1113 | continue; | |
1114 | } | |
1115 | ||
8a138aed | 1116 | create_attr.name = map->name; |
f0307a7e | 1117 | create_attr.map_ifindex = map->map_ifindex; |
8a138aed MKL |
1118 | create_attr.map_type = def->type; |
1119 | create_attr.map_flags = def->map_flags; | |
1120 | create_attr.key_size = def->key_size; | |
1121 | create_attr.value_size = def->value_size; | |
1122 | create_attr.max_entries = def->max_entries; | |
1123 | create_attr.btf_fd = 0; | |
61746dbe MKL |
1124 | create_attr.btf_key_type_id = 0; |
1125 | create_attr.btf_value_type_id = 0; | |
8a138aed MKL |
1126 | |
1127 | if (obj->btf && !bpf_map_find_btf_info(map, obj->btf)) { | |
1128 | create_attr.btf_fd = btf__fd(obj->btf); | |
61746dbe MKL |
1129 | create_attr.btf_key_type_id = map->btf_key_type_id; |
1130 | create_attr.btf_value_type_id = map->btf_value_type_id; | |
8a138aed MKL |
1131 | } |
1132 | ||
1133 | *pfd = bpf_create_map_xattr(&create_attr); | |
61746dbe | 1134 | if (*pfd < 0 && create_attr.btf_key_type_id) { |
24d6a808 | 1135 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
8a138aed | 1136 | pr_warning("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", |
1ce6a9fc | 1137 | map->name, cp, errno); |
8a138aed | 1138 | create_attr.btf_fd = 0; |
61746dbe MKL |
1139 | create_attr.btf_key_type_id = 0; |
1140 | create_attr.btf_value_type_id = 0; | |
1141 | map->btf_key_type_id = 0; | |
1142 | map->btf_value_type_id = 0; | |
8a138aed MKL |
1143 | *pfd = bpf_create_map_xattr(&create_attr); |
1144 | } | |
1145 | ||
52d3352e WN |
1146 | if (*pfd < 0) { |
1147 | size_t j; | |
52d3352e | 1148 | |
8a138aed | 1149 | err = *pfd; |
24d6a808 | 1150 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
49bf4b36 | 1151 | pr_warning("failed to create map (name: '%s'): %s\n", |
1ce6a9fc | 1152 | map->name, cp); |
52d3352e | 1153 | for (j = 0; j < i; j++) |
9d759a9b | 1154 | zclose(obj->maps[j].fd); |
52d3352e WN |
1155 | return err; |
1156 | } | |
8a138aed | 1157 | pr_debug("create map %s: fd=%d\n", map->name, *pfd); |
52d3352e WN |
1158 | } |
1159 | ||
52d3352e WN |
1160 | return 0; |
1161 | } | |
1162 | ||
48cca7e4 AS |
1163 | static int |
1164 | bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, | |
1165 | struct reloc_desc *relo) | |
1166 | { | |
1167 | struct bpf_insn *insn, *new_insn; | |
1168 | struct bpf_program *text; | |
1169 | size_t new_cnt; | |
1170 | ||
1171 | if (relo->type != RELO_CALL) | |
1172 | return -LIBBPF_ERRNO__RELOC; | |
1173 | ||
1174 | if (prog->idx == obj->efile.text_shndx) { | |
1175 | pr_warning("relo in .text insn %d into off %d\n", | |
1176 | relo->insn_idx, relo->text_off); | |
1177 | return -LIBBPF_ERRNO__RELOC; | |
1178 | } | |
1179 | ||
1180 | if (prog->main_prog_cnt == 0) { | |
1181 | text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); | |
1182 | if (!text) { | |
1183 | pr_warning("no .text section found yet relo into text exist\n"); | |
1184 | return -LIBBPF_ERRNO__RELOC; | |
1185 | } | |
1186 | new_cnt = prog->insns_cnt + text->insns_cnt; | |
531b014e | 1187 | new_insn = reallocarray(prog->insns, new_cnt, sizeof(*insn)); |
48cca7e4 AS |
1188 | if (!new_insn) { |
1189 | pr_warning("oom in prog realloc\n"); | |
1190 | return -ENOMEM; | |
1191 | } | |
1192 | memcpy(new_insn + prog->insns_cnt, text->insns, | |
1193 | text->insns_cnt * sizeof(*insn)); | |
1194 | prog->insns = new_insn; | |
1195 | prog->main_prog_cnt = prog->insns_cnt; | |
1196 | prog->insns_cnt = new_cnt; | |
b1a2ce82 JC |
1197 | pr_debug("added %zd insn from %s to prog %s\n", |
1198 | text->insns_cnt, text->section_name, | |
1199 | prog->section_name); | |
48cca7e4 AS |
1200 | } |
1201 | insn = &prog->insns[relo->insn_idx]; | |
1202 | insn->imm += prog->main_prog_cnt - relo->insn_idx; | |
48cca7e4 AS |
1203 | return 0; |
1204 | } | |
1205 | ||
8a47a6c5 | 1206 | static int |
9d759a9b | 1207 | bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) |
8a47a6c5 | 1208 | { |
48cca7e4 | 1209 | int i, err; |
8a47a6c5 WN |
1210 | |
1211 | if (!prog || !prog->reloc_desc) | |
1212 | return 0; | |
1213 | ||
1214 | for (i = 0; i < prog->nr_reloc; i++) { | |
48cca7e4 AS |
1215 | if (prog->reloc_desc[i].type == RELO_LD64) { |
1216 | struct bpf_insn *insns = prog->insns; | |
1217 | int insn_idx, map_idx; | |
8a47a6c5 | 1218 | |
48cca7e4 AS |
1219 | insn_idx = prog->reloc_desc[i].insn_idx; |
1220 | map_idx = prog->reloc_desc[i].map_idx; | |
8a47a6c5 | 1221 | |
48cca7e4 AS |
1222 | if (insn_idx >= (int)prog->insns_cnt) { |
1223 | pr_warning("relocation out of range: '%s'\n", | |
1224 | prog->section_name); | |
1225 | return -LIBBPF_ERRNO__RELOC; | |
1226 | } | |
1227 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
1228 | insns[insn_idx].imm = obj->maps[map_idx].fd; | |
1229 | } else { | |
1230 | err = bpf_program__reloc_text(prog, obj, | |
1231 | &prog->reloc_desc[i]); | |
1232 | if (err) | |
1233 | return err; | |
8a47a6c5 | 1234 | } |
8a47a6c5 WN |
1235 | } |
1236 | ||
1237 | zfree(&prog->reloc_desc); | |
1238 | prog->nr_reloc = 0; | |
1239 | return 0; | |
1240 | } | |
1241 | ||
1242 | ||
1243 | static int | |
1244 | bpf_object__relocate(struct bpf_object *obj) | |
1245 | { | |
1246 | struct bpf_program *prog; | |
1247 | size_t i; | |
1248 | int err; | |
1249 | ||
1250 | for (i = 0; i < obj->nr_programs; i++) { | |
1251 | prog = &obj->programs[i]; | |
1252 | ||
9d759a9b | 1253 | err = bpf_program__relocate(prog, obj); |
8a47a6c5 WN |
1254 | if (err) { |
1255 | pr_warning("failed to relocate '%s'\n", | |
1256 | prog->section_name); | |
1257 | return err; | |
1258 | } | |
1259 | } | |
1260 | return 0; | |
1261 | } | |
1262 | ||
34090915 WN |
1263 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
1264 | { | |
1265 | int i, err; | |
1266 | ||
1267 | if (!obj_elf_valid(obj)) { | |
1268 | pr_warning("Internal error: elf object is closed\n"); | |
6371ca3b | 1269 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
1270 | } |
1271 | ||
1272 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
1273 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
1274 | Elf_Data *data = obj->efile.reloc[i].data; | |
1275 | int idx = shdr->sh_info; | |
1276 | struct bpf_program *prog; | |
34090915 WN |
1277 | |
1278 | if (shdr->sh_type != SHT_REL) { | |
1279 | pr_warning("internal error at %d\n", __LINE__); | |
6371ca3b | 1280 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
1281 | } |
1282 | ||
1283 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
1284 | if (!prog) { | |
077c066a | 1285 | pr_warning("relocation failed: no section(%d)\n", idx); |
6371ca3b | 1286 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
1287 | } |
1288 | ||
48cca7e4 | 1289 | err = bpf_program__collect_reloc(prog, |
34090915 | 1290 | shdr, data, |
48cca7e4 | 1291 | obj); |
34090915 | 1292 | if (err) |
6371ca3b | 1293 | return err; |
34090915 WN |
1294 | } |
1295 | return 0; | |
1296 | } | |
1297 | ||
55cffde2 | 1298 | static int |
d7be143b AI |
1299 | load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type, |
1300 | const char *name, struct bpf_insn *insns, int insns_cnt, | |
438363c0 | 1301 | char *license, __u32 kern_version, int *pfd, int prog_ifindex) |
55cffde2 | 1302 | { |
d7be143b | 1303 | struct bpf_load_program_attr load_attr; |
1ce6a9fc | 1304 | char *cp, errmsg[STRERR_BUFSIZE]; |
55cffde2 | 1305 | char *log_buf; |
d7be143b | 1306 | int ret; |
55cffde2 | 1307 | |
d7be143b AI |
1308 | memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); |
1309 | load_attr.prog_type = type; | |
1310 | load_attr.expected_attach_type = expected_attach_type; | |
1311 | load_attr.name = name; | |
1312 | load_attr.insns = insns; | |
1313 | load_attr.insns_cnt = insns_cnt; | |
1314 | load_attr.license = license; | |
1315 | load_attr.kern_version = kern_version; | |
f0307a7e | 1316 | load_attr.prog_ifindex = prog_ifindex; |
d7be143b AI |
1317 | |
1318 | if (!load_attr.insns || !load_attr.insns_cnt) | |
55cffde2 WN |
1319 | return -EINVAL; |
1320 | ||
1321 | log_buf = malloc(BPF_LOG_BUF_SIZE); | |
1322 | if (!log_buf) | |
1323 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
1324 | ||
d7be143b | 1325 | ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE); |
55cffde2 WN |
1326 | |
1327 | if (ret >= 0) { | |
1328 | *pfd = ret; | |
1329 | ret = 0; | |
1330 | goto out; | |
1331 | } | |
1332 | ||
6371ca3b | 1333 | ret = -LIBBPF_ERRNO__LOAD; |
24d6a808 | 1334 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 1335 | pr_warning("load bpf program failed: %s\n", cp); |
55cffde2 | 1336 | |
6371ca3b WN |
1337 | if (log_buf && log_buf[0] != '\0') { |
1338 | ret = -LIBBPF_ERRNO__VERIFY; | |
55cffde2 WN |
1339 | pr_warning("-- BEGIN DUMP LOG ---\n"); |
1340 | pr_warning("\n%s\n", log_buf); | |
1341 | pr_warning("-- END LOG --\n"); | |
d7be143b AI |
1342 | } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { |
1343 | pr_warning("Program too large (%zu insns), at most %d insns\n", | |
1344 | load_attr.insns_cnt, BPF_MAXINSNS); | |
705fa219 | 1345 | ret = -LIBBPF_ERRNO__PROG2BIG; |
6371ca3b | 1346 | } else { |
705fa219 | 1347 | /* Wrong program type? */ |
d7be143b | 1348 | if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { |
705fa219 WN |
1349 | int fd; |
1350 | ||
d7be143b AI |
1351 | load_attr.prog_type = BPF_PROG_TYPE_KPROBE; |
1352 | load_attr.expected_attach_type = 0; | |
1353 | fd = bpf_load_program_xattr(&load_attr, NULL, 0); | |
705fa219 WN |
1354 | if (fd >= 0) { |
1355 | close(fd); | |
1356 | ret = -LIBBPF_ERRNO__PROGTYPE; | |
1357 | goto out; | |
1358 | } | |
6371ca3b | 1359 | } |
705fa219 WN |
1360 | |
1361 | if (log_buf) | |
1362 | ret = -LIBBPF_ERRNO__KVER; | |
55cffde2 WN |
1363 | } |
1364 | ||
1365 | out: | |
1366 | free(log_buf); | |
1367 | return ret; | |
1368 | } | |
1369 | ||
29cd77f4 | 1370 | int |
55cffde2 | 1371 | bpf_program__load(struct bpf_program *prog, |
e5b0863c | 1372 | char *license, __u32 kern_version) |
55cffde2 | 1373 | { |
b580563e | 1374 | int err = 0, fd, i; |
55cffde2 | 1375 | |
b580563e WN |
1376 | if (prog->instances.nr < 0 || !prog->instances.fds) { |
1377 | if (prog->preprocessor) { | |
1378 | pr_warning("Internal error: can't load program '%s'\n", | |
1379 | prog->section_name); | |
1380 | return -LIBBPF_ERRNO__INTERNAL; | |
1381 | } | |
55cffde2 | 1382 | |
b580563e WN |
1383 | prog->instances.fds = malloc(sizeof(int)); |
1384 | if (!prog->instances.fds) { | |
1385 | pr_warning("Not enough memory for BPF fds\n"); | |
1386 | return -ENOMEM; | |
1387 | } | |
1388 | prog->instances.nr = 1; | |
1389 | prog->instances.fds[0] = -1; | |
1390 | } | |
1391 | ||
1392 | if (!prog->preprocessor) { | |
1393 | if (prog->instances.nr != 1) { | |
1394 | pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", | |
1395 | prog->section_name, prog->instances.nr); | |
1396 | } | |
d7be143b AI |
1397 | err = load_program(prog->type, prog->expected_attach_type, |
1398 | prog->name, prog->insns, prog->insns_cnt, | |
f0307a7e DB |
1399 | license, kern_version, &fd, |
1400 | prog->prog_ifindex); | |
b580563e WN |
1401 | if (!err) |
1402 | prog->instances.fds[0] = fd; | |
1403 | goto out; | |
1404 | } | |
1405 | ||
1406 | for (i = 0; i < prog->instances.nr; i++) { | |
1407 | struct bpf_prog_prep_result result; | |
1408 | bpf_program_prep_t preprocessor = prog->preprocessor; | |
1409 | ||
1410 | bzero(&result, sizeof(result)); | |
1411 | err = preprocessor(prog, i, prog->insns, | |
1412 | prog->insns_cnt, &result); | |
1413 | if (err) { | |
1414 | pr_warning("Preprocessing the %dth instance of program '%s' failed\n", | |
1415 | i, prog->section_name); | |
1416 | goto out; | |
1417 | } | |
1418 | ||
1419 | if (!result.new_insn_ptr || !result.new_insn_cnt) { | |
1420 | pr_debug("Skip loading the %dth instance of program '%s'\n", | |
1421 | i, prog->section_name); | |
1422 | prog->instances.fds[i] = -1; | |
1423 | if (result.pfd) | |
1424 | *result.pfd = -1; | |
1425 | continue; | |
1426 | } | |
1427 | ||
d7be143b AI |
1428 | err = load_program(prog->type, prog->expected_attach_type, |
1429 | prog->name, result.new_insn_ptr, | |
b580563e | 1430 | result.new_insn_cnt, |
f0307a7e DB |
1431 | license, kern_version, &fd, |
1432 | prog->prog_ifindex); | |
b580563e WN |
1433 | |
1434 | if (err) { | |
1435 | pr_warning("Loading the %dth instance of program '%s' failed\n", | |
1436 | i, prog->section_name); | |
1437 | goto out; | |
1438 | } | |
1439 | ||
1440 | if (result.pfd) | |
1441 | *result.pfd = fd; | |
1442 | prog->instances.fds[i] = fd; | |
1443 | } | |
1444 | out: | |
55cffde2 WN |
1445 | if (err) |
1446 | pr_warning("failed to load program '%s'\n", | |
1447 | prog->section_name); | |
1448 | zfree(&prog->insns); | |
1449 | prog->insns_cnt = 0; | |
1450 | return err; | |
1451 | } | |
1452 | ||
9a94f277 JK |
1453 | static bool bpf_program__is_function_storage(struct bpf_program *prog, |
1454 | struct bpf_object *obj) | |
1455 | { | |
1456 | return prog->idx == obj->efile.text_shndx && obj->has_pseudo_calls; | |
1457 | } | |
1458 | ||
55cffde2 WN |
1459 | static int |
1460 | bpf_object__load_progs(struct bpf_object *obj) | |
1461 | { | |
1462 | size_t i; | |
1463 | int err; | |
1464 | ||
1465 | for (i = 0; i < obj->nr_programs; i++) { | |
9a94f277 | 1466 | if (bpf_program__is_function_storage(&obj->programs[i], obj)) |
48cca7e4 | 1467 | continue; |
55cffde2 WN |
1468 | err = bpf_program__load(&obj->programs[i], |
1469 | obj->license, | |
1470 | obj->kern_version); | |
1471 | if (err) | |
1472 | return err; | |
1473 | } | |
1474 | return 0; | |
1475 | } | |
1476 | ||
17387dd5 JK |
1477 | static bool bpf_prog_type__needs_kver(enum bpf_prog_type type) |
1478 | { | |
1479 | switch (type) { | |
1480 | case BPF_PROG_TYPE_SOCKET_FILTER: | |
1481 | case BPF_PROG_TYPE_SCHED_CLS: | |
1482 | case BPF_PROG_TYPE_SCHED_ACT: | |
1483 | case BPF_PROG_TYPE_XDP: | |
1484 | case BPF_PROG_TYPE_CGROUP_SKB: | |
1485 | case BPF_PROG_TYPE_CGROUP_SOCK: | |
1486 | case BPF_PROG_TYPE_LWT_IN: | |
1487 | case BPF_PROG_TYPE_LWT_OUT: | |
1488 | case BPF_PROG_TYPE_LWT_XMIT: | |
004d4b27 | 1489 | case BPF_PROG_TYPE_LWT_SEG6LOCAL: |
17387dd5 JK |
1490 | case BPF_PROG_TYPE_SOCK_OPS: |
1491 | case BPF_PROG_TYPE_SK_SKB: | |
1492 | case BPF_PROG_TYPE_CGROUP_DEVICE: | |
1493 | case BPF_PROG_TYPE_SK_MSG: | |
1494 | case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: | |
6bdd533c | 1495 | case BPF_PROG_TYPE_LIRC_MODE2: |
6bc8529c | 1496 | case BPF_PROG_TYPE_SK_REUSEPORT: |
c22fbae7 | 1497 | case BPF_PROG_TYPE_FLOW_DISSECTOR: |
17387dd5 JK |
1498 | return false; |
1499 | case BPF_PROG_TYPE_UNSPEC: | |
1500 | case BPF_PROG_TYPE_KPROBE: | |
1501 | case BPF_PROG_TYPE_TRACEPOINT: | |
1502 | case BPF_PROG_TYPE_PERF_EVENT: | |
1503 | case BPF_PROG_TYPE_RAW_TRACEPOINT: | |
1504 | default: | |
1505 | return true; | |
1506 | } | |
1507 | } | |
1508 | ||
1509 | static int bpf_object__validate(struct bpf_object *obj, bool needs_kver) | |
cb1e5e96 | 1510 | { |
17387dd5 | 1511 | if (needs_kver && obj->kern_version == 0) { |
cb1e5e96 WN |
1512 | pr_warning("%s doesn't provide kernel version\n", |
1513 | obj->path); | |
6371ca3b | 1514 | return -LIBBPF_ERRNO__KVERSION; |
cb1e5e96 WN |
1515 | } |
1516 | return 0; | |
1517 | } | |
1518 | ||
1a5e3fb1 | 1519 | static struct bpf_object * |
17387dd5 | 1520 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz, |
c034a177 | 1521 | bool needs_kver, int flags) |
1a5e3fb1 WN |
1522 | { |
1523 | struct bpf_object *obj; | |
6371ca3b | 1524 | int err; |
1a5e3fb1 WN |
1525 | |
1526 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
1527 | pr_warning("failed to init libelf for %s\n", path); | |
6371ca3b | 1528 | return ERR_PTR(-LIBBPF_ERRNO__LIBELF); |
1a5e3fb1 WN |
1529 | } |
1530 | ||
6c956392 | 1531 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
6371ca3b WN |
1532 | if (IS_ERR(obj)) |
1533 | return obj; | |
1a5e3fb1 | 1534 | |
6371ca3b WN |
1535 | CHECK_ERR(bpf_object__elf_init(obj), err, out); |
1536 | CHECK_ERR(bpf_object__check_endianness(obj), err, out); | |
c034a177 | 1537 | CHECK_ERR(bpf_object__elf_collect(obj, flags), err, out); |
6371ca3b | 1538 | CHECK_ERR(bpf_object__collect_reloc(obj), err, out); |
17387dd5 | 1539 | CHECK_ERR(bpf_object__validate(obj, needs_kver), err, out); |
1a5e3fb1 WN |
1540 | |
1541 | bpf_object__elf_finish(obj); | |
1542 | return obj; | |
1543 | out: | |
1544 | bpf_object__close(obj); | |
6371ca3b | 1545 | return ERR_PTR(err); |
1a5e3fb1 WN |
1546 | } |
1547 | ||
c034a177 JF |
1548 | struct bpf_object *__bpf_object__open_xattr(struct bpf_object_open_attr *attr, |
1549 | int flags) | |
1a5e3fb1 WN |
1550 | { |
1551 | /* param validation */ | |
07f2d4ea | 1552 | if (!attr->file) |
1a5e3fb1 WN |
1553 | return NULL; |
1554 | ||
07f2d4ea JK |
1555 | pr_debug("loading %s\n", attr->file); |
1556 | ||
1557 | return __bpf_object__open(attr->file, NULL, 0, | |
c034a177 JF |
1558 | bpf_prog_type__needs_kver(attr->prog_type), |
1559 | flags); | |
1560 | } | |
1561 | ||
1562 | struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) | |
1563 | { | |
1564 | return __bpf_object__open_xattr(attr, 0); | |
07f2d4ea JK |
1565 | } |
1566 | ||
1567 | struct bpf_object *bpf_object__open(const char *path) | |
1568 | { | |
1569 | struct bpf_object_open_attr attr = { | |
1570 | .file = path, | |
1571 | .prog_type = BPF_PROG_TYPE_UNSPEC, | |
1572 | }; | |
1a5e3fb1 | 1573 | |
07f2d4ea | 1574 | return bpf_object__open_xattr(&attr); |
6c956392 WN |
1575 | } |
1576 | ||
1577 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
acf860ae WN |
1578 | size_t obj_buf_sz, |
1579 | const char *name) | |
6c956392 | 1580 | { |
acf860ae WN |
1581 | char tmp_name[64]; |
1582 | ||
6c956392 WN |
1583 | /* param validation */ |
1584 | if (!obj_buf || obj_buf_sz <= 0) | |
1585 | return NULL; | |
1586 | ||
acf860ae WN |
1587 | if (!name) { |
1588 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
1589 | (unsigned long)obj_buf, | |
1590 | (unsigned long)obj_buf_sz); | |
1591 | tmp_name[sizeof(tmp_name) - 1] = '\0'; | |
1592 | name = tmp_name; | |
1593 | } | |
1594 | pr_debug("loading object '%s' from buffer\n", | |
1595 | name); | |
6c956392 | 1596 | |
c034a177 | 1597 | return __bpf_object__open(name, obj_buf, obj_buf_sz, true, true); |
1a5e3fb1 WN |
1598 | } |
1599 | ||
52d3352e WN |
1600 | int bpf_object__unload(struct bpf_object *obj) |
1601 | { | |
1602 | size_t i; | |
1603 | ||
1604 | if (!obj) | |
1605 | return -EINVAL; | |
1606 | ||
9d759a9b WN |
1607 | for (i = 0; i < obj->nr_maps; i++) |
1608 | zclose(obj->maps[i].fd); | |
52d3352e | 1609 | |
55cffde2 WN |
1610 | for (i = 0; i < obj->nr_programs; i++) |
1611 | bpf_program__unload(&obj->programs[i]); | |
1612 | ||
52d3352e WN |
1613 | return 0; |
1614 | } | |
1615 | ||
1616 | int bpf_object__load(struct bpf_object *obj) | |
1617 | { | |
6371ca3b WN |
1618 | int err; |
1619 | ||
52d3352e WN |
1620 | if (!obj) |
1621 | return -EINVAL; | |
1622 | ||
1623 | if (obj->loaded) { | |
1624 | pr_warning("object should not be loaded twice\n"); | |
1625 | return -EINVAL; | |
1626 | } | |
1627 | ||
1628 | obj->loaded = true; | |
6371ca3b WN |
1629 | |
1630 | CHECK_ERR(bpf_object__create_maps(obj), err, out); | |
1631 | CHECK_ERR(bpf_object__relocate(obj), err, out); | |
1632 | CHECK_ERR(bpf_object__load_progs(obj), err, out); | |
52d3352e WN |
1633 | |
1634 | return 0; | |
1635 | out: | |
1636 | bpf_object__unload(obj); | |
1637 | pr_warning("failed to load object '%s'\n", obj->path); | |
6371ca3b | 1638 | return err; |
52d3352e WN |
1639 | } |
1640 | ||
f367540c JS |
1641 | static int check_path(const char *path) |
1642 | { | |
1ce6a9fc | 1643 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
1644 | struct statfs st_fs; |
1645 | char *dname, *dir; | |
1646 | int err = 0; | |
1647 | ||
1648 | if (path == NULL) | |
1649 | return -EINVAL; | |
1650 | ||
1651 | dname = strdup(path); | |
1652 | if (dname == NULL) | |
1653 | return -ENOMEM; | |
1654 | ||
1655 | dir = dirname(dname); | |
1656 | if (statfs(dir, &st_fs)) { | |
24d6a808 | 1657 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 1658 | pr_warning("failed to statfs %s: %s\n", dir, cp); |
f367540c JS |
1659 | err = -errno; |
1660 | } | |
1661 | free(dname); | |
1662 | ||
1663 | if (!err && st_fs.f_type != BPF_FS_MAGIC) { | |
1664 | pr_warning("specified path %s is not on BPF FS\n", path); | |
1665 | err = -EINVAL; | |
1666 | } | |
1667 | ||
1668 | return err; | |
1669 | } | |
1670 | ||
1671 | int bpf_program__pin_instance(struct bpf_program *prog, const char *path, | |
1672 | int instance) | |
1673 | { | |
1ce6a9fc | 1674 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
1675 | int err; |
1676 | ||
1677 | err = check_path(path); | |
1678 | if (err) | |
1679 | return err; | |
1680 | ||
1681 | if (prog == NULL) { | |
1682 | pr_warning("invalid program pointer\n"); | |
1683 | return -EINVAL; | |
1684 | } | |
1685 | ||
1686 | if (instance < 0 || instance >= prog->instances.nr) { | |
1687 | pr_warning("invalid prog instance %d of prog %s (max %d)\n", | |
1688 | instance, prog->section_name, prog->instances.nr); | |
1689 | return -EINVAL; | |
1690 | } | |
1691 | ||
1692 | if (bpf_obj_pin(prog->instances.fds[instance], path)) { | |
24d6a808 | 1693 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 1694 | pr_warning("failed to pin program: %s\n", cp); |
f367540c JS |
1695 | return -errno; |
1696 | } | |
1697 | pr_debug("pinned program '%s'\n", path); | |
1698 | ||
1699 | return 0; | |
1700 | } | |
1701 | ||
1702 | static int make_dir(const char *path) | |
1703 | { | |
1ce6a9fc | 1704 | char *cp, errmsg[STRERR_BUFSIZE]; |
f367540c JS |
1705 | int err = 0; |
1706 | ||
1707 | if (mkdir(path, 0700) && errno != EEXIST) | |
1708 | err = -errno; | |
1709 | ||
1ce6a9fc | 1710 | if (err) { |
24d6a808 | 1711 | cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); |
1ce6a9fc TR |
1712 | pr_warning("failed to mkdir %s: %s\n", path, cp); |
1713 | } | |
f367540c JS |
1714 | return err; |
1715 | } | |
1716 | ||
1717 | int bpf_program__pin(struct bpf_program *prog, const char *path) | |
1718 | { | |
1719 | int i, err; | |
1720 | ||
1721 | err = check_path(path); | |
1722 | if (err) | |
1723 | return err; | |
1724 | ||
1725 | if (prog == NULL) { | |
1726 | pr_warning("invalid program pointer\n"); | |
1727 | return -EINVAL; | |
1728 | } | |
1729 | ||
1730 | if (prog->instances.nr <= 0) { | |
1731 | pr_warning("no instances of prog %s to pin\n", | |
1732 | prog->section_name); | |
1733 | return -EINVAL; | |
1734 | } | |
1735 | ||
1736 | err = make_dir(path); | |
1737 | if (err) | |
1738 | return err; | |
1739 | ||
1740 | for (i = 0; i < prog->instances.nr; i++) { | |
1741 | char buf[PATH_MAX]; | |
1742 | int len; | |
1743 | ||
1744 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
1745 | if (len < 0) | |
1746 | return -EINVAL; | |
1747 | else if (len >= PATH_MAX) | |
1748 | return -ENAMETOOLONG; | |
1749 | ||
1750 | err = bpf_program__pin_instance(prog, buf, i); | |
1751 | if (err) | |
1752 | return err; | |
1753 | } | |
1754 | ||
1755 | return 0; | |
1756 | } | |
1757 | ||
b6989f35 JS |
1758 | int bpf_map__pin(struct bpf_map *map, const char *path) |
1759 | { | |
1ce6a9fc | 1760 | char *cp, errmsg[STRERR_BUFSIZE]; |
b6989f35 JS |
1761 | int err; |
1762 | ||
1763 | err = check_path(path); | |
1764 | if (err) | |
1765 | return err; | |
1766 | ||
1767 | if (map == NULL) { | |
1768 | pr_warning("invalid map pointer\n"); | |
1769 | return -EINVAL; | |
1770 | } | |
1771 | ||
1772 | if (bpf_obj_pin(map->fd, path)) { | |
24d6a808 | 1773 | cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); |
1ce6a9fc | 1774 | pr_warning("failed to pin map: %s\n", cp); |
b6989f35 JS |
1775 | return -errno; |
1776 | } | |
1777 | ||
1778 | pr_debug("pinned map '%s'\n", path); | |
1779 | return 0; | |
1780 | } | |
1781 | ||
d5148d85 JS |
1782 | int bpf_object__pin(struct bpf_object *obj, const char *path) |
1783 | { | |
1784 | struct bpf_program *prog; | |
1785 | struct bpf_map *map; | |
1786 | int err; | |
1787 | ||
1788 | if (!obj) | |
1789 | return -ENOENT; | |
1790 | ||
1791 | if (!obj->loaded) { | |
1792 | pr_warning("object not yet loaded; load it first\n"); | |
1793 | return -ENOENT; | |
1794 | } | |
1795 | ||
1796 | err = make_dir(path); | |
1797 | if (err) | |
1798 | return err; | |
1799 | ||
1800 | bpf_map__for_each(map, obj) { | |
1801 | char buf[PATH_MAX]; | |
1802 | int len; | |
1803 | ||
1804 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
1805 | bpf_map__name(map)); | |
1806 | if (len < 0) | |
1807 | return -EINVAL; | |
1808 | else if (len >= PATH_MAX) | |
1809 | return -ENAMETOOLONG; | |
1810 | ||
1811 | err = bpf_map__pin(map, buf); | |
1812 | if (err) | |
1813 | return err; | |
1814 | } | |
1815 | ||
1816 | bpf_object__for_each_program(prog, obj) { | |
1817 | char buf[PATH_MAX]; | |
1818 | int len; | |
1819 | ||
1820 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
1821 | prog->section_name); | |
1822 | if (len < 0) | |
1823 | return -EINVAL; | |
1824 | else if (len >= PATH_MAX) | |
1825 | return -ENAMETOOLONG; | |
1826 | ||
1827 | err = bpf_program__pin(prog, buf); | |
1828 | if (err) | |
1829 | return err; | |
1830 | } | |
1831 | ||
1832 | return 0; | |
1833 | } | |
1834 | ||
1a5e3fb1 WN |
1835 | void bpf_object__close(struct bpf_object *obj) |
1836 | { | |
a5b8bd47 WN |
1837 | size_t i; |
1838 | ||
1a5e3fb1 WN |
1839 | if (!obj) |
1840 | return; | |
1841 | ||
10931d24 WN |
1842 | if (obj->clear_priv) |
1843 | obj->clear_priv(obj, obj->priv); | |
1844 | ||
1a5e3fb1 | 1845 | bpf_object__elf_finish(obj); |
52d3352e | 1846 | bpf_object__unload(obj); |
8a138aed | 1847 | btf__free(obj->btf); |
1a5e3fb1 | 1848 | |
9d759a9b | 1849 | for (i = 0; i < obj->nr_maps; i++) { |
561bbcca | 1850 | zfree(&obj->maps[i].name); |
9d759a9b WN |
1851 | if (obj->maps[i].clear_priv) |
1852 | obj->maps[i].clear_priv(&obj->maps[i], | |
1853 | obj->maps[i].priv); | |
1854 | obj->maps[i].priv = NULL; | |
1855 | obj->maps[i].clear_priv = NULL; | |
1856 | } | |
1857 | zfree(&obj->maps); | |
1858 | obj->nr_maps = 0; | |
a5b8bd47 WN |
1859 | |
1860 | if (obj->programs && obj->nr_programs) { | |
1861 | for (i = 0; i < obj->nr_programs; i++) | |
1862 | bpf_program__exit(&obj->programs[i]); | |
1863 | } | |
1864 | zfree(&obj->programs); | |
1865 | ||
9a208eff | 1866 | list_del(&obj->list); |
1a5e3fb1 WN |
1867 | free(obj); |
1868 | } | |
aa9b1ac3 | 1869 | |
9a208eff WN |
1870 | struct bpf_object * |
1871 | bpf_object__next(struct bpf_object *prev) | |
1872 | { | |
1873 | struct bpf_object *next; | |
1874 | ||
1875 | if (!prev) | |
1876 | next = list_first_entry(&bpf_objects_list, | |
1877 | struct bpf_object, | |
1878 | list); | |
1879 | else | |
1880 | next = list_next_entry(prev, list); | |
1881 | ||
1882 | /* Empty list is noticed here so don't need checking on entry. */ | |
1883 | if (&next->list == &bpf_objects_list) | |
1884 | return NULL; | |
1885 | ||
1886 | return next; | |
1887 | } | |
1888 | ||
a7fe0450 | 1889 | const char *bpf_object__name(struct bpf_object *obj) |
acf860ae | 1890 | { |
a7fe0450 | 1891 | return obj ? obj->path : ERR_PTR(-EINVAL); |
acf860ae WN |
1892 | } |
1893 | ||
a7fe0450 | 1894 | unsigned int bpf_object__kversion(struct bpf_object *obj) |
45825d8a | 1895 | { |
a7fe0450 | 1896 | return obj ? obj->kern_version : 0; |
45825d8a WN |
1897 | } |
1898 | ||
8a138aed MKL |
1899 | int bpf_object__btf_fd(const struct bpf_object *obj) |
1900 | { | |
1901 | return obj->btf ? btf__fd(obj->btf) : -1; | |
1902 | } | |
1903 | ||
10931d24 WN |
1904 | int bpf_object__set_priv(struct bpf_object *obj, void *priv, |
1905 | bpf_object_clear_priv_t clear_priv) | |
1906 | { | |
1907 | if (obj->priv && obj->clear_priv) | |
1908 | obj->clear_priv(obj, obj->priv); | |
1909 | ||
1910 | obj->priv = priv; | |
1911 | obj->clear_priv = clear_priv; | |
1912 | return 0; | |
1913 | } | |
1914 | ||
1915 | void *bpf_object__priv(struct bpf_object *obj) | |
1916 | { | |
1917 | return obj ? obj->priv : ERR_PTR(-EINVAL); | |
1918 | } | |
1919 | ||
eac7d845 JK |
1920 | static struct bpf_program * |
1921 | __bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) | |
aa9b1ac3 WN |
1922 | { |
1923 | size_t idx; | |
1924 | ||
1925 | if (!obj->programs) | |
1926 | return NULL; | |
1927 | /* First handler */ | |
1928 | if (prev == NULL) | |
1929 | return &obj->programs[0]; | |
1930 | ||
1931 | if (prev->obj != obj) { | |
1932 | pr_warning("error: program handler doesn't match object\n"); | |
1933 | return NULL; | |
1934 | } | |
1935 | ||
1936 | idx = (prev - obj->programs) + 1; | |
1937 | if (idx >= obj->nr_programs) | |
1938 | return NULL; | |
1939 | return &obj->programs[idx]; | |
1940 | } | |
1941 | ||
eac7d845 JK |
1942 | struct bpf_program * |
1943 | bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) | |
1944 | { | |
1945 | struct bpf_program *prog = prev; | |
1946 | ||
1947 | do { | |
1948 | prog = __bpf_program__next(prog, obj); | |
1949 | } while (prog && bpf_program__is_function_storage(prog, obj)); | |
1950 | ||
1951 | return prog; | |
1952 | } | |
1953 | ||
edb13ed4 ACM |
1954 | int bpf_program__set_priv(struct bpf_program *prog, void *priv, |
1955 | bpf_program_clear_priv_t clear_priv) | |
aa9b1ac3 WN |
1956 | { |
1957 | if (prog->priv && prog->clear_priv) | |
1958 | prog->clear_priv(prog, prog->priv); | |
1959 | ||
1960 | prog->priv = priv; | |
1961 | prog->clear_priv = clear_priv; | |
1962 | return 0; | |
1963 | } | |
1964 | ||
be834ffb | 1965 | void *bpf_program__priv(struct bpf_program *prog) |
aa9b1ac3 | 1966 | { |
be834ffb | 1967 | return prog ? prog->priv : ERR_PTR(-EINVAL); |
aa9b1ac3 WN |
1968 | } |
1969 | ||
9aba3613 JK |
1970 | void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) |
1971 | { | |
1972 | prog->prog_ifindex = ifindex; | |
1973 | } | |
1974 | ||
715f8db9 | 1975 | const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
1976 | { |
1977 | const char *title; | |
1978 | ||
1979 | title = prog->section_name; | |
715f8db9 | 1980 | if (needs_copy) { |
aa9b1ac3 WN |
1981 | title = strdup(title); |
1982 | if (!title) { | |
1983 | pr_warning("failed to strdup program title\n"); | |
6371ca3b | 1984 | return ERR_PTR(-ENOMEM); |
aa9b1ac3 WN |
1985 | } |
1986 | } | |
1987 | ||
1988 | return title; | |
1989 | } | |
1990 | ||
1991 | int bpf_program__fd(struct bpf_program *prog) | |
1992 | { | |
b580563e WN |
1993 | return bpf_program__nth_fd(prog, 0); |
1994 | } | |
1995 | ||
1996 | int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, | |
1997 | bpf_program_prep_t prep) | |
1998 | { | |
1999 | int *instances_fds; | |
2000 | ||
2001 | if (nr_instances <= 0 || !prep) | |
2002 | return -EINVAL; | |
2003 | ||
2004 | if (prog->instances.nr > 0 || prog->instances.fds) { | |
2005 | pr_warning("Can't set pre-processor after loading\n"); | |
2006 | return -EINVAL; | |
2007 | } | |
2008 | ||
2009 | instances_fds = malloc(sizeof(int) * nr_instances); | |
2010 | if (!instances_fds) { | |
2011 | pr_warning("alloc memory failed for fds\n"); | |
2012 | return -ENOMEM; | |
2013 | } | |
2014 | ||
2015 | /* fill all fd with -1 */ | |
2016 | memset(instances_fds, -1, sizeof(int) * nr_instances); | |
2017 | ||
2018 | prog->instances.nr = nr_instances; | |
2019 | prog->instances.fds = instances_fds; | |
2020 | prog->preprocessor = prep; | |
2021 | return 0; | |
2022 | } | |
2023 | ||
2024 | int bpf_program__nth_fd(struct bpf_program *prog, int n) | |
2025 | { | |
2026 | int fd; | |
2027 | ||
1e960043 JK |
2028 | if (!prog) |
2029 | return -EINVAL; | |
2030 | ||
b580563e WN |
2031 | if (n >= prog->instances.nr || n < 0) { |
2032 | pr_warning("Can't get the %dth fd from program %s: only %d instances\n", | |
2033 | n, prog->section_name, prog->instances.nr); | |
2034 | return -EINVAL; | |
2035 | } | |
2036 | ||
2037 | fd = prog->instances.fds[n]; | |
2038 | if (fd < 0) { | |
2039 | pr_warning("%dth instance of program '%s' is invalid\n", | |
2040 | n, prog->section_name); | |
2041 | return -ENOENT; | |
2042 | } | |
2043 | ||
2044 | return fd; | |
aa9b1ac3 | 2045 | } |
9d759a9b | 2046 | |
dd26b7f5 | 2047 | void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) |
5f44e4c8 WN |
2048 | { |
2049 | prog->type = type; | |
2050 | } | |
2051 | ||
5f44e4c8 WN |
2052 | static bool bpf_program__is_type(struct bpf_program *prog, |
2053 | enum bpf_prog_type type) | |
2054 | { | |
2055 | return prog ? (prog->type == type) : false; | |
2056 | } | |
2057 | ||
ed794073 JS |
2058 | #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ |
2059 | int bpf_program__set_##NAME(struct bpf_program *prog) \ | |
2060 | { \ | |
2061 | if (!prog) \ | |
2062 | return -EINVAL; \ | |
2063 | bpf_program__set_type(prog, TYPE); \ | |
2064 | return 0; \ | |
2065 | } \ | |
2066 | \ | |
2067 | bool bpf_program__is_##NAME(struct bpf_program *prog) \ | |
2068 | { \ | |
2069 | return bpf_program__is_type(prog, TYPE); \ | |
2070 | } \ | |
2071 | ||
7803ba73 | 2072 | BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); |
ed794073 | 2073 | BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); |
7803ba73 JS |
2074 | BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); |
2075 | BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); | |
ed794073 | 2076 | BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); |
e14c93fd | 2077 | BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); |
7803ba73 JS |
2078 | BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); |
2079 | BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); | |
5f44e4c8 | 2080 | |
16962b24 JF |
2081 | void bpf_program__set_expected_attach_type(struct bpf_program *prog, |
2082 | enum bpf_attach_type type) | |
d7be143b AI |
2083 | { |
2084 | prog->expected_attach_type = type; | |
2085 | } | |
2086 | ||
36153532 AI |
2087 | #define BPF_PROG_SEC_IMPL(string, ptype, eatype, is_attachable, atype) \ |
2088 | { string, sizeof(string) - 1, ptype, eatype, is_attachable, atype } | |
d7be143b | 2089 | |
956b620f | 2090 | /* Programs that can NOT be attached. */ |
36153532 | 2091 | #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_IMPL(string, ptype, 0, 0, 0) |
d7be143b | 2092 | |
956b620f AI |
2093 | /* Programs that can be attached. */ |
2094 | #define BPF_APROG_SEC(string, ptype, atype) \ | |
36153532 | 2095 | BPF_PROG_SEC_IMPL(string, ptype, 0, 1, atype) |
81efee75 | 2096 | |
956b620f AI |
2097 | /* Programs that must specify expected attach type at load time. */ |
2098 | #define BPF_EAPROG_SEC(string, ptype, eatype) \ | |
36153532 | 2099 | BPF_PROG_SEC_IMPL(string, ptype, eatype, 1, eatype) |
956b620f AI |
2100 | |
2101 | /* Programs that can be attached but attach type can't be identified by section | |
2102 | * name. Kept for backward compatibility. | |
2103 | */ | |
2104 | #define BPF_APROG_COMPAT(string, ptype) BPF_PROG_SEC(string, ptype) | |
e50b0a6f | 2105 | |
583c9009 RG |
2106 | static const struct { |
2107 | const char *sec; | |
2108 | size_t len; | |
2109 | enum bpf_prog_type prog_type; | |
d7be143b | 2110 | enum bpf_attach_type expected_attach_type; |
36153532 | 2111 | int is_attachable; |
956b620f | 2112 | enum bpf_attach_type attach_type; |
583c9009 | 2113 | } section_names[] = { |
956b620f AI |
2114 | BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), |
2115 | BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), | |
2116 | BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), | |
2117 | BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), | |
2118 | BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), | |
2119 | BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), | |
2120 | BPF_PROG_SEC("raw_tracepoint/", BPF_PROG_TYPE_RAW_TRACEPOINT), | |
2121 | BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), | |
2122 | BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), | |
2123 | BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), | |
2124 | BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), | |
2125 | BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), | |
2126 | BPF_PROG_SEC("lwt_seg6local", BPF_PROG_TYPE_LWT_SEG6LOCAL), | |
bafa7afe AI |
2127 | BPF_APROG_SEC("cgroup_skb/ingress", BPF_PROG_TYPE_CGROUP_SKB, |
2128 | BPF_CGROUP_INET_INGRESS), | |
2129 | BPF_APROG_SEC("cgroup_skb/egress", BPF_PROG_TYPE_CGROUP_SKB, | |
2130 | BPF_CGROUP_INET_EGRESS), | |
956b620f AI |
2131 | BPF_APROG_COMPAT("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), |
2132 | BPF_APROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK, | |
2133 | BPF_CGROUP_INET_SOCK_CREATE), | |
2134 | BPF_EAPROG_SEC("cgroup/post_bind4", BPF_PROG_TYPE_CGROUP_SOCK, | |
2135 | BPF_CGROUP_INET4_POST_BIND), | |
2136 | BPF_EAPROG_SEC("cgroup/post_bind6", BPF_PROG_TYPE_CGROUP_SOCK, | |
2137 | BPF_CGROUP_INET6_POST_BIND), | |
2138 | BPF_APROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE, | |
2139 | BPF_CGROUP_DEVICE), | |
2140 | BPF_APROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS, | |
2141 | BPF_CGROUP_SOCK_OPS), | |
c6f6851b AI |
2142 | BPF_APROG_SEC("sk_skb/stream_parser", BPF_PROG_TYPE_SK_SKB, |
2143 | BPF_SK_SKB_STREAM_PARSER), | |
2144 | BPF_APROG_SEC("sk_skb/stream_verdict", BPF_PROG_TYPE_SK_SKB, | |
2145 | BPF_SK_SKB_STREAM_VERDICT), | |
956b620f AI |
2146 | BPF_APROG_COMPAT("sk_skb", BPF_PROG_TYPE_SK_SKB), |
2147 | BPF_APROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG, | |
2148 | BPF_SK_MSG_VERDICT), | |
2149 | BPF_APROG_SEC("lirc_mode2", BPF_PROG_TYPE_LIRC_MODE2, | |
2150 | BPF_LIRC_MODE2), | |
2151 | BPF_APROG_SEC("flow_dissector", BPF_PROG_TYPE_FLOW_DISSECTOR, | |
2152 | BPF_FLOW_DISSECTOR), | |
2153 | BPF_EAPROG_SEC("cgroup/bind4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
2154 | BPF_CGROUP_INET4_BIND), | |
2155 | BPF_EAPROG_SEC("cgroup/bind6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
2156 | BPF_CGROUP_INET6_BIND), | |
2157 | BPF_EAPROG_SEC("cgroup/connect4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
2158 | BPF_CGROUP_INET4_CONNECT), | |
2159 | BPF_EAPROG_SEC("cgroup/connect6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
2160 | BPF_CGROUP_INET6_CONNECT), | |
2161 | BPF_EAPROG_SEC("cgroup/sendmsg4", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
2162 | BPF_CGROUP_UDP4_SENDMSG), | |
2163 | BPF_EAPROG_SEC("cgroup/sendmsg6", BPF_PROG_TYPE_CGROUP_SOCK_ADDR, | |
2164 | BPF_CGROUP_UDP6_SENDMSG), | |
583c9009 | 2165 | }; |
d7be143b | 2166 | |
956b620f | 2167 | #undef BPF_PROG_SEC_IMPL |
583c9009 | 2168 | #undef BPF_PROG_SEC |
956b620f AI |
2169 | #undef BPF_APROG_SEC |
2170 | #undef BPF_EAPROG_SEC | |
2171 | #undef BPF_APROG_COMPAT | |
583c9009 | 2172 | |
b60df2a0 JK |
2173 | int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, |
2174 | enum bpf_attach_type *expected_attach_type) | |
583c9009 RG |
2175 | { |
2176 | int i; | |
2177 | ||
b60df2a0 JK |
2178 | if (!name) |
2179 | return -EINVAL; | |
583c9009 | 2180 | |
b60df2a0 JK |
2181 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { |
2182 | if (strncmp(name, section_names[i].sec, section_names[i].len)) | |
2183 | continue; | |
2184 | *prog_type = section_names[i].prog_type; | |
2185 | *expected_attach_type = section_names[i].expected_attach_type; | |
2186 | return 0; | |
2187 | } | |
2188 | return -EINVAL; | |
2189 | } | |
583c9009 | 2190 | |
956b620f AI |
2191 | int libbpf_attach_type_by_name(const char *name, |
2192 | enum bpf_attach_type *attach_type) | |
2193 | { | |
2194 | int i; | |
2195 | ||
2196 | if (!name) | |
2197 | return -EINVAL; | |
2198 | ||
2199 | for (i = 0; i < ARRAY_SIZE(section_names); i++) { | |
2200 | if (strncmp(name, section_names[i].sec, section_names[i].len)) | |
2201 | continue; | |
36153532 | 2202 | if (!section_names[i].is_attachable) |
956b620f AI |
2203 | return -EINVAL; |
2204 | *attach_type = section_names[i].attach_type; | |
2205 | return 0; | |
2206 | } | |
2207 | return -EINVAL; | |
2208 | } | |
2209 | ||
b60df2a0 JK |
2210 | static int |
2211 | bpf_program__identify_section(struct bpf_program *prog, | |
2212 | enum bpf_prog_type *prog_type, | |
2213 | enum bpf_attach_type *expected_attach_type) | |
2214 | { | |
2215 | return libbpf_prog_type_by_name(prog->section_name, prog_type, | |
2216 | expected_attach_type); | |
583c9009 RG |
2217 | } |
2218 | ||
6e009e65 | 2219 | int bpf_map__fd(struct bpf_map *map) |
9d759a9b | 2220 | { |
6e009e65 | 2221 | return map ? map->fd : -EINVAL; |
9d759a9b WN |
2222 | } |
2223 | ||
53897a78 | 2224 | const struct bpf_map_def *bpf_map__def(struct bpf_map *map) |
9d759a9b | 2225 | { |
53897a78 | 2226 | return map ? &map->def : ERR_PTR(-EINVAL); |
9d759a9b WN |
2227 | } |
2228 | ||
009ad5d5 | 2229 | const char *bpf_map__name(struct bpf_map *map) |
561bbcca | 2230 | { |
009ad5d5 | 2231 | return map ? map->name : NULL; |
561bbcca WN |
2232 | } |
2233 | ||
5b891af7 | 2234 | __u32 bpf_map__btf_key_type_id(const struct bpf_map *map) |
8a138aed | 2235 | { |
61746dbe | 2236 | return map ? map->btf_key_type_id : 0; |
8a138aed MKL |
2237 | } |
2238 | ||
5b891af7 | 2239 | __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) |
8a138aed | 2240 | { |
61746dbe | 2241 | return map ? map->btf_value_type_id : 0; |
8a138aed MKL |
2242 | } |
2243 | ||
edb13ed4 ACM |
2244 | int bpf_map__set_priv(struct bpf_map *map, void *priv, |
2245 | bpf_map_clear_priv_t clear_priv) | |
9d759a9b WN |
2246 | { |
2247 | if (!map) | |
2248 | return -EINVAL; | |
2249 | ||
2250 | if (map->priv) { | |
2251 | if (map->clear_priv) | |
2252 | map->clear_priv(map, map->priv); | |
2253 | } | |
2254 | ||
2255 | map->priv = priv; | |
2256 | map->clear_priv = clear_priv; | |
2257 | return 0; | |
2258 | } | |
2259 | ||
b4cbfa56 | 2260 | void *bpf_map__priv(struct bpf_map *map) |
9d759a9b | 2261 | { |
b4cbfa56 | 2262 | return map ? map->priv : ERR_PTR(-EINVAL); |
9d759a9b WN |
2263 | } |
2264 | ||
f83fb22c JK |
2265 | bool bpf_map__is_offload_neutral(struct bpf_map *map) |
2266 | { | |
2267 | return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; | |
2268 | } | |
2269 | ||
9aba3613 JK |
2270 | void bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) |
2271 | { | |
2272 | map->map_ifindex = ifindex; | |
2273 | } | |
2274 | ||
9d759a9b WN |
2275 | struct bpf_map * |
2276 | bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) | |
2277 | { | |
2278 | size_t idx; | |
2279 | struct bpf_map *s, *e; | |
2280 | ||
2281 | if (!obj || !obj->maps) | |
2282 | return NULL; | |
2283 | ||
2284 | s = obj->maps; | |
2285 | e = obj->maps + obj->nr_maps; | |
2286 | ||
2287 | if (prev == NULL) | |
2288 | return s; | |
2289 | ||
2290 | if ((prev < s) || (prev >= e)) { | |
2291 | pr_warning("error in %s: map handler doesn't belong to object\n", | |
2292 | __func__); | |
2293 | return NULL; | |
2294 | } | |
2295 | ||
2296 | idx = (prev - obj->maps) + 1; | |
2297 | if (idx >= obj->nr_maps) | |
2298 | return NULL; | |
2299 | return &obj->maps[idx]; | |
2300 | } | |
561bbcca WN |
2301 | |
2302 | struct bpf_map * | |
a7fe0450 | 2303 | bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) |
561bbcca WN |
2304 | { |
2305 | struct bpf_map *pos; | |
2306 | ||
2307 | bpf_map__for_each(pos, obj) { | |
973170e6 | 2308 | if (pos->name && !strcmp(pos->name, name)) |
561bbcca WN |
2309 | return pos; |
2310 | } | |
2311 | return NULL; | |
2312 | } | |
5a6acad1 WN |
2313 | |
2314 | struct bpf_map * | |
2315 | bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) | |
2316 | { | |
2317 | int i; | |
2318 | ||
2319 | for (i = 0; i < obj->nr_maps; i++) { | |
2320 | if (obj->maps[i].offset == offset) | |
2321 | return &obj->maps[i]; | |
2322 | } | |
2323 | return ERR_PTR(-ENOENT); | |
2324 | } | |
e28ff1a8 JS |
2325 | |
2326 | long libbpf_get_error(const void *ptr) | |
2327 | { | |
2328 | if (IS_ERR(ptr)) | |
2329 | return PTR_ERR(ptr); | |
2330 | return 0; | |
2331 | } | |
6f6d33f3 JF |
2332 | |
2333 | int bpf_prog_load(const char *file, enum bpf_prog_type type, | |
2334 | struct bpf_object **pobj, int *prog_fd) | |
d7be143b AI |
2335 | { |
2336 | struct bpf_prog_load_attr attr; | |
2337 | ||
2338 | memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); | |
2339 | attr.file = file; | |
2340 | attr.prog_type = type; | |
2341 | attr.expected_attach_type = 0; | |
2342 | ||
2343 | return bpf_prog_load_xattr(&attr, pobj, prog_fd); | |
2344 | } | |
2345 | ||
2346 | int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, | |
2347 | struct bpf_object **pobj, int *prog_fd) | |
6f6d33f3 | 2348 | { |
07f2d4ea JK |
2349 | struct bpf_object_open_attr open_attr = { |
2350 | .file = attr->file, | |
2351 | .prog_type = attr->prog_type, | |
2352 | }; | |
48cca7e4 | 2353 | struct bpf_program *prog, *first_prog = NULL; |
d7be143b AI |
2354 | enum bpf_attach_type expected_attach_type; |
2355 | enum bpf_prog_type prog_type; | |
6f6d33f3 | 2356 | struct bpf_object *obj; |
f0307a7e | 2357 | struct bpf_map *map; |
6f6d33f3 JF |
2358 | int err; |
2359 | ||
d7be143b AI |
2360 | if (!attr) |
2361 | return -EINVAL; | |
17387dd5 JK |
2362 | if (!attr->file) |
2363 | return -EINVAL; | |
d7be143b | 2364 | |
07f2d4ea | 2365 | obj = bpf_object__open_xattr(&open_attr); |
3597683c | 2366 | if (IS_ERR_OR_NULL(obj)) |
6f6d33f3 JF |
2367 | return -ENOENT; |
2368 | ||
48cca7e4 AS |
2369 | bpf_object__for_each_program(prog, obj) { |
2370 | /* | |
2371 | * If type is not specified, try to guess it based on | |
2372 | * section name. | |
2373 | */ | |
d7be143b | 2374 | prog_type = attr->prog_type; |
f0307a7e | 2375 | prog->prog_ifindex = attr->ifindex; |
d7be143b AI |
2376 | expected_attach_type = attr->expected_attach_type; |
2377 | if (prog_type == BPF_PROG_TYPE_UNSPEC) { | |
b60df2a0 JK |
2378 | err = bpf_program__identify_section(prog, &prog_type, |
2379 | &expected_attach_type); | |
2380 | if (err < 0) { | |
2381 | pr_warning("failed to guess program type based on section name %s\n", | |
2382 | prog->section_name); | |
48cca7e4 AS |
2383 | bpf_object__close(obj); |
2384 | return -EINVAL; | |
2385 | } | |
583c9009 | 2386 | } |
48cca7e4 | 2387 | |
d7be143b AI |
2388 | bpf_program__set_type(prog, prog_type); |
2389 | bpf_program__set_expected_attach_type(prog, | |
2390 | expected_attach_type); | |
2391 | ||
69495d2a | 2392 | if (!first_prog) |
48cca7e4 AS |
2393 | first_prog = prog; |
2394 | } | |
2395 | ||
f0307a7e | 2396 | bpf_map__for_each(map, obj) { |
f83fb22c JK |
2397 | if (!bpf_map__is_offload_neutral(map)) |
2398 | map->map_ifindex = attr->ifindex; | |
f0307a7e DB |
2399 | } |
2400 | ||
48cca7e4 AS |
2401 | if (!first_prog) { |
2402 | pr_warning("object file doesn't contain bpf program\n"); | |
2403 | bpf_object__close(obj); | |
2404 | return -ENOENT; | |
583c9009 RG |
2405 | } |
2406 | ||
6f6d33f3 JF |
2407 | err = bpf_object__load(obj); |
2408 | if (err) { | |
2409 | bpf_object__close(obj); | |
2410 | return -EINVAL; | |
2411 | } | |
2412 | ||
2413 | *pobj = obj; | |
48cca7e4 | 2414 | *prog_fd = bpf_program__fd(first_prog); |
6f6d33f3 JF |
2415 | return 0; |
2416 | } | |
d0cabbb0 JK |
2417 | |
2418 | enum bpf_perf_event_ret | |
3dca2115 DB |
2419 | bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, |
2420 | void **copy_mem, size_t *copy_size, | |
2421 | bpf_perf_event_print_t fn, void *private_data) | |
d0cabbb0 | 2422 | { |
3dca2115 | 2423 | struct perf_event_mmap_page *header = mmap_mem; |
a64af0ef | 2424 | __u64 data_head = ring_buffer_read_head(header); |
d0cabbb0 | 2425 | __u64 data_tail = header->data_tail; |
3dca2115 DB |
2426 | void *base = ((__u8 *)header) + page_size; |
2427 | int ret = LIBBPF_PERF_EVENT_CONT; | |
2428 | struct perf_event_header *ehdr; | |
2429 | size_t ehdr_size; | |
2430 | ||
2431 | while (data_head != data_tail) { | |
2432 | ehdr = base + (data_tail & (mmap_size - 1)); | |
2433 | ehdr_size = ehdr->size; | |
2434 | ||
2435 | if (((void *)ehdr) + ehdr_size > base + mmap_size) { | |
2436 | void *copy_start = ehdr; | |
2437 | size_t len_first = base + mmap_size - copy_start; | |
2438 | size_t len_secnd = ehdr_size - len_first; | |
2439 | ||
2440 | if (*copy_size < ehdr_size) { | |
2441 | free(*copy_mem); | |
2442 | *copy_mem = malloc(ehdr_size); | |
2443 | if (!*copy_mem) { | |
2444 | *copy_size = 0; | |
d0cabbb0 JK |
2445 | ret = LIBBPF_PERF_EVENT_ERROR; |
2446 | break; | |
2447 | } | |
3dca2115 | 2448 | *copy_size = ehdr_size; |
d0cabbb0 JK |
2449 | } |
2450 | ||
3dca2115 DB |
2451 | memcpy(*copy_mem, copy_start, len_first); |
2452 | memcpy(*copy_mem + len_first, base, len_secnd); | |
2453 | ehdr = *copy_mem; | |
d0cabbb0 JK |
2454 | } |
2455 | ||
3dca2115 DB |
2456 | ret = fn(ehdr, private_data); |
2457 | data_tail += ehdr_size; | |
d0cabbb0 JK |
2458 | if (ret != LIBBPF_PERF_EVENT_CONT) |
2459 | break; | |
d0cabbb0 JK |
2460 | } |
2461 | ||
a64af0ef | 2462 | ring_buffer_write_tail(header, data_tail); |
d0cabbb0 JK |
2463 | return ret; |
2464 | } |