Commit | Line | Data |
---|---|---|
6061a3d6 EL |
1 | // SPDX-License-Identifier: LGPL-2.1 |
2 | ||
1b76c13e WN |
3 | /* |
4 | * Common eBPF ELF object loading operations. | |
5 | * | |
6 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
7 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
8 | * Copyright (C) 2015 Huawei Inc. | |
f367540c | 9 | * Copyright (C) 2017 Nicira, Inc. |
203d1cac WN |
10 | * |
11 | * This program is free software; you can redistribute it and/or | |
12 | * modify it under the terms of the GNU Lesser General Public | |
13 | * License as published by the Free Software Foundation; | |
14 | * version 2.1 of the License (not later!) | |
15 | * | |
16 | * This program is distributed in the hope that it will be useful, | |
17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
19 | * GNU Lesser General Public License for more details. | |
20 | * | |
21 | * You should have received a copy of the GNU Lesser General Public | |
22 | * License along with this program; if not, see <http://www.gnu.org/licenses> | |
1b76c13e WN |
23 | */ |
24 | ||
25 | #include <stdlib.h> | |
b3f59d66 WN |
26 | #include <stdio.h> |
27 | #include <stdarg.h> | |
f367540c | 28 | #include <libgen.h> |
34090915 | 29 | #include <inttypes.h> |
b3f59d66 | 30 | #include <string.h> |
1b76c13e | 31 | #include <unistd.h> |
1a5e3fb1 WN |
32 | #include <fcntl.h> |
33 | #include <errno.h> | |
1b76c13e | 34 | #include <asm/unistd.h> |
e28ff1a8 | 35 | #include <linux/err.h> |
cb1e5e96 | 36 | #include <linux/kernel.h> |
1b76c13e | 37 | #include <linux/bpf.h> |
9a208eff | 38 | #include <linux/list.h> |
f367540c JS |
39 | #include <linux/limits.h> |
40 | #include <sys/stat.h> | |
41 | #include <sys/types.h> | |
42 | #include <sys/vfs.h> | |
1a5e3fb1 WN |
43 | #include <libelf.h> |
44 | #include <gelf.h> | |
1b76c13e WN |
45 | |
46 | #include "libbpf.h" | |
52d3352e | 47 | #include "bpf.h" |
b3f59d66 | 48 | |
9b16137a WN |
49 | #ifndef EM_BPF |
50 | #define EM_BPF 247 | |
51 | #endif | |
52 | ||
f367540c JS |
53 | #ifndef BPF_FS_MAGIC |
54 | #define BPF_FS_MAGIC 0xcafe4a11 | |
55 | #endif | |
56 | ||
b3f59d66 WN |
57 | #define __printf(a, b) __attribute__((format(printf, a, b))) |
58 | ||
59 | __printf(1, 2) | |
60 | static int __base_pr(const char *format, ...) | |
61 | { | |
62 | va_list args; | |
63 | int err; | |
64 | ||
65 | va_start(args, format); | |
66 | err = vfprintf(stderr, format, args); | |
67 | va_end(args); | |
68 | return err; | |
69 | } | |
70 | ||
71 | static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; | |
72 | static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; | |
73 | static __printf(1, 2) libbpf_print_fn_t __pr_debug; | |
74 | ||
75 | #define __pr(func, fmt, ...) \ | |
76 | do { \ | |
77 | if ((func)) \ | |
78 | (func)("libbpf: " fmt, ##__VA_ARGS__); \ | |
79 | } while (0) | |
80 | ||
81 | #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) | |
82 | #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) | |
83 | #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) | |
84 | ||
85 | void libbpf_set_print(libbpf_print_fn_t warn, | |
86 | libbpf_print_fn_t info, | |
87 | libbpf_print_fn_t debug) | |
88 | { | |
89 | __pr_warning = warn; | |
90 | __pr_info = info; | |
91 | __pr_debug = debug; | |
92 | } | |
1a5e3fb1 | 93 | |
6371ca3b WN |
94 | #define STRERR_BUFSIZE 128 |
95 | ||
96 | #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) | |
97 | #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) | |
98 | #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) | |
99 | ||
100 | static const char *libbpf_strerror_table[NR_ERRNO] = { | |
101 | [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", | |
102 | [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", | |
103 | [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", | |
de8a63bd | 104 | [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", |
6371ca3b WN |
105 | [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", |
106 | [ERRCODE_OFFSET(RELOC)] = "Relocation failed", | |
107 | [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", | |
108 | [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", | |
109 | [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", | |
705fa219 | 110 | [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type", |
949abbe8 EL |
111 | [ERRCODE_OFFSET(WRNGPID)] = "Wrong pid in netlink message", |
112 | [ERRCODE_OFFSET(INVSEQ)] = "Invalid netlink sequence", | |
6371ca3b WN |
113 | }; |
114 | ||
115 | int libbpf_strerror(int err, char *buf, size_t size) | |
116 | { | |
117 | if (!buf || !size) | |
118 | return -1; | |
119 | ||
120 | err = err > 0 ? err : -err; | |
121 | ||
122 | if (err < __LIBBPF_ERRNO__START) { | |
123 | int ret; | |
124 | ||
125 | ret = strerror_r(err, buf, size); | |
126 | buf[size - 1] = '\0'; | |
127 | return ret; | |
128 | } | |
129 | ||
130 | if (err < __LIBBPF_ERRNO__END) { | |
131 | const char *msg; | |
132 | ||
133 | msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; | |
134 | snprintf(buf, size, "%s", msg); | |
135 | buf[size - 1] = '\0'; | |
136 | return 0; | |
137 | } | |
138 | ||
139 | snprintf(buf, size, "Unknown libbpf error %d", err); | |
140 | buf[size - 1] = '\0'; | |
141 | return -1; | |
142 | } | |
143 | ||
144 | #define CHECK_ERR(action, err, out) do { \ | |
145 | err = action; \ | |
146 | if (err) \ | |
147 | goto out; \ | |
148 | } while(0) | |
149 | ||
150 | ||
1a5e3fb1 WN |
151 | /* Copied from tools/perf/util/util.h */ |
152 | #ifndef zfree | |
153 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
154 | #endif | |
155 | ||
156 | #ifndef zclose | |
157 | # define zclose(fd) ({ \ | |
158 | int ___err = 0; \ | |
159 | if ((fd) >= 0) \ | |
160 | ___err = close((fd)); \ | |
161 | fd = -1; \ | |
162 | ___err; }) | |
163 | #endif | |
164 | ||
165 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
166 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
167 | #else | |
168 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
169 | #endif | |
170 | ||
a5b8bd47 WN |
171 | /* |
172 | * bpf_prog should be a better name but it has been used in | |
173 | * linux/filter.h. | |
174 | */ | |
175 | struct bpf_program { | |
176 | /* Index in elf obj file, for relocation use. */ | |
177 | int idx; | |
88cda1c9 | 178 | char *name; |
a5b8bd47 WN |
179 | char *section_name; |
180 | struct bpf_insn *insns; | |
48cca7e4 | 181 | size_t insns_cnt, main_prog_cnt; |
5f44e4c8 | 182 | enum bpf_prog_type type; |
34090915 | 183 | |
48cca7e4 AS |
184 | struct reloc_desc { |
185 | enum { | |
186 | RELO_LD64, | |
187 | RELO_CALL, | |
188 | } type; | |
34090915 | 189 | int insn_idx; |
48cca7e4 AS |
190 | union { |
191 | int map_idx; | |
192 | int text_off; | |
193 | }; | |
34090915 WN |
194 | } *reloc_desc; |
195 | int nr_reloc; | |
55cffde2 | 196 | |
b580563e WN |
197 | struct { |
198 | int nr; | |
199 | int *fds; | |
200 | } instances; | |
201 | bpf_program_prep_t preprocessor; | |
aa9b1ac3 WN |
202 | |
203 | struct bpf_object *obj; | |
204 | void *priv; | |
205 | bpf_program_clear_priv_t clear_priv; | |
a5b8bd47 WN |
206 | }; |
207 | ||
9d759a9b WN |
208 | struct bpf_map { |
209 | int fd; | |
561bbcca | 210 | char *name; |
4708bbda | 211 | size_t offset; |
9d759a9b WN |
212 | struct bpf_map_def def; |
213 | void *priv; | |
214 | bpf_map_clear_priv_t clear_priv; | |
215 | }; | |
216 | ||
9a208eff WN |
217 | static LIST_HEAD(bpf_objects_list); |
218 | ||
1a5e3fb1 | 219 | struct bpf_object { |
cb1e5e96 WN |
220 | char license[64]; |
221 | u32 kern_version; | |
0b3d1efa | 222 | |
a5b8bd47 WN |
223 | struct bpf_program *programs; |
224 | size_t nr_programs; | |
9d759a9b WN |
225 | struct bpf_map *maps; |
226 | size_t nr_maps; | |
227 | ||
52d3352e | 228 | bool loaded; |
a5b8bd47 | 229 | |
1a5e3fb1 WN |
230 | /* |
231 | * Information when doing elf related work. Only valid if fd | |
232 | * is valid. | |
233 | */ | |
234 | struct { | |
235 | int fd; | |
6c956392 WN |
236 | void *obj_buf; |
237 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
238 | Elf *elf; |
239 | GElf_Ehdr ehdr; | |
bec7d68c | 240 | Elf_Data *symbols; |
77ba9a5b | 241 | size_t strtabidx; |
b62f06e8 WN |
242 | struct { |
243 | GElf_Shdr shdr; | |
244 | Elf_Data *data; | |
245 | } *reloc; | |
246 | int nr_reloc; | |
666810e8 | 247 | int maps_shndx; |
48cca7e4 | 248 | int text_shndx; |
1a5e3fb1 | 249 | } efile; |
9a208eff WN |
250 | /* |
251 | * All loaded bpf_object is linked in a list, which is | |
252 | * hidden to caller. bpf_objects__<func> handlers deal with | |
253 | * all objects. | |
254 | */ | |
255 | struct list_head list; | |
10931d24 WN |
256 | |
257 | void *priv; | |
258 | bpf_object_clear_priv_t clear_priv; | |
259 | ||
1a5e3fb1 WN |
260 | char path[]; |
261 | }; | |
262 | #define obj_elf_valid(o) ((o)->efile.elf) | |
263 | ||
55cffde2 WN |
264 | static void bpf_program__unload(struct bpf_program *prog) |
265 | { | |
b580563e WN |
266 | int i; |
267 | ||
55cffde2 WN |
268 | if (!prog) |
269 | return; | |
270 | ||
b580563e WN |
271 | /* |
272 | * If the object is opened but the program was never loaded, | |
273 | * it is possible that prog->instances.nr == -1. | |
274 | */ | |
275 | if (prog->instances.nr > 0) { | |
276 | for (i = 0; i < prog->instances.nr; i++) | |
277 | zclose(prog->instances.fds[i]); | |
278 | } else if (prog->instances.nr != -1) { | |
279 | pr_warning("Internal error: instances.nr is %d\n", | |
280 | prog->instances.nr); | |
281 | } | |
282 | ||
283 | prog->instances.nr = -1; | |
284 | zfree(&prog->instances.fds); | |
55cffde2 WN |
285 | } |
286 | ||
a5b8bd47 WN |
287 | static void bpf_program__exit(struct bpf_program *prog) |
288 | { | |
289 | if (!prog) | |
290 | return; | |
291 | ||
aa9b1ac3 WN |
292 | if (prog->clear_priv) |
293 | prog->clear_priv(prog, prog->priv); | |
294 | ||
295 | prog->priv = NULL; | |
296 | prog->clear_priv = NULL; | |
297 | ||
55cffde2 | 298 | bpf_program__unload(prog); |
88cda1c9 | 299 | zfree(&prog->name); |
a5b8bd47 WN |
300 | zfree(&prog->section_name); |
301 | zfree(&prog->insns); | |
34090915 WN |
302 | zfree(&prog->reloc_desc); |
303 | ||
304 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
305 | prog->insns_cnt = 0; |
306 | prog->idx = -1; | |
307 | } | |
308 | ||
309 | static int | |
88cda1c9 MKL |
310 | bpf_program__init(void *data, size_t size, char *section_name, int idx, |
311 | struct bpf_program *prog) | |
a5b8bd47 WN |
312 | { |
313 | if (size < sizeof(struct bpf_insn)) { | |
88cda1c9 | 314 | pr_warning("corrupted section '%s'\n", section_name); |
a5b8bd47 WN |
315 | return -EINVAL; |
316 | } | |
317 | ||
318 | bzero(prog, sizeof(*prog)); | |
319 | ||
88cda1c9 | 320 | prog->section_name = strdup(section_name); |
a5b8bd47 | 321 | if (!prog->section_name) { |
88cda1c9 MKL |
322 | pr_warning("failed to alloc name for prog under section %s\n", |
323 | section_name); | |
a5b8bd47 WN |
324 | goto errout; |
325 | } | |
326 | ||
327 | prog->insns = malloc(size); | |
328 | if (!prog->insns) { | |
88cda1c9 MKL |
329 | pr_warning("failed to alloc insns for prog under section %s\n", |
330 | section_name); | |
a5b8bd47 WN |
331 | goto errout; |
332 | } | |
333 | prog->insns_cnt = size / sizeof(struct bpf_insn); | |
334 | memcpy(prog->insns, data, | |
335 | prog->insns_cnt * sizeof(struct bpf_insn)); | |
336 | prog->idx = idx; | |
b580563e WN |
337 | prog->instances.fds = NULL; |
338 | prog->instances.nr = -1; | |
5f44e4c8 | 339 | prog->type = BPF_PROG_TYPE_KPROBE; |
a5b8bd47 WN |
340 | |
341 | return 0; | |
342 | errout: | |
343 | bpf_program__exit(prog); | |
344 | return -ENOMEM; | |
345 | } | |
346 | ||
347 | static int | |
348 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
88cda1c9 | 349 | char *section_name, int idx) |
a5b8bd47 WN |
350 | { |
351 | struct bpf_program prog, *progs; | |
352 | int nr_progs, err; | |
353 | ||
88cda1c9 | 354 | err = bpf_program__init(data, size, section_name, idx, &prog); |
a5b8bd47 WN |
355 | if (err) |
356 | return err; | |
357 | ||
358 | progs = obj->programs; | |
359 | nr_progs = obj->nr_programs; | |
360 | ||
361 | progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); | |
362 | if (!progs) { | |
363 | /* | |
364 | * In this case the original obj->programs | |
365 | * is still valid, so don't need special treat for | |
366 | * bpf_close_object(). | |
367 | */ | |
88cda1c9 MKL |
368 | pr_warning("failed to alloc a new program under section '%s'\n", |
369 | section_name); | |
a5b8bd47 WN |
370 | bpf_program__exit(&prog); |
371 | return -ENOMEM; | |
372 | } | |
373 | ||
374 | pr_debug("found program %s\n", prog.section_name); | |
375 | obj->programs = progs; | |
376 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 377 | prog.obj = obj; |
a5b8bd47 WN |
378 | progs[nr_progs] = prog; |
379 | return 0; | |
380 | } | |
381 | ||
88cda1c9 MKL |
382 | static int |
383 | bpf_object__init_prog_names(struct bpf_object *obj) | |
384 | { | |
385 | Elf_Data *symbols = obj->efile.symbols; | |
386 | struct bpf_program *prog; | |
387 | size_t pi, si; | |
388 | ||
389 | for (pi = 0; pi < obj->nr_programs; pi++) { | |
48cca7e4 | 390 | const char *name = NULL; |
88cda1c9 MKL |
391 | |
392 | prog = &obj->programs[pi]; | |
48cca7e4 AS |
393 | if (prog->idx == obj->efile.text_shndx) { |
394 | name = ".text"; | |
395 | goto skip_search; | |
396 | } | |
88cda1c9 MKL |
397 | |
398 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; | |
399 | si++) { | |
400 | GElf_Sym sym; | |
401 | ||
402 | if (!gelf_getsym(symbols, si, &sym)) | |
403 | continue; | |
404 | if (sym.st_shndx != prog->idx) | |
405 | continue; | |
fe4d44b2 RG |
406 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) |
407 | continue; | |
88cda1c9 MKL |
408 | |
409 | name = elf_strptr(obj->efile.elf, | |
410 | obj->efile.strtabidx, | |
411 | sym.st_name); | |
412 | if (!name) { | |
413 | pr_warning("failed to get sym name string for prog %s\n", | |
414 | prog->section_name); | |
415 | return -LIBBPF_ERRNO__LIBELF; | |
416 | } | |
417 | } | |
418 | ||
419 | if (!name) { | |
420 | pr_warning("failed to find sym for prog %s\n", | |
421 | prog->section_name); | |
422 | return -EINVAL; | |
423 | } | |
48cca7e4 | 424 | skip_search: |
88cda1c9 MKL |
425 | prog->name = strdup(name); |
426 | if (!prog->name) { | |
427 | pr_warning("failed to allocate memory for prog sym %s\n", | |
428 | name); | |
429 | return -ENOMEM; | |
430 | } | |
431 | } | |
432 | ||
433 | return 0; | |
434 | } | |
435 | ||
6c956392 WN |
436 | static struct bpf_object *bpf_object__new(const char *path, |
437 | void *obj_buf, | |
438 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
439 | { |
440 | struct bpf_object *obj; | |
441 | ||
442 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
443 | if (!obj) { | |
444 | pr_warning("alloc memory failed for %s\n", path); | |
6371ca3b | 445 | return ERR_PTR(-ENOMEM); |
1a5e3fb1 WN |
446 | } |
447 | ||
448 | strcpy(obj->path, path); | |
449 | obj->efile.fd = -1; | |
6c956392 WN |
450 | |
451 | /* | |
452 | * Caller of this function should also calls | |
453 | * bpf_object__elf_finish() after data collection to return | |
454 | * obj_buf to user. If not, we should duplicate the buffer to | |
455 | * avoid user freeing them before elf finish. | |
456 | */ | |
457 | obj->efile.obj_buf = obj_buf; | |
458 | obj->efile.obj_buf_sz = obj_buf_sz; | |
666810e8 | 459 | obj->efile.maps_shndx = -1; |
6c956392 | 460 | |
52d3352e | 461 | obj->loaded = false; |
9a208eff WN |
462 | |
463 | INIT_LIST_HEAD(&obj->list); | |
464 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
465 | return obj; |
466 | } | |
467 | ||
468 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
469 | { | |
470 | if (!obj_elf_valid(obj)) | |
471 | return; | |
472 | ||
473 | if (obj->efile.elf) { | |
474 | elf_end(obj->efile.elf); | |
475 | obj->efile.elf = NULL; | |
476 | } | |
bec7d68c | 477 | obj->efile.symbols = NULL; |
b62f06e8 WN |
478 | |
479 | zfree(&obj->efile.reloc); | |
480 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 481 | zclose(obj->efile.fd); |
6c956392 WN |
482 | obj->efile.obj_buf = NULL; |
483 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
484 | } |
485 | ||
486 | static int bpf_object__elf_init(struct bpf_object *obj) | |
487 | { | |
488 | int err = 0; | |
489 | GElf_Ehdr *ep; | |
490 | ||
491 | if (obj_elf_valid(obj)) { | |
492 | pr_warning("elf init: internal error\n"); | |
6371ca3b | 493 | return -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
494 | } |
495 | ||
6c956392 WN |
496 | if (obj->efile.obj_buf_sz > 0) { |
497 | /* | |
498 | * obj_buf should have been validated by | |
499 | * bpf_object__open_buffer(). | |
500 | */ | |
501 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
502 | obj->efile.obj_buf_sz); | |
503 | } else { | |
504 | obj->efile.fd = open(obj->path, O_RDONLY); | |
505 | if (obj->efile.fd < 0) { | |
506 | pr_warning("failed to open %s: %s\n", obj->path, | |
507 | strerror(errno)); | |
508 | return -errno; | |
509 | } | |
510 | ||
511 | obj->efile.elf = elf_begin(obj->efile.fd, | |
512 | LIBBPF_ELF_C_READ_MMAP, | |
513 | NULL); | |
1a5e3fb1 WN |
514 | } |
515 | ||
1a5e3fb1 WN |
516 | if (!obj->efile.elf) { |
517 | pr_warning("failed to open %s as ELF file\n", | |
518 | obj->path); | |
6371ca3b | 519 | err = -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
520 | goto errout; |
521 | } | |
522 | ||
523 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
524 | pr_warning("failed to get EHDR from %s\n", | |
525 | obj->path); | |
6371ca3b | 526 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
527 | goto errout; |
528 | } | |
529 | ep = &obj->efile.ehdr; | |
530 | ||
9b16137a WN |
531 | /* Old LLVM set e_machine to EM_NONE */ |
532 | if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { | |
1a5e3fb1 WN |
533 | pr_warning("%s is not an eBPF object file\n", |
534 | obj->path); | |
6371ca3b | 535 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
536 | goto errout; |
537 | } | |
538 | ||
539 | return 0; | |
540 | errout: | |
541 | bpf_object__elf_finish(obj); | |
542 | return err; | |
543 | } | |
544 | ||
cc4228d5 WN |
545 | static int |
546 | bpf_object__check_endianness(struct bpf_object *obj) | |
547 | { | |
548 | static unsigned int const endian = 1; | |
549 | ||
550 | switch (obj->efile.ehdr.e_ident[EI_DATA]) { | |
551 | case ELFDATA2LSB: | |
552 | /* We are big endian, BPF obj is little endian. */ | |
553 | if (*(unsigned char const *)&endian != 1) | |
554 | goto mismatch; | |
555 | break; | |
556 | ||
557 | case ELFDATA2MSB: | |
558 | /* We are little endian, BPF obj is big endian. */ | |
559 | if (*(unsigned char const *)&endian != 0) | |
560 | goto mismatch; | |
561 | break; | |
562 | default: | |
6371ca3b | 563 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
564 | } |
565 | ||
566 | return 0; | |
567 | ||
568 | mismatch: | |
569 | pr_warning("Error: endianness mismatch.\n"); | |
6371ca3b | 570 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
571 | } |
572 | ||
cb1e5e96 WN |
573 | static int |
574 | bpf_object__init_license(struct bpf_object *obj, | |
575 | void *data, size_t size) | |
576 | { | |
577 | memcpy(obj->license, data, | |
578 | min(size, sizeof(obj->license) - 1)); | |
579 | pr_debug("license of %s is %s\n", obj->path, obj->license); | |
580 | return 0; | |
581 | } | |
582 | ||
583 | static int | |
584 | bpf_object__init_kversion(struct bpf_object *obj, | |
585 | void *data, size_t size) | |
586 | { | |
587 | u32 kver; | |
588 | ||
589 | if (size != sizeof(kver)) { | |
590 | pr_warning("invalid kver section in %s\n", obj->path); | |
6371ca3b | 591 | return -LIBBPF_ERRNO__FORMAT; |
cb1e5e96 WN |
592 | } |
593 | memcpy(&kver, data, sizeof(kver)); | |
594 | obj->kern_version = kver; | |
595 | pr_debug("kernel version of %s is %x\n", obj->path, | |
596 | obj->kern_version); | |
597 | return 0; | |
598 | } | |
599 | ||
4708bbda EL |
600 | static int compare_bpf_map(const void *_a, const void *_b) |
601 | { | |
602 | const struct bpf_map *a = _a; | |
603 | const struct bpf_map *b = _b; | |
9d759a9b | 604 | |
4708bbda | 605 | return a->offset - b->offset; |
0b3d1efa WN |
606 | } |
607 | ||
973170e6 | 608 | static int |
4708bbda | 609 | bpf_object__init_maps(struct bpf_object *obj) |
561bbcca | 610 | { |
b13c5c14 | 611 | int i, map_idx, map_def_sz, nr_maps = 0; |
4708bbda EL |
612 | Elf_Scn *scn; |
613 | Elf_Data *data; | |
561bbcca WN |
614 | Elf_Data *symbols = obj->efile.symbols; |
615 | ||
4708bbda EL |
616 | if (obj->efile.maps_shndx < 0) |
617 | return -EINVAL; | |
618 | if (!symbols) | |
619 | return -EINVAL; | |
620 | ||
621 | scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); | |
622 | if (scn) | |
623 | data = elf_getdata(scn, NULL); | |
624 | if (!scn || !data) { | |
625 | pr_warning("failed to get Elf_Data from map section %d\n", | |
626 | obj->efile.maps_shndx); | |
973170e6 | 627 | return -EINVAL; |
4708bbda | 628 | } |
561bbcca | 629 | |
4708bbda EL |
630 | /* |
631 | * Count number of maps. Each map has a name. | |
632 | * Array of maps is not supported: only the first element is | |
633 | * considered. | |
634 | * | |
635 | * TODO: Detect array of map and report error. | |
636 | */ | |
561bbcca WN |
637 | for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { |
638 | GElf_Sym sym; | |
4708bbda EL |
639 | |
640 | if (!gelf_getsym(symbols, i, &sym)) | |
641 | continue; | |
642 | if (sym.st_shndx != obj->efile.maps_shndx) | |
643 | continue; | |
644 | nr_maps++; | |
645 | } | |
646 | ||
647 | /* Alloc obj->maps and fill nr_maps. */ | |
648 | pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, | |
649 | nr_maps, data->d_size); | |
650 | ||
651 | if (!nr_maps) | |
652 | return 0; | |
653 | ||
b13c5c14 CG |
654 | /* Assume equally sized map definitions */ |
655 | map_def_sz = data->d_size / nr_maps; | |
656 | if (!data->d_size || (data->d_size % nr_maps) != 0) { | |
657 | pr_warning("unable to determine map definition size " | |
658 | "section %s, %d maps in %zd bytes\n", | |
659 | obj->path, nr_maps, data->d_size); | |
660 | return -EINVAL; | |
661 | } | |
662 | ||
4708bbda EL |
663 | obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); |
664 | if (!obj->maps) { | |
665 | pr_warning("alloc maps for object failed\n"); | |
666 | return -ENOMEM; | |
667 | } | |
668 | obj->nr_maps = nr_maps; | |
669 | ||
670 | /* | |
671 | * fill all fd with -1 so won't close incorrect | |
672 | * fd (fd=0 is stdin) when failure (zclose won't close | |
673 | * negative fd)). | |
674 | */ | |
675 | for (i = 0; i < nr_maps; i++) | |
676 | obj->maps[i].fd = -1; | |
677 | ||
678 | /* | |
679 | * Fill obj->maps using data in "maps" section. | |
680 | */ | |
681 | for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { | |
682 | GElf_Sym sym; | |
561bbcca | 683 | const char *map_name; |
4708bbda | 684 | struct bpf_map_def *def; |
561bbcca WN |
685 | |
686 | if (!gelf_getsym(symbols, i, &sym)) | |
687 | continue; | |
666810e8 | 688 | if (sym.st_shndx != obj->efile.maps_shndx) |
561bbcca WN |
689 | continue; |
690 | ||
691 | map_name = elf_strptr(obj->efile.elf, | |
77ba9a5b | 692 | obj->efile.strtabidx, |
561bbcca | 693 | sym.st_name); |
4708bbda | 694 | obj->maps[map_idx].offset = sym.st_value; |
b13c5c14 | 695 | if (sym.st_value + map_def_sz > data->d_size) { |
4708bbda EL |
696 | pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", |
697 | obj->path, map_name); | |
698 | return -EINVAL; | |
561bbcca | 699 | } |
4708bbda | 700 | |
561bbcca | 701 | obj->maps[map_idx].name = strdup(map_name); |
973170e6 WN |
702 | if (!obj->maps[map_idx].name) { |
703 | pr_warning("failed to alloc map name\n"); | |
704 | return -ENOMEM; | |
705 | } | |
4708bbda | 706 | pr_debug("map %d is \"%s\"\n", map_idx, |
561bbcca | 707 | obj->maps[map_idx].name); |
4708bbda | 708 | def = (struct bpf_map_def *)(data->d_buf + sym.st_value); |
b13c5c14 CG |
709 | /* |
710 | * If the definition of the map in the object file fits in | |
711 | * bpf_map_def, copy it. Any extra fields in our version | |
712 | * of bpf_map_def will default to zero as a result of the | |
713 | * calloc above. | |
714 | */ | |
715 | if (map_def_sz <= sizeof(struct bpf_map_def)) { | |
716 | memcpy(&obj->maps[map_idx].def, def, map_def_sz); | |
717 | } else { | |
718 | /* | |
719 | * Here the map structure being read is bigger than what | |
720 | * we expect, truncate if the excess bits are all zero. | |
721 | * If they are not zero, reject this map as | |
722 | * incompatible. | |
723 | */ | |
724 | char *b; | |
725 | for (b = ((char *)def) + sizeof(struct bpf_map_def); | |
726 | b < ((char *)def) + map_def_sz; b++) { | |
727 | if (*b != 0) { | |
728 | pr_warning("maps section in %s: \"%s\" " | |
729 | "has unrecognized, non-zero " | |
730 | "options\n", | |
731 | obj->path, map_name); | |
732 | return -EINVAL; | |
733 | } | |
734 | } | |
735 | memcpy(&obj->maps[map_idx].def, def, | |
736 | sizeof(struct bpf_map_def)); | |
737 | } | |
4708bbda | 738 | map_idx++; |
561bbcca | 739 | } |
4708bbda EL |
740 | |
741 | qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); | |
b13c5c14 | 742 | return 0; |
561bbcca WN |
743 | } |
744 | ||
29603665 WN |
745 | static int bpf_object__elf_collect(struct bpf_object *obj) |
746 | { | |
747 | Elf *elf = obj->efile.elf; | |
748 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
749 | Elf_Scn *scn = NULL; | |
666810e8 | 750 | int idx = 0, err = 0; |
29603665 WN |
751 | |
752 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
753 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
754 | pr_warning("failed to get e_shstrndx from %s\n", | |
755 | obj->path); | |
6371ca3b | 756 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
757 | } |
758 | ||
759 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
760 | char *name; | |
761 | GElf_Shdr sh; | |
762 | Elf_Data *data; | |
763 | ||
764 | idx++; | |
765 | if (gelf_getshdr(scn, &sh) != &sh) { | |
766 | pr_warning("failed to get section header from %s\n", | |
767 | obj->path); | |
6371ca3b | 768 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
769 | goto out; |
770 | } | |
771 | ||
772 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
773 | if (!name) { | |
774 | pr_warning("failed to get section name from %s\n", | |
775 | obj->path); | |
6371ca3b | 776 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
777 | goto out; |
778 | } | |
779 | ||
780 | data = elf_getdata(scn, 0); | |
781 | if (!data) { | |
782 | pr_warning("failed to get section data from %s(%s)\n", | |
783 | name, obj->path); | |
6371ca3b | 784 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
785 | goto out; |
786 | } | |
787 | pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", | |
788 | name, (unsigned long)data->d_size, | |
789 | (int)sh.sh_link, (unsigned long)sh.sh_flags, | |
790 | (int)sh.sh_type); | |
cb1e5e96 WN |
791 | |
792 | if (strcmp(name, "license") == 0) | |
793 | err = bpf_object__init_license(obj, | |
794 | data->d_buf, | |
795 | data->d_size); | |
796 | else if (strcmp(name, "version") == 0) | |
797 | err = bpf_object__init_kversion(obj, | |
798 | data->d_buf, | |
799 | data->d_size); | |
4708bbda | 800 | else if (strcmp(name, "maps") == 0) |
666810e8 | 801 | obj->efile.maps_shndx = idx; |
4708bbda | 802 | else if (sh.sh_type == SHT_SYMTAB) { |
bec7d68c WN |
803 | if (obj->efile.symbols) { |
804 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
805 | obj->path); | |
6371ca3b | 806 | err = -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 807 | } else { |
bec7d68c | 808 | obj->efile.symbols = data; |
77ba9a5b WN |
809 | obj->efile.strtabidx = sh.sh_link; |
810 | } | |
a5b8bd47 WN |
811 | } else if ((sh.sh_type == SHT_PROGBITS) && |
812 | (sh.sh_flags & SHF_EXECINSTR) && | |
813 | (data->d_size > 0)) { | |
48cca7e4 AS |
814 | if (strcmp(name, ".text") == 0) |
815 | obj->efile.text_shndx = idx; | |
a5b8bd47 WN |
816 | err = bpf_object__add_program(obj, data->d_buf, |
817 | data->d_size, name, idx); | |
818 | if (err) { | |
6371ca3b WN |
819 | char errmsg[STRERR_BUFSIZE]; |
820 | ||
a5b8bd47 WN |
821 | strerror_r(-err, errmsg, sizeof(errmsg)); |
822 | pr_warning("failed to alloc program %s (%s): %s", | |
823 | name, obj->path, errmsg); | |
824 | } | |
b62f06e8 WN |
825 | } else if (sh.sh_type == SHT_REL) { |
826 | void *reloc = obj->efile.reloc; | |
827 | int nr_reloc = obj->efile.nr_reloc + 1; | |
828 | ||
829 | reloc = realloc(reloc, | |
830 | sizeof(*obj->efile.reloc) * nr_reloc); | |
831 | if (!reloc) { | |
832 | pr_warning("realloc failed\n"); | |
833 | err = -ENOMEM; | |
834 | } else { | |
835 | int n = nr_reloc - 1; | |
836 | ||
837 | obj->efile.reloc = reloc; | |
838 | obj->efile.nr_reloc = nr_reloc; | |
839 | ||
840 | obj->efile.reloc[n].shdr = sh; | |
841 | obj->efile.reloc[n].data = data; | |
842 | } | |
bec7d68c | 843 | } |
cb1e5e96 WN |
844 | if (err) |
845 | goto out; | |
29603665 | 846 | } |
561bbcca | 847 | |
77ba9a5b WN |
848 | if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { |
849 | pr_warning("Corrupted ELF file: index of strtab invalid\n"); | |
850 | return LIBBPF_ERRNO__FORMAT; | |
851 | } | |
88cda1c9 | 852 | if (obj->efile.maps_shndx >= 0) { |
4708bbda | 853 | err = bpf_object__init_maps(obj); |
88cda1c9 MKL |
854 | if (err) |
855 | goto out; | |
856 | } | |
857 | err = bpf_object__init_prog_names(obj); | |
29603665 WN |
858 | out: |
859 | return err; | |
860 | } | |
861 | ||
34090915 WN |
862 | static struct bpf_program * |
863 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
864 | { | |
865 | struct bpf_program *prog; | |
866 | size_t i; | |
867 | ||
868 | for (i = 0; i < obj->nr_programs; i++) { | |
869 | prog = &obj->programs[i]; | |
870 | if (prog->idx == idx) | |
871 | return prog; | |
872 | } | |
873 | return NULL; | |
874 | } | |
875 | ||
876 | static int | |
48cca7e4 AS |
877 | bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, |
878 | Elf_Data *data, struct bpf_object *obj) | |
34090915 | 879 | { |
48cca7e4 AS |
880 | Elf_Data *symbols = obj->efile.symbols; |
881 | int text_shndx = obj->efile.text_shndx; | |
882 | int maps_shndx = obj->efile.maps_shndx; | |
883 | struct bpf_map *maps = obj->maps; | |
884 | size_t nr_maps = obj->nr_maps; | |
34090915 WN |
885 | int i, nrels; |
886 | ||
887 | pr_debug("collecting relocating info for: '%s'\n", | |
888 | prog->section_name); | |
889 | nrels = shdr->sh_size / shdr->sh_entsize; | |
890 | ||
891 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
892 | if (!prog->reloc_desc) { | |
893 | pr_warning("failed to alloc memory in relocation\n"); | |
894 | return -ENOMEM; | |
895 | } | |
896 | prog->nr_reloc = nrels; | |
897 | ||
898 | for (i = 0; i < nrels; i++) { | |
899 | GElf_Sym sym; | |
900 | GElf_Rel rel; | |
901 | unsigned int insn_idx; | |
902 | struct bpf_insn *insns = prog->insns; | |
903 | size_t map_idx; | |
904 | ||
905 | if (!gelf_getrel(data, i, &rel)) { | |
906 | pr_warning("relocation: failed to get %d reloc\n", i); | |
6371ca3b | 907 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
908 | } |
909 | ||
34090915 WN |
910 | if (!gelf_getsym(symbols, |
911 | GELF_R_SYM(rel.r_info), | |
912 | &sym)) { | |
913 | pr_warning("relocation: symbol %"PRIx64" not found\n", | |
914 | GELF_R_SYM(rel.r_info)); | |
6371ca3b | 915 | return -LIBBPF_ERRNO__FORMAT; |
34090915 | 916 | } |
7d9890ef DM |
917 | pr_debug("relo for %lld value %lld name %d\n", |
918 | (long long) (rel.r_info >> 32), | |
919 | (long long) sym.st_value, sym.st_name); | |
34090915 | 920 | |
48cca7e4 | 921 | if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { |
666810e8 WN |
922 | pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", |
923 | prog->section_name, sym.st_shndx); | |
924 | return -LIBBPF_ERRNO__RELOC; | |
925 | } | |
926 | ||
927 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); | |
928 | pr_debug("relocation: insn_idx=%u\n", insn_idx); | |
929 | ||
48cca7e4 AS |
930 | if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { |
931 | if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { | |
932 | pr_warning("incorrect bpf_call opcode\n"); | |
933 | return -LIBBPF_ERRNO__RELOC; | |
934 | } | |
935 | prog->reloc_desc[i].type = RELO_CALL; | |
936 | prog->reloc_desc[i].insn_idx = insn_idx; | |
937 | prog->reloc_desc[i].text_off = sym.st_value; | |
938 | continue; | |
939 | } | |
940 | ||
34090915 WN |
941 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { |
942 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
943 | insn_idx, insns[insn_idx].code); | |
6371ca3b | 944 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
945 | } |
946 | ||
94e5adec JS |
947 | /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ |
948 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { | |
949 | if (maps[map_idx].offset == sym.st_value) { | |
950 | pr_debug("relocation: find map %zd (%s) for insn %u\n", | |
951 | map_idx, maps[map_idx].name, insn_idx); | |
952 | break; | |
953 | } | |
954 | } | |
955 | ||
34090915 WN |
956 | if (map_idx >= nr_maps) { |
957 | pr_warning("bpf relocation: map_idx %d large than %d\n", | |
958 | (int)map_idx, (int)nr_maps - 1); | |
6371ca3b | 959 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
960 | } |
961 | ||
48cca7e4 | 962 | prog->reloc_desc[i].type = RELO_LD64; |
34090915 WN |
963 | prog->reloc_desc[i].insn_idx = insn_idx; |
964 | prog->reloc_desc[i].map_idx = map_idx; | |
965 | } | |
966 | return 0; | |
967 | } | |
968 | ||
52d3352e WN |
969 | static int |
970 | bpf_object__create_maps(struct bpf_object *obj) | |
971 | { | |
972 | unsigned int i; | |
52d3352e | 973 | |
9d759a9b WN |
974 | for (i = 0; i < obj->nr_maps; i++) { |
975 | struct bpf_map_def *def = &obj->maps[i].def; | |
976 | int *pfd = &obj->maps[i].fd; | |
52d3352e | 977 | |
88cda1c9 MKL |
978 | *pfd = bpf_create_map_name(def->type, |
979 | obj->maps[i].name, | |
980 | def->key_size, | |
981 | def->value_size, | |
982 | def->max_entries, | |
fe9b5f77 | 983 | def->map_flags); |
52d3352e WN |
984 | if (*pfd < 0) { |
985 | size_t j; | |
986 | int err = *pfd; | |
987 | ||
49bf4b36 EL |
988 | pr_warning("failed to create map (name: '%s'): %s\n", |
989 | obj->maps[i].name, | |
52d3352e WN |
990 | strerror(errno)); |
991 | for (j = 0; j < i; j++) | |
9d759a9b | 992 | zclose(obj->maps[j].fd); |
52d3352e WN |
993 | return err; |
994 | } | |
4708bbda | 995 | pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd); |
52d3352e WN |
996 | } |
997 | ||
52d3352e WN |
998 | return 0; |
999 | } | |
1000 | ||
48cca7e4 AS |
1001 | static int |
1002 | bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, | |
1003 | struct reloc_desc *relo) | |
1004 | { | |
1005 | struct bpf_insn *insn, *new_insn; | |
1006 | struct bpf_program *text; | |
1007 | size_t new_cnt; | |
1008 | ||
1009 | if (relo->type != RELO_CALL) | |
1010 | return -LIBBPF_ERRNO__RELOC; | |
1011 | ||
1012 | if (prog->idx == obj->efile.text_shndx) { | |
1013 | pr_warning("relo in .text insn %d into off %d\n", | |
1014 | relo->insn_idx, relo->text_off); | |
1015 | return -LIBBPF_ERRNO__RELOC; | |
1016 | } | |
1017 | ||
1018 | if (prog->main_prog_cnt == 0) { | |
1019 | text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); | |
1020 | if (!text) { | |
1021 | pr_warning("no .text section found yet relo into text exist\n"); | |
1022 | return -LIBBPF_ERRNO__RELOC; | |
1023 | } | |
1024 | new_cnt = prog->insns_cnt + text->insns_cnt; | |
1025 | new_insn = realloc(prog->insns, new_cnt * sizeof(*insn)); | |
1026 | if (!new_insn) { | |
1027 | pr_warning("oom in prog realloc\n"); | |
1028 | return -ENOMEM; | |
1029 | } | |
1030 | memcpy(new_insn + prog->insns_cnt, text->insns, | |
1031 | text->insns_cnt * sizeof(*insn)); | |
1032 | prog->insns = new_insn; | |
1033 | prog->main_prog_cnt = prog->insns_cnt; | |
1034 | prog->insns_cnt = new_cnt; | |
1035 | } | |
1036 | insn = &prog->insns[relo->insn_idx]; | |
1037 | insn->imm += prog->main_prog_cnt - relo->insn_idx; | |
1038 | pr_debug("added %zd insn from %s to prog %s\n", | |
1039 | text->insns_cnt, text->section_name, prog->section_name); | |
1040 | return 0; | |
1041 | } | |
1042 | ||
8a47a6c5 | 1043 | static int |
9d759a9b | 1044 | bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) |
8a47a6c5 | 1045 | { |
48cca7e4 | 1046 | int i, err; |
8a47a6c5 WN |
1047 | |
1048 | if (!prog || !prog->reloc_desc) | |
1049 | return 0; | |
1050 | ||
1051 | for (i = 0; i < prog->nr_reloc; i++) { | |
48cca7e4 AS |
1052 | if (prog->reloc_desc[i].type == RELO_LD64) { |
1053 | struct bpf_insn *insns = prog->insns; | |
1054 | int insn_idx, map_idx; | |
8a47a6c5 | 1055 | |
48cca7e4 AS |
1056 | insn_idx = prog->reloc_desc[i].insn_idx; |
1057 | map_idx = prog->reloc_desc[i].map_idx; | |
8a47a6c5 | 1058 | |
48cca7e4 AS |
1059 | if (insn_idx >= (int)prog->insns_cnt) { |
1060 | pr_warning("relocation out of range: '%s'\n", | |
1061 | prog->section_name); | |
1062 | return -LIBBPF_ERRNO__RELOC; | |
1063 | } | |
1064 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
1065 | insns[insn_idx].imm = obj->maps[map_idx].fd; | |
1066 | } else { | |
1067 | err = bpf_program__reloc_text(prog, obj, | |
1068 | &prog->reloc_desc[i]); | |
1069 | if (err) | |
1070 | return err; | |
8a47a6c5 | 1071 | } |
8a47a6c5 WN |
1072 | } |
1073 | ||
1074 | zfree(&prog->reloc_desc); | |
1075 | prog->nr_reloc = 0; | |
1076 | return 0; | |
1077 | } | |
1078 | ||
1079 | ||
1080 | static int | |
1081 | bpf_object__relocate(struct bpf_object *obj) | |
1082 | { | |
1083 | struct bpf_program *prog; | |
1084 | size_t i; | |
1085 | int err; | |
1086 | ||
1087 | for (i = 0; i < obj->nr_programs; i++) { | |
1088 | prog = &obj->programs[i]; | |
1089 | ||
9d759a9b | 1090 | err = bpf_program__relocate(prog, obj); |
8a47a6c5 WN |
1091 | if (err) { |
1092 | pr_warning("failed to relocate '%s'\n", | |
1093 | prog->section_name); | |
1094 | return err; | |
1095 | } | |
1096 | } | |
1097 | return 0; | |
1098 | } | |
1099 | ||
34090915 WN |
1100 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
1101 | { | |
1102 | int i, err; | |
1103 | ||
1104 | if (!obj_elf_valid(obj)) { | |
1105 | pr_warning("Internal error: elf object is closed\n"); | |
6371ca3b | 1106 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
1107 | } |
1108 | ||
1109 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
1110 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
1111 | Elf_Data *data = obj->efile.reloc[i].data; | |
1112 | int idx = shdr->sh_info; | |
1113 | struct bpf_program *prog; | |
34090915 WN |
1114 | |
1115 | if (shdr->sh_type != SHT_REL) { | |
1116 | pr_warning("internal error at %d\n", __LINE__); | |
6371ca3b | 1117 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
1118 | } |
1119 | ||
1120 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
1121 | if (!prog) { | |
1122 | pr_warning("relocation failed: no %d section\n", | |
1123 | idx); | |
6371ca3b | 1124 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
1125 | } |
1126 | ||
48cca7e4 | 1127 | err = bpf_program__collect_reloc(prog, |
34090915 | 1128 | shdr, data, |
48cca7e4 | 1129 | obj); |
34090915 | 1130 | if (err) |
6371ca3b | 1131 | return err; |
34090915 WN |
1132 | } |
1133 | return 0; | |
1134 | } | |
1135 | ||
55cffde2 | 1136 | static int |
88cda1c9 | 1137 | load_program(enum bpf_prog_type type, const char *name, struct bpf_insn *insns, |
5f44e4c8 | 1138 | int insns_cnt, char *license, u32 kern_version, int *pfd) |
55cffde2 WN |
1139 | { |
1140 | int ret; | |
1141 | char *log_buf; | |
1142 | ||
1143 | if (!insns || !insns_cnt) | |
1144 | return -EINVAL; | |
1145 | ||
1146 | log_buf = malloc(BPF_LOG_BUF_SIZE); | |
1147 | if (!log_buf) | |
1148 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
1149 | ||
88cda1c9 MKL |
1150 | ret = bpf_load_program_name(type, name, insns, insns_cnt, license, |
1151 | kern_version, log_buf, BPF_LOG_BUF_SIZE); | |
55cffde2 WN |
1152 | |
1153 | if (ret >= 0) { | |
1154 | *pfd = ret; | |
1155 | ret = 0; | |
1156 | goto out; | |
1157 | } | |
1158 | ||
6371ca3b | 1159 | ret = -LIBBPF_ERRNO__LOAD; |
55cffde2 WN |
1160 | pr_warning("load bpf program failed: %s\n", strerror(errno)); |
1161 | ||
6371ca3b WN |
1162 | if (log_buf && log_buf[0] != '\0') { |
1163 | ret = -LIBBPF_ERRNO__VERIFY; | |
55cffde2 WN |
1164 | pr_warning("-- BEGIN DUMP LOG ---\n"); |
1165 | pr_warning("\n%s\n", log_buf); | |
1166 | pr_warning("-- END LOG --\n"); | |
705fa219 WN |
1167 | } else if (insns_cnt >= BPF_MAXINSNS) { |
1168 | pr_warning("Program too large (%d insns), at most %d insns\n", | |
1169 | insns_cnt, BPF_MAXINSNS); | |
1170 | ret = -LIBBPF_ERRNO__PROG2BIG; | |
6371ca3b | 1171 | } else { |
705fa219 WN |
1172 | /* Wrong program type? */ |
1173 | if (type != BPF_PROG_TYPE_KPROBE) { | |
1174 | int fd; | |
1175 | ||
88cda1c9 MKL |
1176 | fd = bpf_load_program_name(BPF_PROG_TYPE_KPROBE, name, |
1177 | insns, insns_cnt, license, | |
1178 | kern_version, NULL, 0); | |
705fa219 WN |
1179 | if (fd >= 0) { |
1180 | close(fd); | |
1181 | ret = -LIBBPF_ERRNO__PROGTYPE; | |
1182 | goto out; | |
1183 | } | |
6371ca3b | 1184 | } |
705fa219 WN |
1185 | |
1186 | if (log_buf) | |
1187 | ret = -LIBBPF_ERRNO__KVER; | |
55cffde2 WN |
1188 | } |
1189 | ||
1190 | out: | |
1191 | free(log_buf); | |
1192 | return ret; | |
1193 | } | |
1194 | ||
1195 | static int | |
1196 | bpf_program__load(struct bpf_program *prog, | |
1197 | char *license, u32 kern_version) | |
1198 | { | |
b580563e | 1199 | int err = 0, fd, i; |
55cffde2 | 1200 | |
b580563e WN |
1201 | if (prog->instances.nr < 0 || !prog->instances.fds) { |
1202 | if (prog->preprocessor) { | |
1203 | pr_warning("Internal error: can't load program '%s'\n", | |
1204 | prog->section_name); | |
1205 | return -LIBBPF_ERRNO__INTERNAL; | |
1206 | } | |
55cffde2 | 1207 | |
b580563e WN |
1208 | prog->instances.fds = malloc(sizeof(int)); |
1209 | if (!prog->instances.fds) { | |
1210 | pr_warning("Not enough memory for BPF fds\n"); | |
1211 | return -ENOMEM; | |
1212 | } | |
1213 | prog->instances.nr = 1; | |
1214 | prog->instances.fds[0] = -1; | |
1215 | } | |
1216 | ||
1217 | if (!prog->preprocessor) { | |
1218 | if (prog->instances.nr != 1) { | |
1219 | pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", | |
1220 | prog->section_name, prog->instances.nr); | |
1221 | } | |
88cda1c9 MKL |
1222 | err = load_program(prog->type, prog->name, prog->insns, |
1223 | prog->insns_cnt, license, kern_version, &fd); | |
b580563e WN |
1224 | if (!err) |
1225 | prog->instances.fds[0] = fd; | |
1226 | goto out; | |
1227 | } | |
1228 | ||
1229 | for (i = 0; i < prog->instances.nr; i++) { | |
1230 | struct bpf_prog_prep_result result; | |
1231 | bpf_program_prep_t preprocessor = prog->preprocessor; | |
1232 | ||
1233 | bzero(&result, sizeof(result)); | |
1234 | err = preprocessor(prog, i, prog->insns, | |
1235 | prog->insns_cnt, &result); | |
1236 | if (err) { | |
1237 | pr_warning("Preprocessing the %dth instance of program '%s' failed\n", | |
1238 | i, prog->section_name); | |
1239 | goto out; | |
1240 | } | |
1241 | ||
1242 | if (!result.new_insn_ptr || !result.new_insn_cnt) { | |
1243 | pr_debug("Skip loading the %dth instance of program '%s'\n", | |
1244 | i, prog->section_name); | |
1245 | prog->instances.fds[i] = -1; | |
1246 | if (result.pfd) | |
1247 | *result.pfd = -1; | |
1248 | continue; | |
1249 | } | |
1250 | ||
88cda1c9 MKL |
1251 | err = load_program(prog->type, prog->name, |
1252 | result.new_insn_ptr, | |
b580563e WN |
1253 | result.new_insn_cnt, |
1254 | license, kern_version, &fd); | |
1255 | ||
1256 | if (err) { | |
1257 | pr_warning("Loading the %dth instance of program '%s' failed\n", | |
1258 | i, prog->section_name); | |
1259 | goto out; | |
1260 | } | |
1261 | ||
1262 | if (result.pfd) | |
1263 | *result.pfd = fd; | |
1264 | prog->instances.fds[i] = fd; | |
1265 | } | |
1266 | out: | |
55cffde2 WN |
1267 | if (err) |
1268 | pr_warning("failed to load program '%s'\n", | |
1269 | prog->section_name); | |
1270 | zfree(&prog->insns); | |
1271 | prog->insns_cnt = 0; | |
1272 | return err; | |
1273 | } | |
1274 | ||
1275 | static int | |
1276 | bpf_object__load_progs(struct bpf_object *obj) | |
1277 | { | |
1278 | size_t i; | |
1279 | int err; | |
1280 | ||
1281 | for (i = 0; i < obj->nr_programs; i++) { | |
48cca7e4 AS |
1282 | if (obj->programs[i].idx == obj->efile.text_shndx) |
1283 | continue; | |
55cffde2 WN |
1284 | err = bpf_program__load(&obj->programs[i], |
1285 | obj->license, | |
1286 | obj->kern_version); | |
1287 | if (err) | |
1288 | return err; | |
1289 | } | |
1290 | return 0; | |
1291 | } | |
1292 | ||
cb1e5e96 WN |
1293 | static int bpf_object__validate(struct bpf_object *obj) |
1294 | { | |
1295 | if (obj->kern_version == 0) { | |
1296 | pr_warning("%s doesn't provide kernel version\n", | |
1297 | obj->path); | |
6371ca3b | 1298 | return -LIBBPF_ERRNO__KVERSION; |
cb1e5e96 WN |
1299 | } |
1300 | return 0; | |
1301 | } | |
1302 | ||
1a5e3fb1 | 1303 | static struct bpf_object * |
6c956392 | 1304 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) |
1a5e3fb1 WN |
1305 | { |
1306 | struct bpf_object *obj; | |
6371ca3b | 1307 | int err; |
1a5e3fb1 WN |
1308 | |
1309 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
1310 | pr_warning("failed to init libelf for %s\n", path); | |
6371ca3b | 1311 | return ERR_PTR(-LIBBPF_ERRNO__LIBELF); |
1a5e3fb1 WN |
1312 | } |
1313 | ||
6c956392 | 1314 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
6371ca3b WN |
1315 | if (IS_ERR(obj)) |
1316 | return obj; | |
1a5e3fb1 | 1317 | |
6371ca3b WN |
1318 | CHECK_ERR(bpf_object__elf_init(obj), err, out); |
1319 | CHECK_ERR(bpf_object__check_endianness(obj), err, out); | |
1320 | CHECK_ERR(bpf_object__elf_collect(obj), err, out); | |
1321 | CHECK_ERR(bpf_object__collect_reloc(obj), err, out); | |
1322 | CHECK_ERR(bpf_object__validate(obj), err, out); | |
1a5e3fb1 WN |
1323 | |
1324 | bpf_object__elf_finish(obj); | |
1325 | return obj; | |
1326 | out: | |
1327 | bpf_object__close(obj); | |
6371ca3b | 1328 | return ERR_PTR(err); |
1a5e3fb1 WN |
1329 | } |
1330 | ||
1331 | struct bpf_object *bpf_object__open(const char *path) | |
1332 | { | |
1333 | /* param validation */ | |
1334 | if (!path) | |
1335 | return NULL; | |
1336 | ||
1337 | pr_debug("loading %s\n", path); | |
1338 | ||
6c956392 WN |
1339 | return __bpf_object__open(path, NULL, 0); |
1340 | } | |
1341 | ||
1342 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
acf860ae WN |
1343 | size_t obj_buf_sz, |
1344 | const char *name) | |
6c956392 | 1345 | { |
acf860ae WN |
1346 | char tmp_name[64]; |
1347 | ||
6c956392 WN |
1348 | /* param validation */ |
1349 | if (!obj_buf || obj_buf_sz <= 0) | |
1350 | return NULL; | |
1351 | ||
acf860ae WN |
1352 | if (!name) { |
1353 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
1354 | (unsigned long)obj_buf, | |
1355 | (unsigned long)obj_buf_sz); | |
1356 | tmp_name[sizeof(tmp_name) - 1] = '\0'; | |
1357 | name = tmp_name; | |
1358 | } | |
1359 | pr_debug("loading object '%s' from buffer\n", | |
1360 | name); | |
6c956392 | 1361 | |
acf860ae | 1362 | return __bpf_object__open(name, obj_buf, obj_buf_sz); |
1a5e3fb1 WN |
1363 | } |
1364 | ||
52d3352e WN |
1365 | int bpf_object__unload(struct bpf_object *obj) |
1366 | { | |
1367 | size_t i; | |
1368 | ||
1369 | if (!obj) | |
1370 | return -EINVAL; | |
1371 | ||
9d759a9b WN |
1372 | for (i = 0; i < obj->nr_maps; i++) |
1373 | zclose(obj->maps[i].fd); | |
52d3352e | 1374 | |
55cffde2 WN |
1375 | for (i = 0; i < obj->nr_programs; i++) |
1376 | bpf_program__unload(&obj->programs[i]); | |
1377 | ||
52d3352e WN |
1378 | return 0; |
1379 | } | |
1380 | ||
1381 | int bpf_object__load(struct bpf_object *obj) | |
1382 | { | |
6371ca3b WN |
1383 | int err; |
1384 | ||
52d3352e WN |
1385 | if (!obj) |
1386 | return -EINVAL; | |
1387 | ||
1388 | if (obj->loaded) { | |
1389 | pr_warning("object should not be loaded twice\n"); | |
1390 | return -EINVAL; | |
1391 | } | |
1392 | ||
1393 | obj->loaded = true; | |
6371ca3b WN |
1394 | |
1395 | CHECK_ERR(bpf_object__create_maps(obj), err, out); | |
1396 | CHECK_ERR(bpf_object__relocate(obj), err, out); | |
1397 | CHECK_ERR(bpf_object__load_progs(obj), err, out); | |
52d3352e WN |
1398 | |
1399 | return 0; | |
1400 | out: | |
1401 | bpf_object__unload(obj); | |
1402 | pr_warning("failed to load object '%s'\n", obj->path); | |
6371ca3b | 1403 | return err; |
52d3352e WN |
1404 | } |
1405 | ||
f367540c JS |
1406 | static int check_path(const char *path) |
1407 | { | |
1408 | struct statfs st_fs; | |
1409 | char *dname, *dir; | |
1410 | int err = 0; | |
1411 | ||
1412 | if (path == NULL) | |
1413 | return -EINVAL; | |
1414 | ||
1415 | dname = strdup(path); | |
1416 | if (dname == NULL) | |
1417 | return -ENOMEM; | |
1418 | ||
1419 | dir = dirname(dname); | |
1420 | if (statfs(dir, &st_fs)) { | |
1421 | pr_warning("failed to statfs %s: %s\n", dir, strerror(errno)); | |
1422 | err = -errno; | |
1423 | } | |
1424 | free(dname); | |
1425 | ||
1426 | if (!err && st_fs.f_type != BPF_FS_MAGIC) { | |
1427 | pr_warning("specified path %s is not on BPF FS\n", path); | |
1428 | err = -EINVAL; | |
1429 | } | |
1430 | ||
1431 | return err; | |
1432 | } | |
1433 | ||
1434 | int bpf_program__pin_instance(struct bpf_program *prog, const char *path, | |
1435 | int instance) | |
1436 | { | |
1437 | int err; | |
1438 | ||
1439 | err = check_path(path); | |
1440 | if (err) | |
1441 | return err; | |
1442 | ||
1443 | if (prog == NULL) { | |
1444 | pr_warning("invalid program pointer\n"); | |
1445 | return -EINVAL; | |
1446 | } | |
1447 | ||
1448 | if (instance < 0 || instance >= prog->instances.nr) { | |
1449 | pr_warning("invalid prog instance %d of prog %s (max %d)\n", | |
1450 | instance, prog->section_name, prog->instances.nr); | |
1451 | return -EINVAL; | |
1452 | } | |
1453 | ||
1454 | if (bpf_obj_pin(prog->instances.fds[instance], path)) { | |
1455 | pr_warning("failed to pin program: %s\n", strerror(errno)); | |
1456 | return -errno; | |
1457 | } | |
1458 | pr_debug("pinned program '%s'\n", path); | |
1459 | ||
1460 | return 0; | |
1461 | } | |
1462 | ||
1463 | static int make_dir(const char *path) | |
1464 | { | |
1465 | int err = 0; | |
1466 | ||
1467 | if (mkdir(path, 0700) && errno != EEXIST) | |
1468 | err = -errno; | |
1469 | ||
1470 | if (err) | |
1471 | pr_warning("failed to mkdir %s: %s\n", path, strerror(-err)); | |
1472 | return err; | |
1473 | } | |
1474 | ||
1475 | int bpf_program__pin(struct bpf_program *prog, const char *path) | |
1476 | { | |
1477 | int i, err; | |
1478 | ||
1479 | err = check_path(path); | |
1480 | if (err) | |
1481 | return err; | |
1482 | ||
1483 | if (prog == NULL) { | |
1484 | pr_warning("invalid program pointer\n"); | |
1485 | return -EINVAL; | |
1486 | } | |
1487 | ||
1488 | if (prog->instances.nr <= 0) { | |
1489 | pr_warning("no instances of prog %s to pin\n", | |
1490 | prog->section_name); | |
1491 | return -EINVAL; | |
1492 | } | |
1493 | ||
1494 | err = make_dir(path); | |
1495 | if (err) | |
1496 | return err; | |
1497 | ||
1498 | for (i = 0; i < prog->instances.nr; i++) { | |
1499 | char buf[PATH_MAX]; | |
1500 | int len; | |
1501 | ||
1502 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
1503 | if (len < 0) | |
1504 | return -EINVAL; | |
1505 | else if (len >= PATH_MAX) | |
1506 | return -ENAMETOOLONG; | |
1507 | ||
1508 | err = bpf_program__pin_instance(prog, buf, i); | |
1509 | if (err) | |
1510 | return err; | |
1511 | } | |
1512 | ||
1513 | return 0; | |
1514 | } | |
1515 | ||
b6989f35 JS |
1516 | int bpf_map__pin(struct bpf_map *map, const char *path) |
1517 | { | |
1518 | int err; | |
1519 | ||
1520 | err = check_path(path); | |
1521 | if (err) | |
1522 | return err; | |
1523 | ||
1524 | if (map == NULL) { | |
1525 | pr_warning("invalid map pointer\n"); | |
1526 | return -EINVAL; | |
1527 | } | |
1528 | ||
1529 | if (bpf_obj_pin(map->fd, path)) { | |
1530 | pr_warning("failed to pin map: %s\n", strerror(errno)); | |
1531 | return -errno; | |
1532 | } | |
1533 | ||
1534 | pr_debug("pinned map '%s'\n", path); | |
1535 | return 0; | |
1536 | } | |
1537 | ||
d5148d85 JS |
1538 | int bpf_object__pin(struct bpf_object *obj, const char *path) |
1539 | { | |
1540 | struct bpf_program *prog; | |
1541 | struct bpf_map *map; | |
1542 | int err; | |
1543 | ||
1544 | if (!obj) | |
1545 | return -ENOENT; | |
1546 | ||
1547 | if (!obj->loaded) { | |
1548 | pr_warning("object not yet loaded; load it first\n"); | |
1549 | return -ENOENT; | |
1550 | } | |
1551 | ||
1552 | err = make_dir(path); | |
1553 | if (err) | |
1554 | return err; | |
1555 | ||
1556 | bpf_map__for_each(map, obj) { | |
1557 | char buf[PATH_MAX]; | |
1558 | int len; | |
1559 | ||
1560 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
1561 | bpf_map__name(map)); | |
1562 | if (len < 0) | |
1563 | return -EINVAL; | |
1564 | else if (len >= PATH_MAX) | |
1565 | return -ENAMETOOLONG; | |
1566 | ||
1567 | err = bpf_map__pin(map, buf); | |
1568 | if (err) | |
1569 | return err; | |
1570 | } | |
1571 | ||
1572 | bpf_object__for_each_program(prog, obj) { | |
1573 | char buf[PATH_MAX]; | |
1574 | int len; | |
1575 | ||
1576 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
1577 | prog->section_name); | |
1578 | if (len < 0) | |
1579 | return -EINVAL; | |
1580 | else if (len >= PATH_MAX) | |
1581 | return -ENAMETOOLONG; | |
1582 | ||
1583 | err = bpf_program__pin(prog, buf); | |
1584 | if (err) | |
1585 | return err; | |
1586 | } | |
1587 | ||
1588 | return 0; | |
1589 | } | |
1590 | ||
1a5e3fb1 WN |
1591 | void bpf_object__close(struct bpf_object *obj) |
1592 | { | |
a5b8bd47 WN |
1593 | size_t i; |
1594 | ||
1a5e3fb1 WN |
1595 | if (!obj) |
1596 | return; | |
1597 | ||
10931d24 WN |
1598 | if (obj->clear_priv) |
1599 | obj->clear_priv(obj, obj->priv); | |
1600 | ||
1a5e3fb1 | 1601 | bpf_object__elf_finish(obj); |
52d3352e | 1602 | bpf_object__unload(obj); |
1a5e3fb1 | 1603 | |
9d759a9b | 1604 | for (i = 0; i < obj->nr_maps; i++) { |
561bbcca | 1605 | zfree(&obj->maps[i].name); |
9d759a9b WN |
1606 | if (obj->maps[i].clear_priv) |
1607 | obj->maps[i].clear_priv(&obj->maps[i], | |
1608 | obj->maps[i].priv); | |
1609 | obj->maps[i].priv = NULL; | |
1610 | obj->maps[i].clear_priv = NULL; | |
1611 | } | |
1612 | zfree(&obj->maps); | |
1613 | obj->nr_maps = 0; | |
a5b8bd47 WN |
1614 | |
1615 | if (obj->programs && obj->nr_programs) { | |
1616 | for (i = 0; i < obj->nr_programs; i++) | |
1617 | bpf_program__exit(&obj->programs[i]); | |
1618 | } | |
1619 | zfree(&obj->programs); | |
1620 | ||
9a208eff | 1621 | list_del(&obj->list); |
1a5e3fb1 WN |
1622 | free(obj); |
1623 | } | |
aa9b1ac3 | 1624 | |
9a208eff WN |
1625 | struct bpf_object * |
1626 | bpf_object__next(struct bpf_object *prev) | |
1627 | { | |
1628 | struct bpf_object *next; | |
1629 | ||
1630 | if (!prev) | |
1631 | next = list_first_entry(&bpf_objects_list, | |
1632 | struct bpf_object, | |
1633 | list); | |
1634 | else | |
1635 | next = list_next_entry(prev, list); | |
1636 | ||
1637 | /* Empty list is noticed here so don't need checking on entry. */ | |
1638 | if (&next->list == &bpf_objects_list) | |
1639 | return NULL; | |
1640 | ||
1641 | return next; | |
1642 | } | |
1643 | ||
a7fe0450 | 1644 | const char *bpf_object__name(struct bpf_object *obj) |
acf860ae | 1645 | { |
a7fe0450 | 1646 | return obj ? obj->path : ERR_PTR(-EINVAL); |
acf860ae WN |
1647 | } |
1648 | ||
a7fe0450 | 1649 | unsigned int bpf_object__kversion(struct bpf_object *obj) |
45825d8a | 1650 | { |
a7fe0450 | 1651 | return obj ? obj->kern_version : 0; |
45825d8a WN |
1652 | } |
1653 | ||
10931d24 WN |
1654 | int bpf_object__set_priv(struct bpf_object *obj, void *priv, |
1655 | bpf_object_clear_priv_t clear_priv) | |
1656 | { | |
1657 | if (obj->priv && obj->clear_priv) | |
1658 | obj->clear_priv(obj, obj->priv); | |
1659 | ||
1660 | obj->priv = priv; | |
1661 | obj->clear_priv = clear_priv; | |
1662 | return 0; | |
1663 | } | |
1664 | ||
1665 | void *bpf_object__priv(struct bpf_object *obj) | |
1666 | { | |
1667 | return obj ? obj->priv : ERR_PTR(-EINVAL); | |
1668 | } | |
1669 | ||
aa9b1ac3 WN |
1670 | struct bpf_program * |
1671 | bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) | |
1672 | { | |
1673 | size_t idx; | |
1674 | ||
1675 | if (!obj->programs) | |
1676 | return NULL; | |
1677 | /* First handler */ | |
1678 | if (prev == NULL) | |
1679 | return &obj->programs[0]; | |
1680 | ||
1681 | if (prev->obj != obj) { | |
1682 | pr_warning("error: program handler doesn't match object\n"); | |
1683 | return NULL; | |
1684 | } | |
1685 | ||
1686 | idx = (prev - obj->programs) + 1; | |
1687 | if (idx >= obj->nr_programs) | |
1688 | return NULL; | |
1689 | return &obj->programs[idx]; | |
1690 | } | |
1691 | ||
edb13ed4 ACM |
1692 | int bpf_program__set_priv(struct bpf_program *prog, void *priv, |
1693 | bpf_program_clear_priv_t clear_priv) | |
aa9b1ac3 WN |
1694 | { |
1695 | if (prog->priv && prog->clear_priv) | |
1696 | prog->clear_priv(prog, prog->priv); | |
1697 | ||
1698 | prog->priv = priv; | |
1699 | prog->clear_priv = clear_priv; | |
1700 | return 0; | |
1701 | } | |
1702 | ||
be834ffb | 1703 | void *bpf_program__priv(struct bpf_program *prog) |
aa9b1ac3 | 1704 | { |
be834ffb | 1705 | return prog ? prog->priv : ERR_PTR(-EINVAL); |
aa9b1ac3 WN |
1706 | } |
1707 | ||
715f8db9 | 1708 | const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
1709 | { |
1710 | const char *title; | |
1711 | ||
1712 | title = prog->section_name; | |
715f8db9 | 1713 | if (needs_copy) { |
aa9b1ac3 WN |
1714 | title = strdup(title); |
1715 | if (!title) { | |
1716 | pr_warning("failed to strdup program title\n"); | |
6371ca3b | 1717 | return ERR_PTR(-ENOMEM); |
aa9b1ac3 WN |
1718 | } |
1719 | } | |
1720 | ||
1721 | return title; | |
1722 | } | |
1723 | ||
1724 | int bpf_program__fd(struct bpf_program *prog) | |
1725 | { | |
b580563e WN |
1726 | return bpf_program__nth_fd(prog, 0); |
1727 | } | |
1728 | ||
1729 | int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, | |
1730 | bpf_program_prep_t prep) | |
1731 | { | |
1732 | int *instances_fds; | |
1733 | ||
1734 | if (nr_instances <= 0 || !prep) | |
1735 | return -EINVAL; | |
1736 | ||
1737 | if (prog->instances.nr > 0 || prog->instances.fds) { | |
1738 | pr_warning("Can't set pre-processor after loading\n"); | |
1739 | return -EINVAL; | |
1740 | } | |
1741 | ||
1742 | instances_fds = malloc(sizeof(int) * nr_instances); | |
1743 | if (!instances_fds) { | |
1744 | pr_warning("alloc memory failed for fds\n"); | |
1745 | return -ENOMEM; | |
1746 | } | |
1747 | ||
1748 | /* fill all fd with -1 */ | |
1749 | memset(instances_fds, -1, sizeof(int) * nr_instances); | |
1750 | ||
1751 | prog->instances.nr = nr_instances; | |
1752 | prog->instances.fds = instances_fds; | |
1753 | prog->preprocessor = prep; | |
1754 | return 0; | |
1755 | } | |
1756 | ||
1757 | int bpf_program__nth_fd(struct bpf_program *prog, int n) | |
1758 | { | |
1759 | int fd; | |
1760 | ||
1761 | if (n >= prog->instances.nr || n < 0) { | |
1762 | pr_warning("Can't get the %dth fd from program %s: only %d instances\n", | |
1763 | n, prog->section_name, prog->instances.nr); | |
1764 | return -EINVAL; | |
1765 | } | |
1766 | ||
1767 | fd = prog->instances.fds[n]; | |
1768 | if (fd < 0) { | |
1769 | pr_warning("%dth instance of program '%s' is invalid\n", | |
1770 | n, prog->section_name); | |
1771 | return -ENOENT; | |
1772 | } | |
1773 | ||
1774 | return fd; | |
aa9b1ac3 | 1775 | } |
9d759a9b | 1776 | |
dd26b7f5 | 1777 | void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) |
5f44e4c8 WN |
1778 | { |
1779 | prog->type = type; | |
1780 | } | |
1781 | ||
5f44e4c8 WN |
1782 | static bool bpf_program__is_type(struct bpf_program *prog, |
1783 | enum bpf_prog_type type) | |
1784 | { | |
1785 | return prog ? (prog->type == type) : false; | |
1786 | } | |
1787 | ||
ed794073 JS |
1788 | #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ |
1789 | int bpf_program__set_##NAME(struct bpf_program *prog) \ | |
1790 | { \ | |
1791 | if (!prog) \ | |
1792 | return -EINVAL; \ | |
1793 | bpf_program__set_type(prog, TYPE); \ | |
1794 | return 0; \ | |
1795 | } \ | |
1796 | \ | |
1797 | bool bpf_program__is_##NAME(struct bpf_program *prog) \ | |
1798 | { \ | |
1799 | return bpf_program__is_type(prog, TYPE); \ | |
1800 | } \ | |
1801 | ||
7803ba73 | 1802 | BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); |
ed794073 | 1803 | BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); |
7803ba73 JS |
1804 | BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); |
1805 | BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); | |
ed794073 | 1806 | BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); |
7803ba73 JS |
1807 | BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); |
1808 | BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); | |
5f44e4c8 | 1809 | |
d77be689 | 1810 | #define BPF_PROG_SEC(string, type) { string, sizeof(string) - 1, type } |
583c9009 RG |
1811 | static const struct { |
1812 | const char *sec; | |
1813 | size_t len; | |
1814 | enum bpf_prog_type prog_type; | |
1815 | } section_names[] = { | |
1816 | BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), | |
1817 | BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), | |
1818 | BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), | |
0badd331 QM |
1819 | BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), |
1820 | BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), | |
583c9009 RG |
1821 | BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), |
1822 | BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), | |
1823 | BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), | |
1824 | BPF_PROG_SEC("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), | |
1825 | BPF_PROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK), | |
1826 | BPF_PROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE), | |
0badd331 QM |
1827 | BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), |
1828 | BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), | |
1829 | BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), | |
583c9009 RG |
1830 | BPF_PROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS), |
1831 | BPF_PROG_SEC("sk_skb", BPF_PROG_TYPE_SK_SKB), | |
1832 | }; | |
1833 | #undef BPF_PROG_SEC | |
1834 | ||
1835 | static enum bpf_prog_type bpf_program__guess_type(struct bpf_program *prog) | |
1836 | { | |
1837 | int i; | |
1838 | ||
1839 | if (!prog->section_name) | |
1840 | goto err; | |
1841 | ||
1842 | for (i = 0; i < ARRAY_SIZE(section_names); i++) | |
1843 | if (strncmp(prog->section_name, section_names[i].sec, | |
1844 | section_names[i].len) == 0) | |
1845 | return section_names[i].prog_type; | |
1846 | ||
1847 | err: | |
1848 | pr_warning("failed to guess program type based on section name %s\n", | |
1849 | prog->section_name); | |
1850 | ||
1851 | return BPF_PROG_TYPE_UNSPEC; | |
1852 | } | |
1853 | ||
6e009e65 | 1854 | int bpf_map__fd(struct bpf_map *map) |
9d759a9b | 1855 | { |
6e009e65 | 1856 | return map ? map->fd : -EINVAL; |
9d759a9b WN |
1857 | } |
1858 | ||
53897a78 | 1859 | const struct bpf_map_def *bpf_map__def(struct bpf_map *map) |
9d759a9b | 1860 | { |
53897a78 | 1861 | return map ? &map->def : ERR_PTR(-EINVAL); |
9d759a9b WN |
1862 | } |
1863 | ||
009ad5d5 | 1864 | const char *bpf_map__name(struct bpf_map *map) |
561bbcca | 1865 | { |
009ad5d5 | 1866 | return map ? map->name : NULL; |
561bbcca WN |
1867 | } |
1868 | ||
edb13ed4 ACM |
1869 | int bpf_map__set_priv(struct bpf_map *map, void *priv, |
1870 | bpf_map_clear_priv_t clear_priv) | |
9d759a9b WN |
1871 | { |
1872 | if (!map) | |
1873 | return -EINVAL; | |
1874 | ||
1875 | if (map->priv) { | |
1876 | if (map->clear_priv) | |
1877 | map->clear_priv(map, map->priv); | |
1878 | } | |
1879 | ||
1880 | map->priv = priv; | |
1881 | map->clear_priv = clear_priv; | |
1882 | return 0; | |
1883 | } | |
1884 | ||
b4cbfa56 | 1885 | void *bpf_map__priv(struct bpf_map *map) |
9d759a9b | 1886 | { |
b4cbfa56 | 1887 | return map ? map->priv : ERR_PTR(-EINVAL); |
9d759a9b WN |
1888 | } |
1889 | ||
1890 | struct bpf_map * | |
1891 | bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) | |
1892 | { | |
1893 | size_t idx; | |
1894 | struct bpf_map *s, *e; | |
1895 | ||
1896 | if (!obj || !obj->maps) | |
1897 | return NULL; | |
1898 | ||
1899 | s = obj->maps; | |
1900 | e = obj->maps + obj->nr_maps; | |
1901 | ||
1902 | if (prev == NULL) | |
1903 | return s; | |
1904 | ||
1905 | if ((prev < s) || (prev >= e)) { | |
1906 | pr_warning("error in %s: map handler doesn't belong to object\n", | |
1907 | __func__); | |
1908 | return NULL; | |
1909 | } | |
1910 | ||
1911 | idx = (prev - obj->maps) + 1; | |
1912 | if (idx >= obj->nr_maps) | |
1913 | return NULL; | |
1914 | return &obj->maps[idx]; | |
1915 | } | |
561bbcca WN |
1916 | |
1917 | struct bpf_map * | |
a7fe0450 | 1918 | bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) |
561bbcca WN |
1919 | { |
1920 | struct bpf_map *pos; | |
1921 | ||
1922 | bpf_map__for_each(pos, obj) { | |
973170e6 | 1923 | if (pos->name && !strcmp(pos->name, name)) |
561bbcca WN |
1924 | return pos; |
1925 | } | |
1926 | return NULL; | |
1927 | } | |
5a6acad1 WN |
1928 | |
1929 | struct bpf_map * | |
1930 | bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) | |
1931 | { | |
1932 | int i; | |
1933 | ||
1934 | for (i = 0; i < obj->nr_maps; i++) { | |
1935 | if (obj->maps[i].offset == offset) | |
1936 | return &obj->maps[i]; | |
1937 | } | |
1938 | return ERR_PTR(-ENOENT); | |
1939 | } | |
e28ff1a8 JS |
1940 | |
1941 | long libbpf_get_error(const void *ptr) | |
1942 | { | |
1943 | if (IS_ERR(ptr)) | |
1944 | return PTR_ERR(ptr); | |
1945 | return 0; | |
1946 | } | |
6f6d33f3 JF |
1947 | |
1948 | int bpf_prog_load(const char *file, enum bpf_prog_type type, | |
1949 | struct bpf_object **pobj, int *prog_fd) | |
1950 | { | |
48cca7e4 | 1951 | struct bpf_program *prog, *first_prog = NULL; |
6f6d33f3 JF |
1952 | struct bpf_object *obj; |
1953 | int err; | |
1954 | ||
1955 | obj = bpf_object__open(file); | |
1956 | if (IS_ERR(obj)) | |
1957 | return -ENOENT; | |
1958 | ||
48cca7e4 AS |
1959 | bpf_object__for_each_program(prog, obj) { |
1960 | /* | |
1961 | * If type is not specified, try to guess it based on | |
1962 | * section name. | |
1963 | */ | |
583c9009 | 1964 | if (type == BPF_PROG_TYPE_UNSPEC) { |
48cca7e4 AS |
1965 | type = bpf_program__guess_type(prog); |
1966 | if (type == BPF_PROG_TYPE_UNSPEC) { | |
1967 | bpf_object__close(obj); | |
1968 | return -EINVAL; | |
1969 | } | |
583c9009 | 1970 | } |
48cca7e4 AS |
1971 | |
1972 | bpf_program__set_type(prog, type); | |
1973 | if (prog->idx != obj->efile.text_shndx && !first_prog) | |
1974 | first_prog = prog; | |
1975 | } | |
1976 | ||
1977 | if (!first_prog) { | |
1978 | pr_warning("object file doesn't contain bpf program\n"); | |
1979 | bpf_object__close(obj); | |
1980 | return -ENOENT; | |
583c9009 RG |
1981 | } |
1982 | ||
6f6d33f3 JF |
1983 | err = bpf_object__load(obj); |
1984 | if (err) { | |
1985 | bpf_object__close(obj); | |
1986 | return -EINVAL; | |
1987 | } | |
1988 | ||
1989 | *pobj = obj; | |
48cca7e4 | 1990 | *prog_fd = bpf_program__fd(first_prog); |
6f6d33f3 JF |
1991 | return 0; |
1992 | } |