Commit | Line | Data |
---|---|---|
1b76c13e WN |
1 | /* |
2 | * Common eBPF ELF object loading operations. | |
3 | * | |
4 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
5 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
6 | * Copyright (C) 2015 Huawei Inc. | |
203d1cac WN |
7 | * |
8 | * This program is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License as published by the Free Software Foundation; | |
11 | * version 2.1 of the License (not later!) | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, | |
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
16 | * GNU Lesser General Public License for more details. | |
17 | * | |
18 | * You should have received a copy of the GNU Lesser General Public | |
19 | * License along with this program; if not, see <http://www.gnu.org/licenses> | |
1b76c13e WN |
20 | */ |
21 | ||
22 | #include <stdlib.h> | |
b3f59d66 WN |
23 | #include <stdio.h> |
24 | #include <stdarg.h> | |
34090915 | 25 | #include <inttypes.h> |
b3f59d66 | 26 | #include <string.h> |
1b76c13e | 27 | #include <unistd.h> |
1a5e3fb1 WN |
28 | #include <fcntl.h> |
29 | #include <errno.h> | |
1b76c13e | 30 | #include <asm/unistd.h> |
cb1e5e96 | 31 | #include <linux/kernel.h> |
1b76c13e | 32 | #include <linux/bpf.h> |
9a208eff | 33 | #include <linux/list.h> |
1a5e3fb1 WN |
34 | #include <libelf.h> |
35 | #include <gelf.h> | |
1b76c13e WN |
36 | |
37 | #include "libbpf.h" | |
52d3352e | 38 | #include "bpf.h" |
b3f59d66 | 39 | |
9b16137a WN |
40 | #ifndef EM_BPF |
41 | #define EM_BPF 247 | |
42 | #endif | |
43 | ||
b3f59d66 WN |
44 | #define __printf(a, b) __attribute__((format(printf, a, b))) |
45 | ||
46 | __printf(1, 2) | |
47 | static int __base_pr(const char *format, ...) | |
48 | { | |
49 | va_list args; | |
50 | int err; | |
51 | ||
52 | va_start(args, format); | |
53 | err = vfprintf(stderr, format, args); | |
54 | va_end(args); | |
55 | return err; | |
56 | } | |
57 | ||
58 | static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; | |
59 | static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; | |
60 | static __printf(1, 2) libbpf_print_fn_t __pr_debug; | |
61 | ||
62 | #define __pr(func, fmt, ...) \ | |
63 | do { \ | |
64 | if ((func)) \ | |
65 | (func)("libbpf: " fmt, ##__VA_ARGS__); \ | |
66 | } while (0) | |
67 | ||
68 | #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) | |
69 | #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) | |
70 | #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) | |
71 | ||
72 | void libbpf_set_print(libbpf_print_fn_t warn, | |
73 | libbpf_print_fn_t info, | |
74 | libbpf_print_fn_t debug) | |
75 | { | |
76 | __pr_warning = warn; | |
77 | __pr_info = info; | |
78 | __pr_debug = debug; | |
79 | } | |
1a5e3fb1 | 80 | |
6371ca3b WN |
81 | #define STRERR_BUFSIZE 128 |
82 | ||
83 | #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) | |
84 | #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) | |
85 | #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) | |
86 | ||
87 | static const char *libbpf_strerror_table[NR_ERRNO] = { | |
88 | [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", | |
89 | [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", | |
90 | [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", | |
de8a63bd | 91 | [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", |
6371ca3b WN |
92 | [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", |
93 | [ERRCODE_OFFSET(RELOC)] = "Relocation failed", | |
94 | [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", | |
95 | [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", | |
96 | [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", | |
705fa219 | 97 | [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type", |
6371ca3b WN |
98 | }; |
99 | ||
100 | int libbpf_strerror(int err, char *buf, size_t size) | |
101 | { | |
102 | if (!buf || !size) | |
103 | return -1; | |
104 | ||
105 | err = err > 0 ? err : -err; | |
106 | ||
107 | if (err < __LIBBPF_ERRNO__START) { | |
108 | int ret; | |
109 | ||
110 | ret = strerror_r(err, buf, size); | |
111 | buf[size - 1] = '\0'; | |
112 | return ret; | |
113 | } | |
114 | ||
115 | if (err < __LIBBPF_ERRNO__END) { | |
116 | const char *msg; | |
117 | ||
118 | msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; | |
119 | snprintf(buf, size, "%s", msg); | |
120 | buf[size - 1] = '\0'; | |
121 | return 0; | |
122 | } | |
123 | ||
124 | snprintf(buf, size, "Unknown libbpf error %d", err); | |
125 | buf[size - 1] = '\0'; | |
126 | return -1; | |
127 | } | |
128 | ||
129 | #define CHECK_ERR(action, err, out) do { \ | |
130 | err = action; \ | |
131 | if (err) \ | |
132 | goto out; \ | |
133 | } while(0) | |
134 | ||
135 | ||
1a5e3fb1 WN |
136 | /* Copied from tools/perf/util/util.h */ |
137 | #ifndef zfree | |
138 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
139 | #endif | |
140 | ||
141 | #ifndef zclose | |
142 | # define zclose(fd) ({ \ | |
143 | int ___err = 0; \ | |
144 | if ((fd) >= 0) \ | |
145 | ___err = close((fd)); \ | |
146 | fd = -1; \ | |
147 | ___err; }) | |
148 | #endif | |
149 | ||
150 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
151 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
152 | #else | |
153 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
154 | #endif | |
155 | ||
a5b8bd47 WN |
156 | /* |
157 | * bpf_prog should be a better name but it has been used in | |
158 | * linux/filter.h. | |
159 | */ | |
160 | struct bpf_program { | |
161 | /* Index in elf obj file, for relocation use. */ | |
162 | int idx; | |
163 | char *section_name; | |
164 | struct bpf_insn *insns; | |
165 | size_t insns_cnt; | |
5f44e4c8 | 166 | enum bpf_prog_type type; |
34090915 WN |
167 | |
168 | struct { | |
169 | int insn_idx; | |
170 | int map_idx; | |
171 | } *reloc_desc; | |
172 | int nr_reloc; | |
55cffde2 | 173 | |
b580563e WN |
174 | struct { |
175 | int nr; | |
176 | int *fds; | |
177 | } instances; | |
178 | bpf_program_prep_t preprocessor; | |
aa9b1ac3 WN |
179 | |
180 | struct bpf_object *obj; | |
181 | void *priv; | |
182 | bpf_program_clear_priv_t clear_priv; | |
a5b8bd47 WN |
183 | }; |
184 | ||
9d759a9b WN |
185 | struct bpf_map { |
186 | int fd; | |
561bbcca | 187 | char *name; |
4708bbda | 188 | size_t offset; |
9d759a9b WN |
189 | struct bpf_map_def def; |
190 | void *priv; | |
191 | bpf_map_clear_priv_t clear_priv; | |
192 | }; | |
193 | ||
9a208eff WN |
194 | static LIST_HEAD(bpf_objects_list); |
195 | ||
1a5e3fb1 | 196 | struct bpf_object { |
cb1e5e96 WN |
197 | char license[64]; |
198 | u32 kern_version; | |
0b3d1efa | 199 | |
a5b8bd47 WN |
200 | struct bpf_program *programs; |
201 | size_t nr_programs; | |
9d759a9b WN |
202 | struct bpf_map *maps; |
203 | size_t nr_maps; | |
204 | ||
52d3352e | 205 | bool loaded; |
a5b8bd47 | 206 | |
1a5e3fb1 WN |
207 | /* |
208 | * Information when doing elf related work. Only valid if fd | |
209 | * is valid. | |
210 | */ | |
211 | struct { | |
212 | int fd; | |
6c956392 WN |
213 | void *obj_buf; |
214 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
215 | Elf *elf; |
216 | GElf_Ehdr ehdr; | |
bec7d68c | 217 | Elf_Data *symbols; |
77ba9a5b | 218 | size_t strtabidx; |
b62f06e8 WN |
219 | struct { |
220 | GElf_Shdr shdr; | |
221 | Elf_Data *data; | |
222 | } *reloc; | |
223 | int nr_reloc; | |
666810e8 | 224 | int maps_shndx; |
1a5e3fb1 | 225 | } efile; |
9a208eff WN |
226 | /* |
227 | * All loaded bpf_object is linked in a list, which is | |
228 | * hidden to caller. bpf_objects__<func> handlers deal with | |
229 | * all objects. | |
230 | */ | |
231 | struct list_head list; | |
10931d24 WN |
232 | |
233 | void *priv; | |
234 | bpf_object_clear_priv_t clear_priv; | |
235 | ||
1a5e3fb1 WN |
236 | char path[]; |
237 | }; | |
238 | #define obj_elf_valid(o) ((o)->efile.elf) | |
239 | ||
55cffde2 WN |
240 | static void bpf_program__unload(struct bpf_program *prog) |
241 | { | |
b580563e WN |
242 | int i; |
243 | ||
55cffde2 WN |
244 | if (!prog) |
245 | return; | |
246 | ||
b580563e WN |
247 | /* |
248 | * If the object is opened but the program was never loaded, | |
249 | * it is possible that prog->instances.nr == -1. | |
250 | */ | |
251 | if (prog->instances.nr > 0) { | |
252 | for (i = 0; i < prog->instances.nr; i++) | |
253 | zclose(prog->instances.fds[i]); | |
254 | } else if (prog->instances.nr != -1) { | |
255 | pr_warning("Internal error: instances.nr is %d\n", | |
256 | prog->instances.nr); | |
257 | } | |
258 | ||
259 | prog->instances.nr = -1; | |
260 | zfree(&prog->instances.fds); | |
55cffde2 WN |
261 | } |
262 | ||
a5b8bd47 WN |
263 | static void bpf_program__exit(struct bpf_program *prog) |
264 | { | |
265 | if (!prog) | |
266 | return; | |
267 | ||
aa9b1ac3 WN |
268 | if (prog->clear_priv) |
269 | prog->clear_priv(prog, prog->priv); | |
270 | ||
271 | prog->priv = NULL; | |
272 | prog->clear_priv = NULL; | |
273 | ||
55cffde2 | 274 | bpf_program__unload(prog); |
a5b8bd47 WN |
275 | zfree(&prog->section_name); |
276 | zfree(&prog->insns); | |
34090915 WN |
277 | zfree(&prog->reloc_desc); |
278 | ||
279 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
280 | prog->insns_cnt = 0; |
281 | prog->idx = -1; | |
282 | } | |
283 | ||
284 | static int | |
285 | bpf_program__init(void *data, size_t size, char *name, int idx, | |
286 | struct bpf_program *prog) | |
287 | { | |
288 | if (size < sizeof(struct bpf_insn)) { | |
289 | pr_warning("corrupted section '%s'\n", name); | |
290 | return -EINVAL; | |
291 | } | |
292 | ||
293 | bzero(prog, sizeof(*prog)); | |
294 | ||
295 | prog->section_name = strdup(name); | |
296 | if (!prog->section_name) { | |
297 | pr_warning("failed to alloc name for prog %s\n", | |
298 | name); | |
299 | goto errout; | |
300 | } | |
301 | ||
302 | prog->insns = malloc(size); | |
303 | if (!prog->insns) { | |
304 | pr_warning("failed to alloc insns for %s\n", name); | |
305 | goto errout; | |
306 | } | |
307 | prog->insns_cnt = size / sizeof(struct bpf_insn); | |
308 | memcpy(prog->insns, data, | |
309 | prog->insns_cnt * sizeof(struct bpf_insn)); | |
310 | prog->idx = idx; | |
b580563e WN |
311 | prog->instances.fds = NULL; |
312 | prog->instances.nr = -1; | |
5f44e4c8 | 313 | prog->type = BPF_PROG_TYPE_KPROBE; |
a5b8bd47 WN |
314 | |
315 | return 0; | |
316 | errout: | |
317 | bpf_program__exit(prog); | |
318 | return -ENOMEM; | |
319 | } | |
320 | ||
321 | static int | |
322 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
323 | char *name, int idx) | |
324 | { | |
325 | struct bpf_program prog, *progs; | |
326 | int nr_progs, err; | |
327 | ||
328 | err = bpf_program__init(data, size, name, idx, &prog); | |
329 | if (err) | |
330 | return err; | |
331 | ||
332 | progs = obj->programs; | |
333 | nr_progs = obj->nr_programs; | |
334 | ||
335 | progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); | |
336 | if (!progs) { | |
337 | /* | |
338 | * In this case the original obj->programs | |
339 | * is still valid, so don't need special treat for | |
340 | * bpf_close_object(). | |
341 | */ | |
342 | pr_warning("failed to alloc a new program '%s'\n", | |
343 | name); | |
344 | bpf_program__exit(&prog); | |
345 | return -ENOMEM; | |
346 | } | |
347 | ||
348 | pr_debug("found program %s\n", prog.section_name); | |
349 | obj->programs = progs; | |
350 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 351 | prog.obj = obj; |
a5b8bd47 WN |
352 | progs[nr_progs] = prog; |
353 | return 0; | |
354 | } | |
355 | ||
6c956392 WN |
356 | static struct bpf_object *bpf_object__new(const char *path, |
357 | void *obj_buf, | |
358 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
359 | { |
360 | struct bpf_object *obj; | |
361 | ||
362 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
363 | if (!obj) { | |
364 | pr_warning("alloc memory failed for %s\n", path); | |
6371ca3b | 365 | return ERR_PTR(-ENOMEM); |
1a5e3fb1 WN |
366 | } |
367 | ||
368 | strcpy(obj->path, path); | |
369 | obj->efile.fd = -1; | |
6c956392 WN |
370 | |
371 | /* | |
372 | * Caller of this function should also calls | |
373 | * bpf_object__elf_finish() after data collection to return | |
374 | * obj_buf to user. If not, we should duplicate the buffer to | |
375 | * avoid user freeing them before elf finish. | |
376 | */ | |
377 | obj->efile.obj_buf = obj_buf; | |
378 | obj->efile.obj_buf_sz = obj_buf_sz; | |
666810e8 | 379 | obj->efile.maps_shndx = -1; |
6c956392 | 380 | |
52d3352e | 381 | obj->loaded = false; |
9a208eff WN |
382 | |
383 | INIT_LIST_HEAD(&obj->list); | |
384 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
385 | return obj; |
386 | } | |
387 | ||
388 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
389 | { | |
390 | if (!obj_elf_valid(obj)) | |
391 | return; | |
392 | ||
393 | if (obj->efile.elf) { | |
394 | elf_end(obj->efile.elf); | |
395 | obj->efile.elf = NULL; | |
396 | } | |
bec7d68c | 397 | obj->efile.symbols = NULL; |
b62f06e8 WN |
398 | |
399 | zfree(&obj->efile.reloc); | |
400 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 401 | zclose(obj->efile.fd); |
6c956392 WN |
402 | obj->efile.obj_buf = NULL; |
403 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
404 | } |
405 | ||
406 | static int bpf_object__elf_init(struct bpf_object *obj) | |
407 | { | |
408 | int err = 0; | |
409 | GElf_Ehdr *ep; | |
410 | ||
411 | if (obj_elf_valid(obj)) { | |
412 | pr_warning("elf init: internal error\n"); | |
6371ca3b | 413 | return -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
414 | } |
415 | ||
6c956392 WN |
416 | if (obj->efile.obj_buf_sz > 0) { |
417 | /* | |
418 | * obj_buf should have been validated by | |
419 | * bpf_object__open_buffer(). | |
420 | */ | |
421 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
422 | obj->efile.obj_buf_sz); | |
423 | } else { | |
424 | obj->efile.fd = open(obj->path, O_RDONLY); | |
425 | if (obj->efile.fd < 0) { | |
426 | pr_warning("failed to open %s: %s\n", obj->path, | |
427 | strerror(errno)); | |
428 | return -errno; | |
429 | } | |
430 | ||
431 | obj->efile.elf = elf_begin(obj->efile.fd, | |
432 | LIBBPF_ELF_C_READ_MMAP, | |
433 | NULL); | |
1a5e3fb1 WN |
434 | } |
435 | ||
1a5e3fb1 WN |
436 | if (!obj->efile.elf) { |
437 | pr_warning("failed to open %s as ELF file\n", | |
438 | obj->path); | |
6371ca3b | 439 | err = -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
440 | goto errout; |
441 | } | |
442 | ||
443 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
444 | pr_warning("failed to get EHDR from %s\n", | |
445 | obj->path); | |
6371ca3b | 446 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
447 | goto errout; |
448 | } | |
449 | ep = &obj->efile.ehdr; | |
450 | ||
9b16137a WN |
451 | /* Old LLVM set e_machine to EM_NONE */ |
452 | if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { | |
1a5e3fb1 WN |
453 | pr_warning("%s is not an eBPF object file\n", |
454 | obj->path); | |
6371ca3b | 455 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
456 | goto errout; |
457 | } | |
458 | ||
459 | return 0; | |
460 | errout: | |
461 | bpf_object__elf_finish(obj); | |
462 | return err; | |
463 | } | |
464 | ||
cc4228d5 WN |
465 | static int |
466 | bpf_object__check_endianness(struct bpf_object *obj) | |
467 | { | |
468 | static unsigned int const endian = 1; | |
469 | ||
470 | switch (obj->efile.ehdr.e_ident[EI_DATA]) { | |
471 | case ELFDATA2LSB: | |
472 | /* We are big endian, BPF obj is little endian. */ | |
473 | if (*(unsigned char const *)&endian != 1) | |
474 | goto mismatch; | |
475 | break; | |
476 | ||
477 | case ELFDATA2MSB: | |
478 | /* We are little endian, BPF obj is big endian. */ | |
479 | if (*(unsigned char const *)&endian != 0) | |
480 | goto mismatch; | |
481 | break; | |
482 | default: | |
6371ca3b | 483 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
484 | } |
485 | ||
486 | return 0; | |
487 | ||
488 | mismatch: | |
489 | pr_warning("Error: endianness mismatch.\n"); | |
6371ca3b | 490 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
491 | } |
492 | ||
cb1e5e96 WN |
493 | static int |
494 | bpf_object__init_license(struct bpf_object *obj, | |
495 | void *data, size_t size) | |
496 | { | |
497 | memcpy(obj->license, data, | |
498 | min(size, sizeof(obj->license) - 1)); | |
499 | pr_debug("license of %s is %s\n", obj->path, obj->license); | |
500 | return 0; | |
501 | } | |
502 | ||
503 | static int | |
504 | bpf_object__init_kversion(struct bpf_object *obj, | |
505 | void *data, size_t size) | |
506 | { | |
507 | u32 kver; | |
508 | ||
509 | if (size != sizeof(kver)) { | |
510 | pr_warning("invalid kver section in %s\n", obj->path); | |
6371ca3b | 511 | return -LIBBPF_ERRNO__FORMAT; |
cb1e5e96 WN |
512 | } |
513 | memcpy(&kver, data, sizeof(kver)); | |
514 | obj->kern_version = kver; | |
515 | pr_debug("kernel version of %s is %x\n", obj->path, | |
516 | obj->kern_version); | |
517 | return 0; | |
518 | } | |
519 | ||
0b3d1efa | 520 | static int |
4708bbda | 521 | bpf_object__validate_maps(struct bpf_object *obj) |
0b3d1efa | 522 | { |
9d759a9b WN |
523 | int i; |
524 | ||
4708bbda EL |
525 | /* |
526 | * If there's only 1 map, the only error case should have been | |
527 | * catched in bpf_object__init_maps(). | |
528 | */ | |
529 | if (!obj->maps || !obj->nr_maps || (obj->nr_maps == 1)) | |
0b3d1efa | 530 | return 0; |
0b3d1efa | 531 | |
4708bbda EL |
532 | for (i = 1; i < obj->nr_maps; i++) { |
533 | const struct bpf_map *a = &obj->maps[i - 1]; | |
534 | const struct bpf_map *b = &obj->maps[i]; | |
9d759a9b | 535 | |
4708bbda EL |
536 | if (b->offset - a->offset < sizeof(struct bpf_map_def)) { |
537 | pr_warning("corrupted map section in %s: map \"%s\" too small\n", | |
538 | obj->path, a->name); | |
539 | return -EINVAL; | |
540 | } | |
0b3d1efa | 541 | } |
4708bbda EL |
542 | return 0; |
543 | } | |
0b3d1efa | 544 | |
4708bbda EL |
545 | static int compare_bpf_map(const void *_a, const void *_b) |
546 | { | |
547 | const struct bpf_map *a = _a; | |
548 | const struct bpf_map *b = _b; | |
9d759a9b | 549 | |
4708bbda | 550 | return a->offset - b->offset; |
0b3d1efa WN |
551 | } |
552 | ||
973170e6 | 553 | static int |
4708bbda | 554 | bpf_object__init_maps(struct bpf_object *obj) |
561bbcca | 555 | { |
4708bbda EL |
556 | int i, map_idx, nr_maps = 0; |
557 | Elf_Scn *scn; | |
558 | Elf_Data *data; | |
561bbcca WN |
559 | Elf_Data *symbols = obj->efile.symbols; |
560 | ||
4708bbda EL |
561 | if (obj->efile.maps_shndx < 0) |
562 | return -EINVAL; | |
563 | if (!symbols) | |
564 | return -EINVAL; | |
565 | ||
566 | scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); | |
567 | if (scn) | |
568 | data = elf_getdata(scn, NULL); | |
569 | if (!scn || !data) { | |
570 | pr_warning("failed to get Elf_Data from map section %d\n", | |
571 | obj->efile.maps_shndx); | |
973170e6 | 572 | return -EINVAL; |
4708bbda | 573 | } |
561bbcca | 574 | |
4708bbda EL |
575 | /* |
576 | * Count number of maps. Each map has a name. | |
577 | * Array of maps is not supported: only the first element is | |
578 | * considered. | |
579 | * | |
580 | * TODO: Detect array of map and report error. | |
581 | */ | |
561bbcca WN |
582 | for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { |
583 | GElf_Sym sym; | |
4708bbda EL |
584 | |
585 | if (!gelf_getsym(symbols, i, &sym)) | |
586 | continue; | |
587 | if (sym.st_shndx != obj->efile.maps_shndx) | |
588 | continue; | |
589 | nr_maps++; | |
590 | } | |
591 | ||
592 | /* Alloc obj->maps and fill nr_maps. */ | |
593 | pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, | |
594 | nr_maps, data->d_size); | |
595 | ||
596 | if (!nr_maps) | |
597 | return 0; | |
598 | ||
599 | obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); | |
600 | if (!obj->maps) { | |
601 | pr_warning("alloc maps for object failed\n"); | |
602 | return -ENOMEM; | |
603 | } | |
604 | obj->nr_maps = nr_maps; | |
605 | ||
606 | /* | |
607 | * fill all fd with -1 so won't close incorrect | |
608 | * fd (fd=0 is stdin) when failure (zclose won't close | |
609 | * negative fd)). | |
610 | */ | |
611 | for (i = 0; i < nr_maps; i++) | |
612 | obj->maps[i].fd = -1; | |
613 | ||
614 | /* | |
615 | * Fill obj->maps using data in "maps" section. | |
616 | */ | |
617 | for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { | |
618 | GElf_Sym sym; | |
561bbcca | 619 | const char *map_name; |
4708bbda | 620 | struct bpf_map_def *def; |
561bbcca WN |
621 | |
622 | if (!gelf_getsym(symbols, i, &sym)) | |
623 | continue; | |
666810e8 | 624 | if (sym.st_shndx != obj->efile.maps_shndx) |
561bbcca WN |
625 | continue; |
626 | ||
627 | map_name = elf_strptr(obj->efile.elf, | |
77ba9a5b | 628 | obj->efile.strtabidx, |
561bbcca | 629 | sym.st_name); |
4708bbda EL |
630 | obj->maps[map_idx].offset = sym.st_value; |
631 | if (sym.st_value + sizeof(struct bpf_map_def) > data->d_size) { | |
632 | pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", | |
633 | obj->path, map_name); | |
634 | return -EINVAL; | |
561bbcca | 635 | } |
4708bbda | 636 | |
561bbcca | 637 | obj->maps[map_idx].name = strdup(map_name); |
973170e6 WN |
638 | if (!obj->maps[map_idx].name) { |
639 | pr_warning("failed to alloc map name\n"); | |
640 | return -ENOMEM; | |
641 | } | |
4708bbda | 642 | pr_debug("map %d is \"%s\"\n", map_idx, |
561bbcca | 643 | obj->maps[map_idx].name); |
4708bbda EL |
644 | def = (struct bpf_map_def *)(data->d_buf + sym.st_value); |
645 | obj->maps[map_idx].def = *def; | |
646 | map_idx++; | |
561bbcca | 647 | } |
4708bbda EL |
648 | |
649 | qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); | |
650 | return bpf_object__validate_maps(obj); | |
561bbcca WN |
651 | } |
652 | ||
29603665 WN |
653 | static int bpf_object__elf_collect(struct bpf_object *obj) |
654 | { | |
655 | Elf *elf = obj->efile.elf; | |
656 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
657 | Elf_Scn *scn = NULL; | |
666810e8 | 658 | int idx = 0, err = 0; |
29603665 WN |
659 | |
660 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
661 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
662 | pr_warning("failed to get e_shstrndx from %s\n", | |
663 | obj->path); | |
6371ca3b | 664 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
665 | } |
666 | ||
667 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
668 | char *name; | |
669 | GElf_Shdr sh; | |
670 | Elf_Data *data; | |
671 | ||
672 | idx++; | |
673 | if (gelf_getshdr(scn, &sh) != &sh) { | |
674 | pr_warning("failed to get section header from %s\n", | |
675 | obj->path); | |
6371ca3b | 676 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
677 | goto out; |
678 | } | |
679 | ||
680 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
681 | if (!name) { | |
682 | pr_warning("failed to get section name from %s\n", | |
683 | obj->path); | |
6371ca3b | 684 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
685 | goto out; |
686 | } | |
687 | ||
688 | data = elf_getdata(scn, 0); | |
689 | if (!data) { | |
690 | pr_warning("failed to get section data from %s(%s)\n", | |
691 | name, obj->path); | |
6371ca3b | 692 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
693 | goto out; |
694 | } | |
695 | pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", | |
696 | name, (unsigned long)data->d_size, | |
697 | (int)sh.sh_link, (unsigned long)sh.sh_flags, | |
698 | (int)sh.sh_type); | |
cb1e5e96 WN |
699 | |
700 | if (strcmp(name, "license") == 0) | |
701 | err = bpf_object__init_license(obj, | |
702 | data->d_buf, | |
703 | data->d_size); | |
704 | else if (strcmp(name, "version") == 0) | |
705 | err = bpf_object__init_kversion(obj, | |
706 | data->d_buf, | |
707 | data->d_size); | |
4708bbda | 708 | else if (strcmp(name, "maps") == 0) |
666810e8 | 709 | obj->efile.maps_shndx = idx; |
4708bbda | 710 | else if (sh.sh_type == SHT_SYMTAB) { |
bec7d68c WN |
711 | if (obj->efile.symbols) { |
712 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
713 | obj->path); | |
6371ca3b | 714 | err = -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 715 | } else { |
bec7d68c | 716 | obj->efile.symbols = data; |
77ba9a5b WN |
717 | obj->efile.strtabidx = sh.sh_link; |
718 | } | |
a5b8bd47 WN |
719 | } else if ((sh.sh_type == SHT_PROGBITS) && |
720 | (sh.sh_flags & SHF_EXECINSTR) && | |
721 | (data->d_size > 0)) { | |
722 | err = bpf_object__add_program(obj, data->d_buf, | |
723 | data->d_size, name, idx); | |
724 | if (err) { | |
6371ca3b WN |
725 | char errmsg[STRERR_BUFSIZE]; |
726 | ||
a5b8bd47 WN |
727 | strerror_r(-err, errmsg, sizeof(errmsg)); |
728 | pr_warning("failed to alloc program %s (%s): %s", | |
729 | name, obj->path, errmsg); | |
730 | } | |
b62f06e8 WN |
731 | } else if (sh.sh_type == SHT_REL) { |
732 | void *reloc = obj->efile.reloc; | |
733 | int nr_reloc = obj->efile.nr_reloc + 1; | |
734 | ||
735 | reloc = realloc(reloc, | |
736 | sizeof(*obj->efile.reloc) * nr_reloc); | |
737 | if (!reloc) { | |
738 | pr_warning("realloc failed\n"); | |
739 | err = -ENOMEM; | |
740 | } else { | |
741 | int n = nr_reloc - 1; | |
742 | ||
743 | obj->efile.reloc = reloc; | |
744 | obj->efile.nr_reloc = nr_reloc; | |
745 | ||
746 | obj->efile.reloc[n].shdr = sh; | |
747 | obj->efile.reloc[n].data = data; | |
748 | } | |
bec7d68c | 749 | } |
cb1e5e96 WN |
750 | if (err) |
751 | goto out; | |
29603665 | 752 | } |
561bbcca | 753 | |
77ba9a5b WN |
754 | if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { |
755 | pr_warning("Corrupted ELF file: index of strtab invalid\n"); | |
756 | return LIBBPF_ERRNO__FORMAT; | |
757 | } | |
666810e8 | 758 | if (obj->efile.maps_shndx >= 0) |
4708bbda | 759 | err = bpf_object__init_maps(obj); |
29603665 WN |
760 | out: |
761 | return err; | |
762 | } | |
763 | ||
34090915 WN |
764 | static struct bpf_program * |
765 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
766 | { | |
767 | struct bpf_program *prog; | |
768 | size_t i; | |
769 | ||
770 | for (i = 0; i < obj->nr_programs; i++) { | |
771 | prog = &obj->programs[i]; | |
772 | if (prog->idx == idx) | |
773 | return prog; | |
774 | } | |
775 | return NULL; | |
776 | } | |
777 | ||
778 | static int | |
779 | bpf_program__collect_reloc(struct bpf_program *prog, | |
780 | size_t nr_maps, GElf_Shdr *shdr, | |
666810e8 WN |
781 | Elf_Data *data, Elf_Data *symbols, |
782 | int maps_shndx) | |
34090915 WN |
783 | { |
784 | int i, nrels; | |
785 | ||
786 | pr_debug("collecting relocating info for: '%s'\n", | |
787 | prog->section_name); | |
788 | nrels = shdr->sh_size / shdr->sh_entsize; | |
789 | ||
790 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
791 | if (!prog->reloc_desc) { | |
792 | pr_warning("failed to alloc memory in relocation\n"); | |
793 | return -ENOMEM; | |
794 | } | |
795 | prog->nr_reloc = nrels; | |
796 | ||
797 | for (i = 0; i < nrels; i++) { | |
798 | GElf_Sym sym; | |
799 | GElf_Rel rel; | |
800 | unsigned int insn_idx; | |
801 | struct bpf_insn *insns = prog->insns; | |
802 | size_t map_idx; | |
803 | ||
804 | if (!gelf_getrel(data, i, &rel)) { | |
805 | pr_warning("relocation: failed to get %d reloc\n", i); | |
6371ca3b | 806 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
807 | } |
808 | ||
34090915 WN |
809 | if (!gelf_getsym(symbols, |
810 | GELF_R_SYM(rel.r_info), | |
811 | &sym)) { | |
812 | pr_warning("relocation: symbol %"PRIx64" not found\n", | |
813 | GELF_R_SYM(rel.r_info)); | |
6371ca3b | 814 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
815 | } |
816 | ||
666810e8 WN |
817 | if (sym.st_shndx != maps_shndx) { |
818 | pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", | |
819 | prog->section_name, sym.st_shndx); | |
820 | return -LIBBPF_ERRNO__RELOC; | |
821 | } | |
822 | ||
823 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); | |
824 | pr_debug("relocation: insn_idx=%u\n", insn_idx); | |
825 | ||
34090915 WN |
826 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { |
827 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
828 | insn_idx, insns[insn_idx].code); | |
6371ca3b | 829 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
830 | } |
831 | ||
832 | map_idx = sym.st_value / sizeof(struct bpf_map_def); | |
833 | if (map_idx >= nr_maps) { | |
834 | pr_warning("bpf relocation: map_idx %d large than %d\n", | |
835 | (int)map_idx, (int)nr_maps - 1); | |
6371ca3b | 836 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
837 | } |
838 | ||
839 | prog->reloc_desc[i].insn_idx = insn_idx; | |
840 | prog->reloc_desc[i].map_idx = map_idx; | |
841 | } | |
842 | return 0; | |
843 | } | |
844 | ||
52d3352e WN |
845 | static int |
846 | bpf_object__create_maps(struct bpf_object *obj) | |
847 | { | |
848 | unsigned int i; | |
52d3352e | 849 | |
9d759a9b WN |
850 | for (i = 0; i < obj->nr_maps; i++) { |
851 | struct bpf_map_def *def = &obj->maps[i].def; | |
852 | int *pfd = &obj->maps[i].fd; | |
52d3352e | 853 | |
9d759a9b WN |
854 | *pfd = bpf_create_map(def->type, |
855 | def->key_size, | |
856 | def->value_size, | |
857 | def->max_entries); | |
52d3352e WN |
858 | if (*pfd < 0) { |
859 | size_t j; | |
860 | int err = *pfd; | |
861 | ||
862 | pr_warning("failed to create map: %s\n", | |
863 | strerror(errno)); | |
864 | for (j = 0; j < i; j++) | |
9d759a9b | 865 | zclose(obj->maps[j].fd); |
52d3352e WN |
866 | return err; |
867 | } | |
4708bbda | 868 | pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd); |
52d3352e WN |
869 | } |
870 | ||
52d3352e WN |
871 | return 0; |
872 | } | |
873 | ||
8a47a6c5 | 874 | static int |
9d759a9b | 875 | bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) |
8a47a6c5 WN |
876 | { |
877 | int i; | |
878 | ||
879 | if (!prog || !prog->reloc_desc) | |
880 | return 0; | |
881 | ||
882 | for (i = 0; i < prog->nr_reloc; i++) { | |
883 | int insn_idx, map_idx; | |
884 | struct bpf_insn *insns = prog->insns; | |
885 | ||
886 | insn_idx = prog->reloc_desc[i].insn_idx; | |
887 | map_idx = prog->reloc_desc[i].map_idx; | |
888 | ||
889 | if (insn_idx >= (int)prog->insns_cnt) { | |
890 | pr_warning("relocation out of range: '%s'\n", | |
891 | prog->section_name); | |
6371ca3b | 892 | return -LIBBPF_ERRNO__RELOC; |
8a47a6c5 WN |
893 | } |
894 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
9d759a9b | 895 | insns[insn_idx].imm = obj->maps[map_idx].fd; |
8a47a6c5 WN |
896 | } |
897 | ||
898 | zfree(&prog->reloc_desc); | |
899 | prog->nr_reloc = 0; | |
900 | return 0; | |
901 | } | |
902 | ||
903 | ||
904 | static int | |
905 | bpf_object__relocate(struct bpf_object *obj) | |
906 | { | |
907 | struct bpf_program *prog; | |
908 | size_t i; | |
909 | int err; | |
910 | ||
911 | for (i = 0; i < obj->nr_programs; i++) { | |
912 | prog = &obj->programs[i]; | |
913 | ||
9d759a9b | 914 | err = bpf_program__relocate(prog, obj); |
8a47a6c5 WN |
915 | if (err) { |
916 | pr_warning("failed to relocate '%s'\n", | |
917 | prog->section_name); | |
918 | return err; | |
919 | } | |
920 | } | |
921 | return 0; | |
922 | } | |
923 | ||
34090915 WN |
924 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
925 | { | |
926 | int i, err; | |
927 | ||
928 | if (!obj_elf_valid(obj)) { | |
929 | pr_warning("Internal error: elf object is closed\n"); | |
6371ca3b | 930 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
931 | } |
932 | ||
933 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
934 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
935 | Elf_Data *data = obj->efile.reloc[i].data; | |
936 | int idx = shdr->sh_info; | |
937 | struct bpf_program *prog; | |
9d759a9b | 938 | size_t nr_maps = obj->nr_maps; |
34090915 WN |
939 | |
940 | if (shdr->sh_type != SHT_REL) { | |
941 | pr_warning("internal error at %d\n", __LINE__); | |
6371ca3b | 942 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
943 | } |
944 | ||
945 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
946 | if (!prog) { | |
947 | pr_warning("relocation failed: no %d section\n", | |
948 | idx); | |
6371ca3b | 949 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
950 | } |
951 | ||
952 | err = bpf_program__collect_reloc(prog, nr_maps, | |
953 | shdr, data, | |
666810e8 WN |
954 | obj->efile.symbols, |
955 | obj->efile.maps_shndx); | |
34090915 | 956 | if (err) |
6371ca3b | 957 | return err; |
34090915 WN |
958 | } |
959 | return 0; | |
960 | } | |
961 | ||
55cffde2 | 962 | static int |
5f44e4c8 WN |
963 | load_program(enum bpf_prog_type type, struct bpf_insn *insns, |
964 | int insns_cnt, char *license, u32 kern_version, int *pfd) | |
55cffde2 WN |
965 | { |
966 | int ret; | |
967 | char *log_buf; | |
968 | ||
969 | if (!insns || !insns_cnt) | |
970 | return -EINVAL; | |
971 | ||
972 | log_buf = malloc(BPF_LOG_BUF_SIZE); | |
973 | if (!log_buf) | |
974 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
975 | ||
5f44e4c8 WN |
976 | ret = bpf_load_program(type, insns, insns_cnt, license, |
977 | kern_version, log_buf, BPF_LOG_BUF_SIZE); | |
55cffde2 WN |
978 | |
979 | if (ret >= 0) { | |
980 | *pfd = ret; | |
981 | ret = 0; | |
982 | goto out; | |
983 | } | |
984 | ||
6371ca3b | 985 | ret = -LIBBPF_ERRNO__LOAD; |
55cffde2 WN |
986 | pr_warning("load bpf program failed: %s\n", strerror(errno)); |
987 | ||
6371ca3b WN |
988 | if (log_buf && log_buf[0] != '\0') { |
989 | ret = -LIBBPF_ERRNO__VERIFY; | |
55cffde2 WN |
990 | pr_warning("-- BEGIN DUMP LOG ---\n"); |
991 | pr_warning("\n%s\n", log_buf); | |
992 | pr_warning("-- END LOG --\n"); | |
705fa219 WN |
993 | } else if (insns_cnt >= BPF_MAXINSNS) { |
994 | pr_warning("Program too large (%d insns), at most %d insns\n", | |
995 | insns_cnt, BPF_MAXINSNS); | |
996 | ret = -LIBBPF_ERRNO__PROG2BIG; | |
6371ca3b | 997 | } else { |
705fa219 WN |
998 | /* Wrong program type? */ |
999 | if (type != BPF_PROG_TYPE_KPROBE) { | |
1000 | int fd; | |
1001 | ||
1002 | fd = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns, | |
1003 | insns_cnt, license, kern_version, | |
1004 | NULL, 0); | |
1005 | if (fd >= 0) { | |
1006 | close(fd); | |
1007 | ret = -LIBBPF_ERRNO__PROGTYPE; | |
1008 | goto out; | |
1009 | } | |
6371ca3b | 1010 | } |
705fa219 WN |
1011 | |
1012 | if (log_buf) | |
1013 | ret = -LIBBPF_ERRNO__KVER; | |
55cffde2 WN |
1014 | } |
1015 | ||
1016 | out: | |
1017 | free(log_buf); | |
1018 | return ret; | |
1019 | } | |
1020 | ||
1021 | static int | |
1022 | bpf_program__load(struct bpf_program *prog, | |
1023 | char *license, u32 kern_version) | |
1024 | { | |
b580563e | 1025 | int err = 0, fd, i; |
55cffde2 | 1026 | |
b580563e WN |
1027 | if (prog->instances.nr < 0 || !prog->instances.fds) { |
1028 | if (prog->preprocessor) { | |
1029 | pr_warning("Internal error: can't load program '%s'\n", | |
1030 | prog->section_name); | |
1031 | return -LIBBPF_ERRNO__INTERNAL; | |
1032 | } | |
55cffde2 | 1033 | |
b580563e WN |
1034 | prog->instances.fds = malloc(sizeof(int)); |
1035 | if (!prog->instances.fds) { | |
1036 | pr_warning("Not enough memory for BPF fds\n"); | |
1037 | return -ENOMEM; | |
1038 | } | |
1039 | prog->instances.nr = 1; | |
1040 | prog->instances.fds[0] = -1; | |
1041 | } | |
1042 | ||
1043 | if (!prog->preprocessor) { | |
1044 | if (prog->instances.nr != 1) { | |
1045 | pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", | |
1046 | prog->section_name, prog->instances.nr); | |
1047 | } | |
5f44e4c8 | 1048 | err = load_program(prog->type, prog->insns, prog->insns_cnt, |
b580563e WN |
1049 | license, kern_version, &fd); |
1050 | if (!err) | |
1051 | prog->instances.fds[0] = fd; | |
1052 | goto out; | |
1053 | } | |
1054 | ||
1055 | for (i = 0; i < prog->instances.nr; i++) { | |
1056 | struct bpf_prog_prep_result result; | |
1057 | bpf_program_prep_t preprocessor = prog->preprocessor; | |
1058 | ||
1059 | bzero(&result, sizeof(result)); | |
1060 | err = preprocessor(prog, i, prog->insns, | |
1061 | prog->insns_cnt, &result); | |
1062 | if (err) { | |
1063 | pr_warning("Preprocessing the %dth instance of program '%s' failed\n", | |
1064 | i, prog->section_name); | |
1065 | goto out; | |
1066 | } | |
1067 | ||
1068 | if (!result.new_insn_ptr || !result.new_insn_cnt) { | |
1069 | pr_debug("Skip loading the %dth instance of program '%s'\n", | |
1070 | i, prog->section_name); | |
1071 | prog->instances.fds[i] = -1; | |
1072 | if (result.pfd) | |
1073 | *result.pfd = -1; | |
1074 | continue; | |
1075 | } | |
1076 | ||
5f44e4c8 | 1077 | err = load_program(prog->type, result.new_insn_ptr, |
b580563e WN |
1078 | result.new_insn_cnt, |
1079 | license, kern_version, &fd); | |
1080 | ||
1081 | if (err) { | |
1082 | pr_warning("Loading the %dth instance of program '%s' failed\n", | |
1083 | i, prog->section_name); | |
1084 | goto out; | |
1085 | } | |
1086 | ||
1087 | if (result.pfd) | |
1088 | *result.pfd = fd; | |
1089 | prog->instances.fds[i] = fd; | |
1090 | } | |
1091 | out: | |
55cffde2 WN |
1092 | if (err) |
1093 | pr_warning("failed to load program '%s'\n", | |
1094 | prog->section_name); | |
1095 | zfree(&prog->insns); | |
1096 | prog->insns_cnt = 0; | |
1097 | return err; | |
1098 | } | |
1099 | ||
1100 | static int | |
1101 | bpf_object__load_progs(struct bpf_object *obj) | |
1102 | { | |
1103 | size_t i; | |
1104 | int err; | |
1105 | ||
1106 | for (i = 0; i < obj->nr_programs; i++) { | |
1107 | err = bpf_program__load(&obj->programs[i], | |
1108 | obj->license, | |
1109 | obj->kern_version); | |
1110 | if (err) | |
1111 | return err; | |
1112 | } | |
1113 | return 0; | |
1114 | } | |
1115 | ||
cb1e5e96 WN |
1116 | static int bpf_object__validate(struct bpf_object *obj) |
1117 | { | |
1118 | if (obj->kern_version == 0) { | |
1119 | pr_warning("%s doesn't provide kernel version\n", | |
1120 | obj->path); | |
6371ca3b | 1121 | return -LIBBPF_ERRNO__KVERSION; |
cb1e5e96 WN |
1122 | } |
1123 | return 0; | |
1124 | } | |
1125 | ||
1a5e3fb1 | 1126 | static struct bpf_object * |
6c956392 | 1127 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) |
1a5e3fb1 WN |
1128 | { |
1129 | struct bpf_object *obj; | |
6371ca3b | 1130 | int err; |
1a5e3fb1 WN |
1131 | |
1132 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
1133 | pr_warning("failed to init libelf for %s\n", path); | |
6371ca3b | 1134 | return ERR_PTR(-LIBBPF_ERRNO__LIBELF); |
1a5e3fb1 WN |
1135 | } |
1136 | ||
6c956392 | 1137 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
6371ca3b WN |
1138 | if (IS_ERR(obj)) |
1139 | return obj; | |
1a5e3fb1 | 1140 | |
6371ca3b WN |
1141 | CHECK_ERR(bpf_object__elf_init(obj), err, out); |
1142 | CHECK_ERR(bpf_object__check_endianness(obj), err, out); | |
1143 | CHECK_ERR(bpf_object__elf_collect(obj), err, out); | |
1144 | CHECK_ERR(bpf_object__collect_reloc(obj), err, out); | |
1145 | CHECK_ERR(bpf_object__validate(obj), err, out); | |
1a5e3fb1 WN |
1146 | |
1147 | bpf_object__elf_finish(obj); | |
1148 | return obj; | |
1149 | out: | |
1150 | bpf_object__close(obj); | |
6371ca3b | 1151 | return ERR_PTR(err); |
1a5e3fb1 WN |
1152 | } |
1153 | ||
1154 | struct bpf_object *bpf_object__open(const char *path) | |
1155 | { | |
1156 | /* param validation */ | |
1157 | if (!path) | |
1158 | return NULL; | |
1159 | ||
1160 | pr_debug("loading %s\n", path); | |
1161 | ||
6c956392 WN |
1162 | return __bpf_object__open(path, NULL, 0); |
1163 | } | |
1164 | ||
1165 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
acf860ae WN |
1166 | size_t obj_buf_sz, |
1167 | const char *name) | |
6c956392 | 1168 | { |
acf860ae WN |
1169 | char tmp_name[64]; |
1170 | ||
6c956392 WN |
1171 | /* param validation */ |
1172 | if (!obj_buf || obj_buf_sz <= 0) | |
1173 | return NULL; | |
1174 | ||
acf860ae WN |
1175 | if (!name) { |
1176 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
1177 | (unsigned long)obj_buf, | |
1178 | (unsigned long)obj_buf_sz); | |
1179 | tmp_name[sizeof(tmp_name) - 1] = '\0'; | |
1180 | name = tmp_name; | |
1181 | } | |
1182 | pr_debug("loading object '%s' from buffer\n", | |
1183 | name); | |
6c956392 | 1184 | |
acf860ae | 1185 | return __bpf_object__open(name, obj_buf, obj_buf_sz); |
1a5e3fb1 WN |
1186 | } |
1187 | ||
52d3352e WN |
1188 | int bpf_object__unload(struct bpf_object *obj) |
1189 | { | |
1190 | size_t i; | |
1191 | ||
1192 | if (!obj) | |
1193 | return -EINVAL; | |
1194 | ||
9d759a9b WN |
1195 | for (i = 0; i < obj->nr_maps; i++) |
1196 | zclose(obj->maps[i].fd); | |
52d3352e | 1197 | |
55cffde2 WN |
1198 | for (i = 0; i < obj->nr_programs; i++) |
1199 | bpf_program__unload(&obj->programs[i]); | |
1200 | ||
52d3352e WN |
1201 | return 0; |
1202 | } | |
1203 | ||
1204 | int bpf_object__load(struct bpf_object *obj) | |
1205 | { | |
6371ca3b WN |
1206 | int err; |
1207 | ||
52d3352e WN |
1208 | if (!obj) |
1209 | return -EINVAL; | |
1210 | ||
1211 | if (obj->loaded) { | |
1212 | pr_warning("object should not be loaded twice\n"); | |
1213 | return -EINVAL; | |
1214 | } | |
1215 | ||
1216 | obj->loaded = true; | |
6371ca3b WN |
1217 | |
1218 | CHECK_ERR(bpf_object__create_maps(obj), err, out); | |
1219 | CHECK_ERR(bpf_object__relocate(obj), err, out); | |
1220 | CHECK_ERR(bpf_object__load_progs(obj), err, out); | |
52d3352e WN |
1221 | |
1222 | return 0; | |
1223 | out: | |
1224 | bpf_object__unload(obj); | |
1225 | pr_warning("failed to load object '%s'\n", obj->path); | |
6371ca3b | 1226 | return err; |
52d3352e WN |
1227 | } |
1228 | ||
1a5e3fb1 WN |
1229 | void bpf_object__close(struct bpf_object *obj) |
1230 | { | |
a5b8bd47 WN |
1231 | size_t i; |
1232 | ||
1a5e3fb1 WN |
1233 | if (!obj) |
1234 | return; | |
1235 | ||
10931d24 WN |
1236 | if (obj->clear_priv) |
1237 | obj->clear_priv(obj, obj->priv); | |
1238 | ||
1a5e3fb1 | 1239 | bpf_object__elf_finish(obj); |
52d3352e | 1240 | bpf_object__unload(obj); |
1a5e3fb1 | 1241 | |
9d759a9b | 1242 | for (i = 0; i < obj->nr_maps; i++) { |
561bbcca | 1243 | zfree(&obj->maps[i].name); |
9d759a9b WN |
1244 | if (obj->maps[i].clear_priv) |
1245 | obj->maps[i].clear_priv(&obj->maps[i], | |
1246 | obj->maps[i].priv); | |
1247 | obj->maps[i].priv = NULL; | |
1248 | obj->maps[i].clear_priv = NULL; | |
1249 | } | |
1250 | zfree(&obj->maps); | |
1251 | obj->nr_maps = 0; | |
a5b8bd47 WN |
1252 | |
1253 | if (obj->programs && obj->nr_programs) { | |
1254 | for (i = 0; i < obj->nr_programs; i++) | |
1255 | bpf_program__exit(&obj->programs[i]); | |
1256 | } | |
1257 | zfree(&obj->programs); | |
1258 | ||
9a208eff | 1259 | list_del(&obj->list); |
1a5e3fb1 WN |
1260 | free(obj); |
1261 | } | |
aa9b1ac3 | 1262 | |
9a208eff WN |
1263 | struct bpf_object * |
1264 | bpf_object__next(struct bpf_object *prev) | |
1265 | { | |
1266 | struct bpf_object *next; | |
1267 | ||
1268 | if (!prev) | |
1269 | next = list_first_entry(&bpf_objects_list, | |
1270 | struct bpf_object, | |
1271 | list); | |
1272 | else | |
1273 | next = list_next_entry(prev, list); | |
1274 | ||
1275 | /* Empty list is noticed here so don't need checking on entry. */ | |
1276 | if (&next->list == &bpf_objects_list) | |
1277 | return NULL; | |
1278 | ||
1279 | return next; | |
1280 | } | |
1281 | ||
a7fe0450 | 1282 | const char *bpf_object__name(struct bpf_object *obj) |
acf860ae | 1283 | { |
a7fe0450 | 1284 | return obj ? obj->path : ERR_PTR(-EINVAL); |
acf860ae WN |
1285 | } |
1286 | ||
a7fe0450 | 1287 | unsigned int bpf_object__kversion(struct bpf_object *obj) |
45825d8a | 1288 | { |
a7fe0450 | 1289 | return obj ? obj->kern_version : 0; |
45825d8a WN |
1290 | } |
1291 | ||
10931d24 WN |
1292 | int bpf_object__set_priv(struct bpf_object *obj, void *priv, |
1293 | bpf_object_clear_priv_t clear_priv) | |
1294 | { | |
1295 | if (obj->priv && obj->clear_priv) | |
1296 | obj->clear_priv(obj, obj->priv); | |
1297 | ||
1298 | obj->priv = priv; | |
1299 | obj->clear_priv = clear_priv; | |
1300 | return 0; | |
1301 | } | |
1302 | ||
1303 | void *bpf_object__priv(struct bpf_object *obj) | |
1304 | { | |
1305 | return obj ? obj->priv : ERR_PTR(-EINVAL); | |
1306 | } | |
1307 | ||
aa9b1ac3 WN |
1308 | struct bpf_program * |
1309 | bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) | |
1310 | { | |
1311 | size_t idx; | |
1312 | ||
1313 | if (!obj->programs) | |
1314 | return NULL; | |
1315 | /* First handler */ | |
1316 | if (prev == NULL) | |
1317 | return &obj->programs[0]; | |
1318 | ||
1319 | if (prev->obj != obj) { | |
1320 | pr_warning("error: program handler doesn't match object\n"); | |
1321 | return NULL; | |
1322 | } | |
1323 | ||
1324 | idx = (prev - obj->programs) + 1; | |
1325 | if (idx >= obj->nr_programs) | |
1326 | return NULL; | |
1327 | return &obj->programs[idx]; | |
1328 | } | |
1329 | ||
edb13ed4 ACM |
1330 | int bpf_program__set_priv(struct bpf_program *prog, void *priv, |
1331 | bpf_program_clear_priv_t clear_priv) | |
aa9b1ac3 WN |
1332 | { |
1333 | if (prog->priv && prog->clear_priv) | |
1334 | prog->clear_priv(prog, prog->priv); | |
1335 | ||
1336 | prog->priv = priv; | |
1337 | prog->clear_priv = clear_priv; | |
1338 | return 0; | |
1339 | } | |
1340 | ||
be834ffb | 1341 | void *bpf_program__priv(struct bpf_program *prog) |
aa9b1ac3 | 1342 | { |
be834ffb | 1343 | return prog ? prog->priv : ERR_PTR(-EINVAL); |
aa9b1ac3 WN |
1344 | } |
1345 | ||
715f8db9 | 1346 | const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
1347 | { |
1348 | const char *title; | |
1349 | ||
1350 | title = prog->section_name; | |
715f8db9 | 1351 | if (needs_copy) { |
aa9b1ac3 WN |
1352 | title = strdup(title); |
1353 | if (!title) { | |
1354 | pr_warning("failed to strdup program title\n"); | |
6371ca3b | 1355 | return ERR_PTR(-ENOMEM); |
aa9b1ac3 WN |
1356 | } |
1357 | } | |
1358 | ||
1359 | return title; | |
1360 | } | |
1361 | ||
1362 | int bpf_program__fd(struct bpf_program *prog) | |
1363 | { | |
b580563e WN |
1364 | return bpf_program__nth_fd(prog, 0); |
1365 | } | |
1366 | ||
1367 | int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, | |
1368 | bpf_program_prep_t prep) | |
1369 | { | |
1370 | int *instances_fds; | |
1371 | ||
1372 | if (nr_instances <= 0 || !prep) | |
1373 | return -EINVAL; | |
1374 | ||
1375 | if (prog->instances.nr > 0 || prog->instances.fds) { | |
1376 | pr_warning("Can't set pre-processor after loading\n"); | |
1377 | return -EINVAL; | |
1378 | } | |
1379 | ||
1380 | instances_fds = malloc(sizeof(int) * nr_instances); | |
1381 | if (!instances_fds) { | |
1382 | pr_warning("alloc memory failed for fds\n"); | |
1383 | return -ENOMEM; | |
1384 | } | |
1385 | ||
1386 | /* fill all fd with -1 */ | |
1387 | memset(instances_fds, -1, sizeof(int) * nr_instances); | |
1388 | ||
1389 | prog->instances.nr = nr_instances; | |
1390 | prog->instances.fds = instances_fds; | |
1391 | prog->preprocessor = prep; | |
1392 | return 0; | |
1393 | } | |
1394 | ||
1395 | int bpf_program__nth_fd(struct bpf_program *prog, int n) | |
1396 | { | |
1397 | int fd; | |
1398 | ||
1399 | if (n >= prog->instances.nr || n < 0) { | |
1400 | pr_warning("Can't get the %dth fd from program %s: only %d instances\n", | |
1401 | n, prog->section_name, prog->instances.nr); | |
1402 | return -EINVAL; | |
1403 | } | |
1404 | ||
1405 | fd = prog->instances.fds[n]; | |
1406 | if (fd < 0) { | |
1407 | pr_warning("%dth instance of program '%s' is invalid\n", | |
1408 | n, prog->section_name); | |
1409 | return -ENOENT; | |
1410 | } | |
1411 | ||
1412 | return fd; | |
aa9b1ac3 | 1413 | } |
9d759a9b | 1414 | |
5f44e4c8 WN |
1415 | static void bpf_program__set_type(struct bpf_program *prog, |
1416 | enum bpf_prog_type type) | |
1417 | { | |
1418 | prog->type = type; | |
1419 | } | |
1420 | ||
1421 | int bpf_program__set_tracepoint(struct bpf_program *prog) | |
1422 | { | |
1423 | if (!prog) | |
1424 | return -EINVAL; | |
1425 | bpf_program__set_type(prog, BPF_PROG_TYPE_TRACEPOINT); | |
1426 | return 0; | |
1427 | } | |
1428 | ||
1429 | int bpf_program__set_kprobe(struct bpf_program *prog) | |
1430 | { | |
1431 | if (!prog) | |
1432 | return -EINVAL; | |
1433 | bpf_program__set_type(prog, BPF_PROG_TYPE_KPROBE); | |
1434 | return 0; | |
1435 | } | |
1436 | ||
1437 | static bool bpf_program__is_type(struct bpf_program *prog, | |
1438 | enum bpf_prog_type type) | |
1439 | { | |
1440 | return prog ? (prog->type == type) : false; | |
1441 | } | |
1442 | ||
1443 | bool bpf_program__is_tracepoint(struct bpf_program *prog) | |
1444 | { | |
1445 | return bpf_program__is_type(prog, BPF_PROG_TYPE_TRACEPOINT); | |
1446 | } | |
1447 | ||
1448 | bool bpf_program__is_kprobe(struct bpf_program *prog) | |
1449 | { | |
1450 | return bpf_program__is_type(prog, BPF_PROG_TYPE_KPROBE); | |
1451 | } | |
1452 | ||
6e009e65 | 1453 | int bpf_map__fd(struct bpf_map *map) |
9d759a9b | 1454 | { |
6e009e65 | 1455 | return map ? map->fd : -EINVAL; |
9d759a9b WN |
1456 | } |
1457 | ||
53897a78 | 1458 | const struct bpf_map_def *bpf_map__def(struct bpf_map *map) |
9d759a9b | 1459 | { |
53897a78 | 1460 | return map ? &map->def : ERR_PTR(-EINVAL); |
9d759a9b WN |
1461 | } |
1462 | ||
009ad5d5 | 1463 | const char *bpf_map__name(struct bpf_map *map) |
561bbcca | 1464 | { |
009ad5d5 | 1465 | return map ? map->name : NULL; |
561bbcca WN |
1466 | } |
1467 | ||
edb13ed4 ACM |
1468 | int bpf_map__set_priv(struct bpf_map *map, void *priv, |
1469 | bpf_map_clear_priv_t clear_priv) | |
9d759a9b WN |
1470 | { |
1471 | if (!map) | |
1472 | return -EINVAL; | |
1473 | ||
1474 | if (map->priv) { | |
1475 | if (map->clear_priv) | |
1476 | map->clear_priv(map, map->priv); | |
1477 | } | |
1478 | ||
1479 | map->priv = priv; | |
1480 | map->clear_priv = clear_priv; | |
1481 | return 0; | |
1482 | } | |
1483 | ||
b4cbfa56 | 1484 | void *bpf_map__priv(struct bpf_map *map) |
9d759a9b | 1485 | { |
b4cbfa56 | 1486 | return map ? map->priv : ERR_PTR(-EINVAL); |
9d759a9b WN |
1487 | } |
1488 | ||
1489 | struct bpf_map * | |
1490 | bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) | |
1491 | { | |
1492 | size_t idx; | |
1493 | struct bpf_map *s, *e; | |
1494 | ||
1495 | if (!obj || !obj->maps) | |
1496 | return NULL; | |
1497 | ||
1498 | s = obj->maps; | |
1499 | e = obj->maps + obj->nr_maps; | |
1500 | ||
1501 | if (prev == NULL) | |
1502 | return s; | |
1503 | ||
1504 | if ((prev < s) || (prev >= e)) { | |
1505 | pr_warning("error in %s: map handler doesn't belong to object\n", | |
1506 | __func__); | |
1507 | return NULL; | |
1508 | } | |
1509 | ||
1510 | idx = (prev - obj->maps) + 1; | |
1511 | if (idx >= obj->nr_maps) | |
1512 | return NULL; | |
1513 | return &obj->maps[idx]; | |
1514 | } | |
561bbcca WN |
1515 | |
1516 | struct bpf_map * | |
a7fe0450 | 1517 | bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) |
561bbcca WN |
1518 | { |
1519 | struct bpf_map *pos; | |
1520 | ||
1521 | bpf_map__for_each(pos, obj) { | |
973170e6 | 1522 | if (pos->name && !strcmp(pos->name, name)) |
561bbcca WN |
1523 | return pos; |
1524 | } | |
1525 | return NULL; | |
1526 | } |