Commit | Line | Data |
---|---|---|
1b76c13e WN |
1 | /* |
2 | * Common eBPF ELF object loading operations. | |
3 | * | |
4 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
5 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
6 | * Copyright (C) 2015 Huawei Inc. | |
7 | */ | |
8 | ||
9 | #include <stdlib.h> | |
b3f59d66 WN |
10 | #include <stdio.h> |
11 | #include <stdarg.h> | |
34090915 | 12 | #include <inttypes.h> |
b3f59d66 | 13 | #include <string.h> |
1b76c13e | 14 | #include <unistd.h> |
1a5e3fb1 WN |
15 | #include <fcntl.h> |
16 | #include <errno.h> | |
1b76c13e | 17 | #include <asm/unistd.h> |
cb1e5e96 | 18 | #include <linux/kernel.h> |
1b76c13e | 19 | #include <linux/bpf.h> |
1a5e3fb1 WN |
20 | #include <libelf.h> |
21 | #include <gelf.h> | |
1b76c13e WN |
22 | |
23 | #include "libbpf.h" | |
52d3352e | 24 | #include "bpf.h" |
b3f59d66 WN |
25 | |
26 | #define __printf(a, b) __attribute__((format(printf, a, b))) | |
27 | ||
28 | __printf(1, 2) | |
29 | static int __base_pr(const char *format, ...) | |
30 | { | |
31 | va_list args; | |
32 | int err; | |
33 | ||
34 | va_start(args, format); | |
35 | err = vfprintf(stderr, format, args); | |
36 | va_end(args); | |
37 | return err; | |
38 | } | |
39 | ||
40 | static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; | |
41 | static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; | |
42 | static __printf(1, 2) libbpf_print_fn_t __pr_debug; | |
43 | ||
44 | #define __pr(func, fmt, ...) \ | |
45 | do { \ | |
46 | if ((func)) \ | |
47 | (func)("libbpf: " fmt, ##__VA_ARGS__); \ | |
48 | } while (0) | |
49 | ||
50 | #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) | |
51 | #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) | |
52 | #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) | |
53 | ||
54 | void libbpf_set_print(libbpf_print_fn_t warn, | |
55 | libbpf_print_fn_t info, | |
56 | libbpf_print_fn_t debug) | |
57 | { | |
58 | __pr_warning = warn; | |
59 | __pr_info = info; | |
60 | __pr_debug = debug; | |
61 | } | |
1a5e3fb1 WN |
62 | |
63 | /* Copied from tools/perf/util/util.h */ | |
64 | #ifndef zfree | |
65 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
66 | #endif | |
67 | ||
68 | #ifndef zclose | |
69 | # define zclose(fd) ({ \ | |
70 | int ___err = 0; \ | |
71 | if ((fd) >= 0) \ | |
72 | ___err = close((fd)); \ | |
73 | fd = -1; \ | |
74 | ___err; }) | |
75 | #endif | |
76 | ||
77 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
78 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
79 | #else | |
80 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
81 | #endif | |
82 | ||
a5b8bd47 WN |
83 | /* |
84 | * bpf_prog should be a better name but it has been used in | |
85 | * linux/filter.h. | |
86 | */ | |
87 | struct bpf_program { | |
88 | /* Index in elf obj file, for relocation use. */ | |
89 | int idx; | |
90 | char *section_name; | |
91 | struct bpf_insn *insns; | |
92 | size_t insns_cnt; | |
34090915 WN |
93 | |
94 | struct { | |
95 | int insn_idx; | |
96 | int map_idx; | |
97 | } *reloc_desc; | |
98 | int nr_reloc; | |
55cffde2 WN |
99 | |
100 | int fd; | |
aa9b1ac3 WN |
101 | |
102 | struct bpf_object *obj; | |
103 | void *priv; | |
104 | bpf_program_clear_priv_t clear_priv; | |
a5b8bd47 WN |
105 | }; |
106 | ||
1a5e3fb1 | 107 | struct bpf_object { |
cb1e5e96 WN |
108 | char license[64]; |
109 | u32 kern_version; | |
0b3d1efa WN |
110 | void *maps_buf; |
111 | size_t maps_buf_sz; | |
112 | ||
a5b8bd47 WN |
113 | struct bpf_program *programs; |
114 | size_t nr_programs; | |
52d3352e WN |
115 | int *map_fds; |
116 | /* | |
117 | * This field is required because maps_buf will be freed and | |
118 | * maps_buf_sz will be set to 0 after loaded. | |
119 | */ | |
120 | size_t nr_map_fds; | |
121 | bool loaded; | |
a5b8bd47 | 122 | |
1a5e3fb1 WN |
123 | /* |
124 | * Information when doing elf related work. Only valid if fd | |
125 | * is valid. | |
126 | */ | |
127 | struct { | |
128 | int fd; | |
6c956392 WN |
129 | void *obj_buf; |
130 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
131 | Elf *elf; |
132 | GElf_Ehdr ehdr; | |
bec7d68c | 133 | Elf_Data *symbols; |
b62f06e8 WN |
134 | struct { |
135 | GElf_Shdr shdr; | |
136 | Elf_Data *data; | |
137 | } *reloc; | |
138 | int nr_reloc; | |
1a5e3fb1 WN |
139 | } efile; |
140 | char path[]; | |
141 | }; | |
142 | #define obj_elf_valid(o) ((o)->efile.elf) | |
143 | ||
55cffde2 WN |
144 | static void bpf_program__unload(struct bpf_program *prog) |
145 | { | |
146 | if (!prog) | |
147 | return; | |
148 | ||
149 | zclose(prog->fd); | |
150 | } | |
151 | ||
a5b8bd47 WN |
152 | static void bpf_program__exit(struct bpf_program *prog) |
153 | { | |
154 | if (!prog) | |
155 | return; | |
156 | ||
aa9b1ac3 WN |
157 | if (prog->clear_priv) |
158 | prog->clear_priv(prog, prog->priv); | |
159 | ||
160 | prog->priv = NULL; | |
161 | prog->clear_priv = NULL; | |
162 | ||
55cffde2 | 163 | bpf_program__unload(prog); |
a5b8bd47 WN |
164 | zfree(&prog->section_name); |
165 | zfree(&prog->insns); | |
34090915 WN |
166 | zfree(&prog->reloc_desc); |
167 | ||
168 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
169 | prog->insns_cnt = 0; |
170 | prog->idx = -1; | |
171 | } | |
172 | ||
173 | static int | |
174 | bpf_program__init(void *data, size_t size, char *name, int idx, | |
175 | struct bpf_program *prog) | |
176 | { | |
177 | if (size < sizeof(struct bpf_insn)) { | |
178 | pr_warning("corrupted section '%s'\n", name); | |
179 | return -EINVAL; | |
180 | } | |
181 | ||
182 | bzero(prog, sizeof(*prog)); | |
183 | ||
184 | prog->section_name = strdup(name); | |
185 | if (!prog->section_name) { | |
186 | pr_warning("failed to alloc name for prog %s\n", | |
187 | name); | |
188 | goto errout; | |
189 | } | |
190 | ||
191 | prog->insns = malloc(size); | |
192 | if (!prog->insns) { | |
193 | pr_warning("failed to alloc insns for %s\n", name); | |
194 | goto errout; | |
195 | } | |
196 | prog->insns_cnt = size / sizeof(struct bpf_insn); | |
197 | memcpy(prog->insns, data, | |
198 | prog->insns_cnt * sizeof(struct bpf_insn)); | |
199 | prog->idx = idx; | |
55cffde2 | 200 | prog->fd = -1; |
a5b8bd47 WN |
201 | |
202 | return 0; | |
203 | errout: | |
204 | bpf_program__exit(prog); | |
205 | return -ENOMEM; | |
206 | } | |
207 | ||
208 | static int | |
209 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
210 | char *name, int idx) | |
211 | { | |
212 | struct bpf_program prog, *progs; | |
213 | int nr_progs, err; | |
214 | ||
215 | err = bpf_program__init(data, size, name, idx, &prog); | |
216 | if (err) | |
217 | return err; | |
218 | ||
219 | progs = obj->programs; | |
220 | nr_progs = obj->nr_programs; | |
221 | ||
222 | progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); | |
223 | if (!progs) { | |
224 | /* | |
225 | * In this case the original obj->programs | |
226 | * is still valid, so don't need special treat for | |
227 | * bpf_close_object(). | |
228 | */ | |
229 | pr_warning("failed to alloc a new program '%s'\n", | |
230 | name); | |
231 | bpf_program__exit(&prog); | |
232 | return -ENOMEM; | |
233 | } | |
234 | ||
235 | pr_debug("found program %s\n", prog.section_name); | |
236 | obj->programs = progs; | |
237 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 238 | prog.obj = obj; |
a5b8bd47 WN |
239 | progs[nr_progs] = prog; |
240 | return 0; | |
241 | } | |
242 | ||
6c956392 WN |
243 | static struct bpf_object *bpf_object__new(const char *path, |
244 | void *obj_buf, | |
245 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
246 | { |
247 | struct bpf_object *obj; | |
248 | ||
249 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
250 | if (!obj) { | |
251 | pr_warning("alloc memory failed for %s\n", path); | |
252 | return NULL; | |
253 | } | |
254 | ||
255 | strcpy(obj->path, path); | |
256 | obj->efile.fd = -1; | |
6c956392 WN |
257 | |
258 | /* | |
259 | * Caller of this function should also calls | |
260 | * bpf_object__elf_finish() after data collection to return | |
261 | * obj_buf to user. If not, we should duplicate the buffer to | |
262 | * avoid user freeing them before elf finish. | |
263 | */ | |
264 | obj->efile.obj_buf = obj_buf; | |
265 | obj->efile.obj_buf_sz = obj_buf_sz; | |
266 | ||
52d3352e | 267 | obj->loaded = false; |
1a5e3fb1 WN |
268 | return obj; |
269 | } | |
270 | ||
271 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
272 | { | |
273 | if (!obj_elf_valid(obj)) | |
274 | return; | |
275 | ||
276 | if (obj->efile.elf) { | |
277 | elf_end(obj->efile.elf); | |
278 | obj->efile.elf = NULL; | |
279 | } | |
bec7d68c | 280 | obj->efile.symbols = NULL; |
b62f06e8 WN |
281 | |
282 | zfree(&obj->efile.reloc); | |
283 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 284 | zclose(obj->efile.fd); |
6c956392 WN |
285 | obj->efile.obj_buf = NULL; |
286 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
287 | } |
288 | ||
289 | static int bpf_object__elf_init(struct bpf_object *obj) | |
290 | { | |
291 | int err = 0; | |
292 | GElf_Ehdr *ep; | |
293 | ||
294 | if (obj_elf_valid(obj)) { | |
295 | pr_warning("elf init: internal error\n"); | |
296 | return -EEXIST; | |
297 | } | |
298 | ||
6c956392 WN |
299 | if (obj->efile.obj_buf_sz > 0) { |
300 | /* | |
301 | * obj_buf should have been validated by | |
302 | * bpf_object__open_buffer(). | |
303 | */ | |
304 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
305 | obj->efile.obj_buf_sz); | |
306 | } else { | |
307 | obj->efile.fd = open(obj->path, O_RDONLY); | |
308 | if (obj->efile.fd < 0) { | |
309 | pr_warning("failed to open %s: %s\n", obj->path, | |
310 | strerror(errno)); | |
311 | return -errno; | |
312 | } | |
313 | ||
314 | obj->efile.elf = elf_begin(obj->efile.fd, | |
315 | LIBBPF_ELF_C_READ_MMAP, | |
316 | NULL); | |
1a5e3fb1 WN |
317 | } |
318 | ||
1a5e3fb1 WN |
319 | if (!obj->efile.elf) { |
320 | pr_warning("failed to open %s as ELF file\n", | |
321 | obj->path); | |
322 | err = -EINVAL; | |
323 | goto errout; | |
324 | } | |
325 | ||
326 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
327 | pr_warning("failed to get EHDR from %s\n", | |
328 | obj->path); | |
329 | err = -EINVAL; | |
330 | goto errout; | |
331 | } | |
332 | ep = &obj->efile.ehdr; | |
333 | ||
334 | if ((ep->e_type != ET_REL) || (ep->e_machine != 0)) { | |
335 | pr_warning("%s is not an eBPF object file\n", | |
336 | obj->path); | |
337 | err = -EINVAL; | |
338 | goto errout; | |
339 | } | |
340 | ||
341 | return 0; | |
342 | errout: | |
343 | bpf_object__elf_finish(obj); | |
344 | return err; | |
345 | } | |
346 | ||
cc4228d5 WN |
347 | static int |
348 | bpf_object__check_endianness(struct bpf_object *obj) | |
349 | { | |
350 | static unsigned int const endian = 1; | |
351 | ||
352 | switch (obj->efile.ehdr.e_ident[EI_DATA]) { | |
353 | case ELFDATA2LSB: | |
354 | /* We are big endian, BPF obj is little endian. */ | |
355 | if (*(unsigned char const *)&endian != 1) | |
356 | goto mismatch; | |
357 | break; | |
358 | ||
359 | case ELFDATA2MSB: | |
360 | /* We are little endian, BPF obj is big endian. */ | |
361 | if (*(unsigned char const *)&endian != 0) | |
362 | goto mismatch; | |
363 | break; | |
364 | default: | |
365 | return -EINVAL; | |
366 | } | |
367 | ||
368 | return 0; | |
369 | ||
370 | mismatch: | |
371 | pr_warning("Error: endianness mismatch.\n"); | |
372 | return -EINVAL; | |
373 | } | |
374 | ||
cb1e5e96 WN |
375 | static int |
376 | bpf_object__init_license(struct bpf_object *obj, | |
377 | void *data, size_t size) | |
378 | { | |
379 | memcpy(obj->license, data, | |
380 | min(size, sizeof(obj->license) - 1)); | |
381 | pr_debug("license of %s is %s\n", obj->path, obj->license); | |
382 | return 0; | |
383 | } | |
384 | ||
385 | static int | |
386 | bpf_object__init_kversion(struct bpf_object *obj, | |
387 | void *data, size_t size) | |
388 | { | |
389 | u32 kver; | |
390 | ||
391 | if (size != sizeof(kver)) { | |
392 | pr_warning("invalid kver section in %s\n", obj->path); | |
393 | return -EINVAL; | |
394 | } | |
395 | memcpy(&kver, data, sizeof(kver)); | |
396 | obj->kern_version = kver; | |
397 | pr_debug("kernel version of %s is %x\n", obj->path, | |
398 | obj->kern_version); | |
399 | return 0; | |
400 | } | |
401 | ||
0b3d1efa WN |
402 | static int |
403 | bpf_object__init_maps(struct bpf_object *obj, void *data, | |
404 | size_t size) | |
405 | { | |
406 | if (size == 0) { | |
407 | pr_debug("%s doesn't need map definition\n", | |
408 | obj->path); | |
409 | return 0; | |
410 | } | |
411 | ||
412 | obj->maps_buf = malloc(size); | |
413 | if (!obj->maps_buf) { | |
414 | pr_warning("malloc maps failed: %s\n", obj->path); | |
415 | return -ENOMEM; | |
416 | } | |
417 | ||
418 | obj->maps_buf_sz = size; | |
419 | memcpy(obj->maps_buf, data, size); | |
420 | pr_debug("maps in %s: %ld bytes\n", obj->path, (long)size); | |
421 | return 0; | |
422 | } | |
423 | ||
29603665 WN |
424 | static int bpf_object__elf_collect(struct bpf_object *obj) |
425 | { | |
426 | Elf *elf = obj->efile.elf; | |
427 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
428 | Elf_Scn *scn = NULL; | |
429 | int idx = 0, err = 0; | |
430 | ||
431 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
432 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
433 | pr_warning("failed to get e_shstrndx from %s\n", | |
434 | obj->path); | |
435 | return -EINVAL; | |
436 | } | |
437 | ||
438 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
439 | char *name; | |
440 | GElf_Shdr sh; | |
441 | Elf_Data *data; | |
442 | ||
443 | idx++; | |
444 | if (gelf_getshdr(scn, &sh) != &sh) { | |
445 | pr_warning("failed to get section header from %s\n", | |
446 | obj->path); | |
447 | err = -EINVAL; | |
448 | goto out; | |
449 | } | |
450 | ||
451 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
452 | if (!name) { | |
453 | pr_warning("failed to get section name from %s\n", | |
454 | obj->path); | |
455 | err = -EINVAL; | |
456 | goto out; | |
457 | } | |
458 | ||
459 | data = elf_getdata(scn, 0); | |
460 | if (!data) { | |
461 | pr_warning("failed to get section data from %s(%s)\n", | |
462 | name, obj->path); | |
463 | err = -EINVAL; | |
464 | goto out; | |
465 | } | |
466 | pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n", | |
467 | name, (unsigned long)data->d_size, | |
468 | (int)sh.sh_link, (unsigned long)sh.sh_flags, | |
469 | (int)sh.sh_type); | |
cb1e5e96 WN |
470 | |
471 | if (strcmp(name, "license") == 0) | |
472 | err = bpf_object__init_license(obj, | |
473 | data->d_buf, | |
474 | data->d_size); | |
475 | else if (strcmp(name, "version") == 0) | |
476 | err = bpf_object__init_kversion(obj, | |
477 | data->d_buf, | |
478 | data->d_size); | |
0b3d1efa WN |
479 | else if (strcmp(name, "maps") == 0) |
480 | err = bpf_object__init_maps(obj, data->d_buf, | |
481 | data->d_size); | |
bec7d68c WN |
482 | else if (sh.sh_type == SHT_SYMTAB) { |
483 | if (obj->efile.symbols) { | |
484 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
485 | obj->path); | |
486 | err = -EEXIST; | |
487 | } else | |
488 | obj->efile.symbols = data; | |
a5b8bd47 WN |
489 | } else if ((sh.sh_type == SHT_PROGBITS) && |
490 | (sh.sh_flags & SHF_EXECINSTR) && | |
491 | (data->d_size > 0)) { | |
492 | err = bpf_object__add_program(obj, data->d_buf, | |
493 | data->d_size, name, idx); | |
494 | if (err) { | |
495 | char errmsg[128]; | |
496 | strerror_r(-err, errmsg, sizeof(errmsg)); | |
497 | pr_warning("failed to alloc program %s (%s): %s", | |
498 | name, obj->path, errmsg); | |
499 | } | |
b62f06e8 WN |
500 | } else if (sh.sh_type == SHT_REL) { |
501 | void *reloc = obj->efile.reloc; | |
502 | int nr_reloc = obj->efile.nr_reloc + 1; | |
503 | ||
504 | reloc = realloc(reloc, | |
505 | sizeof(*obj->efile.reloc) * nr_reloc); | |
506 | if (!reloc) { | |
507 | pr_warning("realloc failed\n"); | |
508 | err = -ENOMEM; | |
509 | } else { | |
510 | int n = nr_reloc - 1; | |
511 | ||
512 | obj->efile.reloc = reloc; | |
513 | obj->efile.nr_reloc = nr_reloc; | |
514 | ||
515 | obj->efile.reloc[n].shdr = sh; | |
516 | obj->efile.reloc[n].data = data; | |
517 | } | |
bec7d68c | 518 | } |
cb1e5e96 WN |
519 | if (err) |
520 | goto out; | |
29603665 WN |
521 | } |
522 | out: | |
523 | return err; | |
524 | } | |
525 | ||
34090915 WN |
526 | static struct bpf_program * |
527 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
528 | { | |
529 | struct bpf_program *prog; | |
530 | size_t i; | |
531 | ||
532 | for (i = 0; i < obj->nr_programs; i++) { | |
533 | prog = &obj->programs[i]; | |
534 | if (prog->idx == idx) | |
535 | return prog; | |
536 | } | |
537 | return NULL; | |
538 | } | |
539 | ||
540 | static int | |
541 | bpf_program__collect_reloc(struct bpf_program *prog, | |
542 | size_t nr_maps, GElf_Shdr *shdr, | |
543 | Elf_Data *data, Elf_Data *symbols) | |
544 | { | |
545 | int i, nrels; | |
546 | ||
547 | pr_debug("collecting relocating info for: '%s'\n", | |
548 | prog->section_name); | |
549 | nrels = shdr->sh_size / shdr->sh_entsize; | |
550 | ||
551 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
552 | if (!prog->reloc_desc) { | |
553 | pr_warning("failed to alloc memory in relocation\n"); | |
554 | return -ENOMEM; | |
555 | } | |
556 | prog->nr_reloc = nrels; | |
557 | ||
558 | for (i = 0; i < nrels; i++) { | |
559 | GElf_Sym sym; | |
560 | GElf_Rel rel; | |
561 | unsigned int insn_idx; | |
562 | struct bpf_insn *insns = prog->insns; | |
563 | size_t map_idx; | |
564 | ||
565 | if (!gelf_getrel(data, i, &rel)) { | |
566 | pr_warning("relocation: failed to get %d reloc\n", i); | |
567 | return -EINVAL; | |
568 | } | |
569 | ||
570 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); | |
571 | pr_debug("relocation: insn_idx=%u\n", insn_idx); | |
572 | ||
573 | if (!gelf_getsym(symbols, | |
574 | GELF_R_SYM(rel.r_info), | |
575 | &sym)) { | |
576 | pr_warning("relocation: symbol %"PRIx64" not found\n", | |
577 | GELF_R_SYM(rel.r_info)); | |
578 | return -EINVAL; | |
579 | } | |
580 | ||
581 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { | |
582 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
583 | insn_idx, insns[insn_idx].code); | |
584 | return -EINVAL; | |
585 | } | |
586 | ||
587 | map_idx = sym.st_value / sizeof(struct bpf_map_def); | |
588 | if (map_idx >= nr_maps) { | |
589 | pr_warning("bpf relocation: map_idx %d large than %d\n", | |
590 | (int)map_idx, (int)nr_maps - 1); | |
591 | return -EINVAL; | |
592 | } | |
593 | ||
594 | prog->reloc_desc[i].insn_idx = insn_idx; | |
595 | prog->reloc_desc[i].map_idx = map_idx; | |
596 | } | |
597 | return 0; | |
598 | } | |
599 | ||
52d3352e WN |
600 | static int |
601 | bpf_object__create_maps(struct bpf_object *obj) | |
602 | { | |
603 | unsigned int i; | |
604 | size_t nr_maps; | |
605 | int *pfd; | |
606 | ||
607 | nr_maps = obj->maps_buf_sz / sizeof(struct bpf_map_def); | |
608 | if (!obj->maps_buf || !nr_maps) { | |
609 | pr_debug("don't need create maps for %s\n", | |
610 | obj->path); | |
611 | return 0; | |
612 | } | |
613 | ||
614 | obj->map_fds = malloc(sizeof(int) * nr_maps); | |
615 | if (!obj->map_fds) { | |
616 | pr_warning("realloc perf_bpf_map_fds failed\n"); | |
617 | return -ENOMEM; | |
618 | } | |
619 | obj->nr_map_fds = nr_maps; | |
620 | ||
621 | /* fill all fd with -1 */ | |
622 | memset(obj->map_fds, -1, sizeof(int) * nr_maps); | |
623 | ||
624 | pfd = obj->map_fds; | |
625 | for (i = 0; i < nr_maps; i++) { | |
626 | struct bpf_map_def def; | |
627 | ||
628 | def = *(struct bpf_map_def *)(obj->maps_buf + | |
629 | i * sizeof(struct bpf_map_def)); | |
630 | ||
631 | *pfd = bpf_create_map(def.type, | |
632 | def.key_size, | |
633 | def.value_size, | |
634 | def.max_entries); | |
635 | if (*pfd < 0) { | |
636 | size_t j; | |
637 | int err = *pfd; | |
638 | ||
639 | pr_warning("failed to create map: %s\n", | |
640 | strerror(errno)); | |
641 | for (j = 0; j < i; j++) | |
642 | zclose(obj->map_fds[j]); | |
643 | obj->nr_map_fds = 0; | |
644 | zfree(&obj->map_fds); | |
645 | return err; | |
646 | } | |
647 | pr_debug("create map: fd=%d\n", *pfd); | |
648 | pfd++; | |
649 | } | |
650 | ||
651 | zfree(&obj->maps_buf); | |
652 | obj->maps_buf_sz = 0; | |
653 | return 0; | |
654 | } | |
655 | ||
8a47a6c5 WN |
656 | static int |
657 | bpf_program__relocate(struct bpf_program *prog, int *map_fds) | |
658 | { | |
659 | int i; | |
660 | ||
661 | if (!prog || !prog->reloc_desc) | |
662 | return 0; | |
663 | ||
664 | for (i = 0; i < prog->nr_reloc; i++) { | |
665 | int insn_idx, map_idx; | |
666 | struct bpf_insn *insns = prog->insns; | |
667 | ||
668 | insn_idx = prog->reloc_desc[i].insn_idx; | |
669 | map_idx = prog->reloc_desc[i].map_idx; | |
670 | ||
671 | if (insn_idx >= (int)prog->insns_cnt) { | |
672 | pr_warning("relocation out of range: '%s'\n", | |
673 | prog->section_name); | |
674 | return -ERANGE; | |
675 | } | |
676 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
677 | insns[insn_idx].imm = map_fds[map_idx]; | |
678 | } | |
679 | ||
680 | zfree(&prog->reloc_desc); | |
681 | prog->nr_reloc = 0; | |
682 | return 0; | |
683 | } | |
684 | ||
685 | ||
686 | static int | |
687 | bpf_object__relocate(struct bpf_object *obj) | |
688 | { | |
689 | struct bpf_program *prog; | |
690 | size_t i; | |
691 | int err; | |
692 | ||
693 | for (i = 0; i < obj->nr_programs; i++) { | |
694 | prog = &obj->programs[i]; | |
695 | ||
696 | err = bpf_program__relocate(prog, obj->map_fds); | |
697 | if (err) { | |
698 | pr_warning("failed to relocate '%s'\n", | |
699 | prog->section_name); | |
700 | return err; | |
701 | } | |
702 | } | |
703 | return 0; | |
704 | } | |
705 | ||
34090915 WN |
706 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
707 | { | |
708 | int i, err; | |
709 | ||
710 | if (!obj_elf_valid(obj)) { | |
711 | pr_warning("Internal error: elf object is closed\n"); | |
712 | return -EINVAL; | |
713 | } | |
714 | ||
715 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
716 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
717 | Elf_Data *data = obj->efile.reloc[i].data; | |
718 | int idx = shdr->sh_info; | |
719 | struct bpf_program *prog; | |
720 | size_t nr_maps = obj->maps_buf_sz / | |
721 | sizeof(struct bpf_map_def); | |
722 | ||
723 | if (shdr->sh_type != SHT_REL) { | |
724 | pr_warning("internal error at %d\n", __LINE__); | |
725 | return -EINVAL; | |
726 | } | |
727 | ||
728 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
729 | if (!prog) { | |
730 | pr_warning("relocation failed: no %d section\n", | |
731 | idx); | |
732 | return -ENOENT; | |
733 | } | |
734 | ||
735 | err = bpf_program__collect_reloc(prog, nr_maps, | |
736 | shdr, data, | |
737 | obj->efile.symbols); | |
738 | if (err) | |
739 | return -EINVAL; | |
740 | } | |
741 | return 0; | |
742 | } | |
743 | ||
55cffde2 WN |
744 | static int |
745 | load_program(struct bpf_insn *insns, int insns_cnt, | |
746 | char *license, u32 kern_version, int *pfd) | |
747 | { | |
748 | int ret; | |
749 | char *log_buf; | |
750 | ||
751 | if (!insns || !insns_cnt) | |
752 | return -EINVAL; | |
753 | ||
754 | log_buf = malloc(BPF_LOG_BUF_SIZE); | |
755 | if (!log_buf) | |
756 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
757 | ||
758 | ret = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns, | |
759 | insns_cnt, license, kern_version, | |
760 | log_buf, BPF_LOG_BUF_SIZE); | |
761 | ||
762 | if (ret >= 0) { | |
763 | *pfd = ret; | |
764 | ret = 0; | |
765 | goto out; | |
766 | } | |
767 | ||
768 | ret = -EINVAL; | |
769 | pr_warning("load bpf program failed: %s\n", strerror(errno)); | |
770 | ||
771 | if (log_buf) { | |
772 | pr_warning("-- BEGIN DUMP LOG ---\n"); | |
773 | pr_warning("\n%s\n", log_buf); | |
774 | pr_warning("-- END LOG --\n"); | |
775 | } | |
776 | ||
777 | out: | |
778 | free(log_buf); | |
779 | return ret; | |
780 | } | |
781 | ||
782 | static int | |
783 | bpf_program__load(struct bpf_program *prog, | |
784 | char *license, u32 kern_version) | |
785 | { | |
786 | int err, fd; | |
787 | ||
788 | err = load_program(prog->insns, prog->insns_cnt, | |
789 | license, kern_version, &fd); | |
790 | if (!err) | |
791 | prog->fd = fd; | |
792 | ||
793 | if (err) | |
794 | pr_warning("failed to load program '%s'\n", | |
795 | prog->section_name); | |
796 | zfree(&prog->insns); | |
797 | prog->insns_cnt = 0; | |
798 | return err; | |
799 | } | |
800 | ||
801 | static int | |
802 | bpf_object__load_progs(struct bpf_object *obj) | |
803 | { | |
804 | size_t i; | |
805 | int err; | |
806 | ||
807 | for (i = 0; i < obj->nr_programs; i++) { | |
808 | err = bpf_program__load(&obj->programs[i], | |
809 | obj->license, | |
810 | obj->kern_version); | |
811 | if (err) | |
812 | return err; | |
813 | } | |
814 | return 0; | |
815 | } | |
816 | ||
cb1e5e96 WN |
817 | static int bpf_object__validate(struct bpf_object *obj) |
818 | { | |
819 | if (obj->kern_version == 0) { | |
820 | pr_warning("%s doesn't provide kernel version\n", | |
821 | obj->path); | |
822 | return -EINVAL; | |
823 | } | |
824 | return 0; | |
825 | } | |
826 | ||
1a5e3fb1 | 827 | static struct bpf_object * |
6c956392 | 828 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) |
1a5e3fb1 WN |
829 | { |
830 | struct bpf_object *obj; | |
831 | ||
832 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
833 | pr_warning("failed to init libelf for %s\n", path); | |
834 | return NULL; | |
835 | } | |
836 | ||
6c956392 | 837 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
1a5e3fb1 WN |
838 | if (!obj) |
839 | return NULL; | |
840 | ||
841 | if (bpf_object__elf_init(obj)) | |
842 | goto out; | |
cc4228d5 WN |
843 | if (bpf_object__check_endianness(obj)) |
844 | goto out; | |
29603665 WN |
845 | if (bpf_object__elf_collect(obj)) |
846 | goto out; | |
34090915 WN |
847 | if (bpf_object__collect_reloc(obj)) |
848 | goto out; | |
cb1e5e96 WN |
849 | if (bpf_object__validate(obj)) |
850 | goto out; | |
1a5e3fb1 WN |
851 | |
852 | bpf_object__elf_finish(obj); | |
853 | return obj; | |
854 | out: | |
855 | bpf_object__close(obj); | |
856 | return NULL; | |
857 | } | |
858 | ||
859 | struct bpf_object *bpf_object__open(const char *path) | |
860 | { | |
861 | /* param validation */ | |
862 | if (!path) | |
863 | return NULL; | |
864 | ||
865 | pr_debug("loading %s\n", path); | |
866 | ||
6c956392 WN |
867 | return __bpf_object__open(path, NULL, 0); |
868 | } | |
869 | ||
870 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
871 | size_t obj_buf_sz) | |
872 | { | |
873 | /* param validation */ | |
874 | if (!obj_buf || obj_buf_sz <= 0) | |
875 | return NULL; | |
876 | ||
877 | pr_debug("loading object from buffer\n"); | |
878 | ||
879 | return __bpf_object__open("[buffer]", obj_buf, obj_buf_sz); | |
1a5e3fb1 WN |
880 | } |
881 | ||
52d3352e WN |
882 | int bpf_object__unload(struct bpf_object *obj) |
883 | { | |
884 | size_t i; | |
885 | ||
886 | if (!obj) | |
887 | return -EINVAL; | |
888 | ||
889 | for (i = 0; i < obj->nr_map_fds; i++) | |
890 | zclose(obj->map_fds[i]); | |
891 | zfree(&obj->map_fds); | |
892 | obj->nr_map_fds = 0; | |
893 | ||
55cffde2 WN |
894 | for (i = 0; i < obj->nr_programs; i++) |
895 | bpf_program__unload(&obj->programs[i]); | |
896 | ||
52d3352e WN |
897 | return 0; |
898 | } | |
899 | ||
900 | int bpf_object__load(struct bpf_object *obj) | |
901 | { | |
902 | if (!obj) | |
903 | return -EINVAL; | |
904 | ||
905 | if (obj->loaded) { | |
906 | pr_warning("object should not be loaded twice\n"); | |
907 | return -EINVAL; | |
908 | } | |
909 | ||
910 | obj->loaded = true; | |
911 | if (bpf_object__create_maps(obj)) | |
912 | goto out; | |
8a47a6c5 WN |
913 | if (bpf_object__relocate(obj)) |
914 | goto out; | |
55cffde2 WN |
915 | if (bpf_object__load_progs(obj)) |
916 | goto out; | |
52d3352e WN |
917 | |
918 | return 0; | |
919 | out: | |
920 | bpf_object__unload(obj); | |
921 | pr_warning("failed to load object '%s'\n", obj->path); | |
922 | return -EINVAL; | |
923 | } | |
924 | ||
1a5e3fb1 WN |
925 | void bpf_object__close(struct bpf_object *obj) |
926 | { | |
a5b8bd47 WN |
927 | size_t i; |
928 | ||
1a5e3fb1 WN |
929 | if (!obj) |
930 | return; | |
931 | ||
932 | bpf_object__elf_finish(obj); | |
52d3352e | 933 | bpf_object__unload(obj); |
1a5e3fb1 | 934 | |
0b3d1efa | 935 | zfree(&obj->maps_buf); |
a5b8bd47 WN |
936 | |
937 | if (obj->programs && obj->nr_programs) { | |
938 | for (i = 0; i < obj->nr_programs; i++) | |
939 | bpf_program__exit(&obj->programs[i]); | |
940 | } | |
941 | zfree(&obj->programs); | |
942 | ||
1a5e3fb1 WN |
943 | free(obj); |
944 | } | |
aa9b1ac3 WN |
945 | |
946 | struct bpf_program * | |
947 | bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) | |
948 | { | |
949 | size_t idx; | |
950 | ||
951 | if (!obj->programs) | |
952 | return NULL; | |
953 | /* First handler */ | |
954 | if (prev == NULL) | |
955 | return &obj->programs[0]; | |
956 | ||
957 | if (prev->obj != obj) { | |
958 | pr_warning("error: program handler doesn't match object\n"); | |
959 | return NULL; | |
960 | } | |
961 | ||
962 | idx = (prev - obj->programs) + 1; | |
963 | if (idx >= obj->nr_programs) | |
964 | return NULL; | |
965 | return &obj->programs[idx]; | |
966 | } | |
967 | ||
968 | int bpf_program__set_private(struct bpf_program *prog, | |
969 | void *priv, | |
970 | bpf_program_clear_priv_t clear_priv) | |
971 | { | |
972 | if (prog->priv && prog->clear_priv) | |
973 | prog->clear_priv(prog, prog->priv); | |
974 | ||
975 | prog->priv = priv; | |
976 | prog->clear_priv = clear_priv; | |
977 | return 0; | |
978 | } | |
979 | ||
980 | int bpf_program__get_private(struct bpf_program *prog, void **ppriv) | |
981 | { | |
982 | *ppriv = prog->priv; | |
983 | return 0; | |
984 | } | |
985 | ||
986 | const char *bpf_program__title(struct bpf_program *prog, bool dup) | |
987 | { | |
988 | const char *title; | |
989 | ||
990 | title = prog->section_name; | |
991 | if (dup) { | |
992 | title = strdup(title); | |
993 | if (!title) { | |
994 | pr_warning("failed to strdup program title\n"); | |
995 | return NULL; | |
996 | } | |
997 | } | |
998 | ||
999 | return title; | |
1000 | } | |
1001 | ||
1002 | int bpf_program__fd(struct bpf_program *prog) | |
1003 | { | |
1004 | return prog->fd; | |
1005 | } |