2 * Common eBPF ELF object loading operations.
4 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
5 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
6 * Copyright (C) 2015 Huawei Inc.
17 #include <asm/unistd.h>
18 #include <linux/kernel.h>
19 #include <linux/bpf.h>
26 #define __printf(a, b) __attribute__((format(printf, a, b)))
29 static int __base_pr(const char *format, ...)
34 va_start(args, format);
35 err = vfprintf(stderr, format, args);
40 static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr;
41 static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr;
42 static __printf(1, 2) libbpf_print_fn_t __pr_debug;
44 #define __pr(func, fmt, ...) \
47 (func)("libbpf: " fmt, ##__VA_ARGS__); \
50 #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__)
51 #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__)
52 #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__)
54 void libbpf_set_print(libbpf_print_fn_t warn,
55 libbpf_print_fn_t info,
56 libbpf_print_fn_t debug)
63 /* Copied from tools/perf/util/util.h */
65 # define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
69 # define zclose(fd) ({ \
72 ___err = close((fd)); \
77 #ifdef HAVE_LIBELF_MMAP_SUPPORT
78 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP
80 # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ
84 * bpf_prog should be a better name but it has been used in
88 /* Index in elf obj file, for relocation use. */
91 struct bpf_insn *insns;
102 struct bpf_object *obj;
104 bpf_program_clear_priv_t clear_priv;
113 struct bpf_program *programs;
117 * This field is required because maps_buf will be freed and
118 * maps_buf_sz will be set to 0 after loaded.
124 * Information when doing elf related work. Only valid if fd
142 #define obj_elf_valid(o) ((o)->efile.elf)
144 static void bpf_program__unload(struct bpf_program *prog)
152 static void bpf_program__exit(struct bpf_program *prog)
157 if (prog->clear_priv)
158 prog->clear_priv(prog, prog->priv);
161 prog->clear_priv = NULL;
163 bpf_program__unload(prog);
164 zfree(&prog->section_name);
166 zfree(&prog->reloc_desc);
174 bpf_program__init(void *data, size_t size, char *name, int idx,
175 struct bpf_program *prog)
177 if (size < sizeof(struct bpf_insn)) {
178 pr_warning("corrupted section '%s'\n", name);
182 bzero(prog, sizeof(*prog));
184 prog->section_name = strdup(name);
185 if (!prog->section_name) {
186 pr_warning("failed to alloc name for prog %s\n",
191 prog->insns = malloc(size);
193 pr_warning("failed to alloc insns for %s\n", name);
196 prog->insns_cnt = size / sizeof(struct bpf_insn);
197 memcpy(prog->insns, data,
198 prog->insns_cnt * sizeof(struct bpf_insn));
204 bpf_program__exit(prog);
209 bpf_object__add_program(struct bpf_object *obj, void *data, size_t size,
212 struct bpf_program prog, *progs;
215 err = bpf_program__init(data, size, name, idx, &prog);
219 progs = obj->programs;
220 nr_progs = obj->nr_programs;
222 progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1));
225 * In this case the original obj->programs
226 * is still valid, so don't need special treat for
227 * bpf_close_object().
229 pr_warning("failed to alloc a new program '%s'\n",
231 bpf_program__exit(&prog);
235 pr_debug("found program %s\n", prog.section_name);
236 obj->programs = progs;
237 obj->nr_programs = nr_progs + 1;
239 progs[nr_progs] = prog;
243 static struct bpf_object *bpf_object__new(const char *path,
247 struct bpf_object *obj;
249 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
251 pr_warning("alloc memory failed for %s\n", path);
255 strcpy(obj->path, path);
259 * Caller of this function should also calls
260 * bpf_object__elf_finish() after data collection to return
261 * obj_buf to user. If not, we should duplicate the buffer to
262 * avoid user freeing them before elf finish.
264 obj->efile.obj_buf = obj_buf;
265 obj->efile.obj_buf_sz = obj_buf_sz;
271 static void bpf_object__elf_finish(struct bpf_object *obj)
273 if (!obj_elf_valid(obj))
276 if (obj->efile.elf) {
277 elf_end(obj->efile.elf);
278 obj->efile.elf = NULL;
280 obj->efile.symbols = NULL;
282 zfree(&obj->efile.reloc);
283 obj->efile.nr_reloc = 0;
284 zclose(obj->efile.fd);
285 obj->efile.obj_buf = NULL;
286 obj->efile.obj_buf_sz = 0;
289 static int bpf_object__elf_init(struct bpf_object *obj)
294 if (obj_elf_valid(obj)) {
295 pr_warning("elf init: internal error\n");
299 if (obj->efile.obj_buf_sz > 0) {
301 * obj_buf should have been validated by
302 * bpf_object__open_buffer().
304 obj->efile.elf = elf_memory(obj->efile.obj_buf,
305 obj->efile.obj_buf_sz);
307 obj->efile.fd = open(obj->path, O_RDONLY);
308 if (obj->efile.fd < 0) {
309 pr_warning("failed to open %s: %s\n", obj->path,
314 obj->efile.elf = elf_begin(obj->efile.fd,
315 LIBBPF_ELF_C_READ_MMAP,
319 if (!obj->efile.elf) {
320 pr_warning("failed to open %s as ELF file\n",
326 if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) {
327 pr_warning("failed to get EHDR from %s\n",
332 ep = &obj->efile.ehdr;
334 if ((ep->e_type != ET_REL) || (ep->e_machine != 0)) {
335 pr_warning("%s is not an eBPF object file\n",
343 bpf_object__elf_finish(obj);
348 bpf_object__check_endianness(struct bpf_object *obj)
350 static unsigned int const endian = 1;
352 switch (obj->efile.ehdr.e_ident[EI_DATA]) {
354 /* We are big endian, BPF obj is little endian. */
355 if (*(unsigned char const *)&endian != 1)
360 /* We are little endian, BPF obj is big endian. */
361 if (*(unsigned char const *)&endian != 0)
371 pr_warning("Error: endianness mismatch.\n");
376 bpf_object__init_license(struct bpf_object *obj,
377 void *data, size_t size)
379 memcpy(obj->license, data,
380 min(size, sizeof(obj->license) - 1));
381 pr_debug("license of %s is %s\n", obj->path, obj->license);
386 bpf_object__init_kversion(struct bpf_object *obj,
387 void *data, size_t size)
391 if (size != sizeof(kver)) {
392 pr_warning("invalid kver section in %s\n", obj->path);
395 memcpy(&kver, data, sizeof(kver));
396 obj->kern_version = kver;
397 pr_debug("kernel version of %s is %x\n", obj->path,
403 bpf_object__init_maps(struct bpf_object *obj, void *data,
407 pr_debug("%s doesn't need map definition\n",
412 obj->maps_buf = malloc(size);
413 if (!obj->maps_buf) {
414 pr_warning("malloc maps failed: %s\n", obj->path);
418 obj->maps_buf_sz = size;
419 memcpy(obj->maps_buf, data, size);
420 pr_debug("maps in %s: %ld bytes\n", obj->path, (long)size);
424 static int bpf_object__elf_collect(struct bpf_object *obj)
426 Elf *elf = obj->efile.elf;
427 GElf_Ehdr *ep = &obj->efile.ehdr;
429 int idx = 0, err = 0;
431 /* Elf is corrupted/truncated, avoid calling elf_strptr. */
432 if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) {
433 pr_warning("failed to get e_shstrndx from %s\n",
438 while ((scn = elf_nextscn(elf, scn)) != NULL) {
444 if (gelf_getshdr(scn, &sh) != &sh) {
445 pr_warning("failed to get section header from %s\n",
451 name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name);
453 pr_warning("failed to get section name from %s\n",
459 data = elf_getdata(scn, 0);
461 pr_warning("failed to get section data from %s(%s)\n",
466 pr_debug("section %s, size %ld, link %d, flags %lx, type=%d\n",
467 name, (unsigned long)data->d_size,
468 (int)sh.sh_link, (unsigned long)sh.sh_flags,
471 if (strcmp(name, "license") == 0)
472 err = bpf_object__init_license(obj,
475 else if (strcmp(name, "version") == 0)
476 err = bpf_object__init_kversion(obj,
479 else if (strcmp(name, "maps") == 0)
480 err = bpf_object__init_maps(obj, data->d_buf,
482 else if (sh.sh_type == SHT_SYMTAB) {
483 if (obj->efile.symbols) {
484 pr_warning("bpf: multiple SYMTAB in %s\n",
488 obj->efile.symbols = data;
489 } else if ((sh.sh_type == SHT_PROGBITS) &&
490 (sh.sh_flags & SHF_EXECINSTR) &&
491 (data->d_size > 0)) {
492 err = bpf_object__add_program(obj, data->d_buf,
493 data->d_size, name, idx);
496 strerror_r(-err, errmsg, sizeof(errmsg));
497 pr_warning("failed to alloc program %s (%s): %s",
498 name, obj->path, errmsg);
500 } else if (sh.sh_type == SHT_REL) {
501 void *reloc = obj->efile.reloc;
502 int nr_reloc = obj->efile.nr_reloc + 1;
504 reloc = realloc(reloc,
505 sizeof(*obj->efile.reloc) * nr_reloc);
507 pr_warning("realloc failed\n");
510 int n = nr_reloc - 1;
512 obj->efile.reloc = reloc;
513 obj->efile.nr_reloc = nr_reloc;
515 obj->efile.reloc[n].shdr = sh;
516 obj->efile.reloc[n].data = data;
526 static struct bpf_program *
527 bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx)
529 struct bpf_program *prog;
532 for (i = 0; i < obj->nr_programs; i++) {
533 prog = &obj->programs[i];
534 if (prog->idx == idx)
541 bpf_program__collect_reloc(struct bpf_program *prog,
542 size_t nr_maps, GElf_Shdr *shdr,
543 Elf_Data *data, Elf_Data *symbols)
547 pr_debug("collecting relocating info for: '%s'\n",
549 nrels = shdr->sh_size / shdr->sh_entsize;
551 prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels);
552 if (!prog->reloc_desc) {
553 pr_warning("failed to alloc memory in relocation\n");
556 prog->nr_reloc = nrels;
558 for (i = 0; i < nrels; i++) {
561 unsigned int insn_idx;
562 struct bpf_insn *insns = prog->insns;
565 if (!gelf_getrel(data, i, &rel)) {
566 pr_warning("relocation: failed to get %d reloc\n", i);
570 insn_idx = rel.r_offset / sizeof(struct bpf_insn);
571 pr_debug("relocation: insn_idx=%u\n", insn_idx);
573 if (!gelf_getsym(symbols,
574 GELF_R_SYM(rel.r_info),
576 pr_warning("relocation: symbol %"PRIx64" not found\n",
577 GELF_R_SYM(rel.r_info));
581 if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) {
582 pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n",
583 insn_idx, insns[insn_idx].code);
587 map_idx = sym.st_value / sizeof(struct bpf_map_def);
588 if (map_idx >= nr_maps) {
589 pr_warning("bpf relocation: map_idx %d large than %d\n",
590 (int)map_idx, (int)nr_maps - 1);
594 prog->reloc_desc[i].insn_idx = insn_idx;
595 prog->reloc_desc[i].map_idx = map_idx;
601 bpf_object__create_maps(struct bpf_object *obj)
607 nr_maps = obj->maps_buf_sz / sizeof(struct bpf_map_def);
608 if (!obj->maps_buf || !nr_maps) {
609 pr_debug("don't need create maps for %s\n",
614 obj->map_fds = malloc(sizeof(int) * nr_maps);
616 pr_warning("realloc perf_bpf_map_fds failed\n");
619 obj->nr_map_fds = nr_maps;
621 /* fill all fd with -1 */
622 memset(obj->map_fds, -1, sizeof(int) * nr_maps);
625 for (i = 0; i < nr_maps; i++) {
626 struct bpf_map_def def;
628 def = *(struct bpf_map_def *)(obj->maps_buf +
629 i * sizeof(struct bpf_map_def));
631 *pfd = bpf_create_map(def.type,
639 pr_warning("failed to create map: %s\n",
641 for (j = 0; j < i; j++)
642 zclose(obj->map_fds[j]);
644 zfree(&obj->map_fds);
647 pr_debug("create map: fd=%d\n", *pfd);
651 zfree(&obj->maps_buf);
652 obj->maps_buf_sz = 0;
657 bpf_program__relocate(struct bpf_program *prog, int *map_fds)
661 if (!prog || !prog->reloc_desc)
664 for (i = 0; i < prog->nr_reloc; i++) {
665 int insn_idx, map_idx;
666 struct bpf_insn *insns = prog->insns;
668 insn_idx = prog->reloc_desc[i].insn_idx;
669 map_idx = prog->reloc_desc[i].map_idx;
671 if (insn_idx >= (int)prog->insns_cnt) {
672 pr_warning("relocation out of range: '%s'\n",
676 insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD;
677 insns[insn_idx].imm = map_fds[map_idx];
680 zfree(&prog->reloc_desc);
687 bpf_object__relocate(struct bpf_object *obj)
689 struct bpf_program *prog;
693 for (i = 0; i < obj->nr_programs; i++) {
694 prog = &obj->programs[i];
696 err = bpf_program__relocate(prog, obj->map_fds);
698 pr_warning("failed to relocate '%s'\n",
706 static int bpf_object__collect_reloc(struct bpf_object *obj)
710 if (!obj_elf_valid(obj)) {
711 pr_warning("Internal error: elf object is closed\n");
715 for (i = 0; i < obj->efile.nr_reloc; i++) {
716 GElf_Shdr *shdr = &obj->efile.reloc[i].shdr;
717 Elf_Data *data = obj->efile.reloc[i].data;
718 int idx = shdr->sh_info;
719 struct bpf_program *prog;
720 size_t nr_maps = obj->maps_buf_sz /
721 sizeof(struct bpf_map_def);
723 if (shdr->sh_type != SHT_REL) {
724 pr_warning("internal error at %d\n", __LINE__);
728 prog = bpf_object__find_prog_by_idx(obj, idx);
730 pr_warning("relocation failed: no %d section\n",
735 err = bpf_program__collect_reloc(prog, nr_maps,
745 load_program(struct bpf_insn *insns, int insns_cnt,
746 char *license, u32 kern_version, int *pfd)
751 if (!insns || !insns_cnt)
754 log_buf = malloc(BPF_LOG_BUF_SIZE);
756 pr_warning("Alloc log buffer for bpf loader error, continue without log\n");
758 ret = bpf_load_program(BPF_PROG_TYPE_KPROBE, insns,
759 insns_cnt, license, kern_version,
760 log_buf, BPF_LOG_BUF_SIZE);
769 pr_warning("load bpf program failed: %s\n", strerror(errno));
772 pr_warning("-- BEGIN DUMP LOG ---\n");
773 pr_warning("\n%s\n", log_buf);
774 pr_warning("-- END LOG --\n");
783 bpf_program__load(struct bpf_program *prog,
784 char *license, u32 kern_version)
788 err = load_program(prog->insns, prog->insns_cnt,
789 license, kern_version, &fd);
794 pr_warning("failed to load program '%s'\n",
802 bpf_object__load_progs(struct bpf_object *obj)
807 for (i = 0; i < obj->nr_programs; i++) {
808 err = bpf_program__load(&obj->programs[i],
817 static int bpf_object__validate(struct bpf_object *obj)
819 if (obj->kern_version == 0) {
820 pr_warning("%s doesn't provide kernel version\n",
827 static struct bpf_object *
828 __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz)
830 struct bpf_object *obj;
832 if (elf_version(EV_CURRENT) == EV_NONE) {
833 pr_warning("failed to init libelf for %s\n", path);
837 obj = bpf_object__new(path, obj_buf, obj_buf_sz);
841 if (bpf_object__elf_init(obj))
843 if (bpf_object__check_endianness(obj))
845 if (bpf_object__elf_collect(obj))
847 if (bpf_object__collect_reloc(obj))
849 if (bpf_object__validate(obj))
852 bpf_object__elf_finish(obj);
855 bpf_object__close(obj);
859 struct bpf_object *bpf_object__open(const char *path)
861 /* param validation */
865 pr_debug("loading %s\n", path);
867 return __bpf_object__open(path, NULL, 0);
870 struct bpf_object *bpf_object__open_buffer(void *obj_buf,
873 /* param validation */
874 if (!obj_buf || obj_buf_sz <= 0)
877 pr_debug("loading object from buffer\n");
879 return __bpf_object__open("[buffer]", obj_buf, obj_buf_sz);
882 int bpf_object__unload(struct bpf_object *obj)
889 for (i = 0; i < obj->nr_map_fds; i++)
890 zclose(obj->map_fds[i]);
891 zfree(&obj->map_fds);
894 for (i = 0; i < obj->nr_programs; i++)
895 bpf_program__unload(&obj->programs[i]);
900 int bpf_object__load(struct bpf_object *obj)
906 pr_warning("object should not be loaded twice\n");
911 if (bpf_object__create_maps(obj))
913 if (bpf_object__relocate(obj))
915 if (bpf_object__load_progs(obj))
920 bpf_object__unload(obj);
921 pr_warning("failed to load object '%s'\n", obj->path);
925 void bpf_object__close(struct bpf_object *obj)
932 bpf_object__elf_finish(obj);
933 bpf_object__unload(obj);
935 zfree(&obj->maps_buf);
937 if (obj->programs && obj->nr_programs) {
938 for (i = 0; i < obj->nr_programs; i++)
939 bpf_program__exit(&obj->programs[i]);
941 zfree(&obj->programs);
947 bpf_program__next(struct bpf_program *prev, struct bpf_object *obj)
955 return &obj->programs[0];
957 if (prev->obj != obj) {
958 pr_warning("error: program handler doesn't match object\n");
962 idx = (prev - obj->programs) + 1;
963 if (idx >= obj->nr_programs)
965 return &obj->programs[idx];
968 int bpf_program__set_private(struct bpf_program *prog,
970 bpf_program_clear_priv_t clear_priv)
972 if (prog->priv && prog->clear_priv)
973 prog->clear_priv(prog, prog->priv);
976 prog->clear_priv = clear_priv;
980 int bpf_program__get_private(struct bpf_program *prog, void **ppriv)
986 const char *bpf_program__title(struct bpf_program *prog, bool dup)
990 title = prog->section_name;
992 title = strdup(title);
994 pr_warning("failed to strdup program title\n");
1002 int bpf_program__fd(struct bpf_program *prog)