Commit | Line | Data |
---|---|---|
6061a3d6 EL |
1 | // SPDX-License-Identifier: LGPL-2.1 |
2 | ||
1b76c13e WN |
3 | /* |
4 | * Common eBPF ELF object loading operations. | |
5 | * | |
6 | * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> | |
7 | * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> | |
8 | * Copyright (C) 2015 Huawei Inc. | |
f367540c | 9 | * Copyright (C) 2017 Nicira, Inc. |
203d1cac WN |
10 | * |
11 | * This program is free software; you can redistribute it and/or | |
12 | * modify it under the terms of the GNU Lesser General Public | |
13 | * License as published by the Free Software Foundation; | |
14 | * version 2.1 of the License (not later!) | |
15 | * | |
16 | * This program is distributed in the hope that it will be useful, | |
17 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
18 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
19 | * GNU Lesser General Public License for more details. | |
20 | * | |
21 | * You should have received a copy of the GNU Lesser General Public | |
22 | * License along with this program; if not, see <http://www.gnu.org/licenses> | |
1b76c13e WN |
23 | */ |
24 | ||
25 | #include <stdlib.h> | |
b3f59d66 WN |
26 | #include <stdio.h> |
27 | #include <stdarg.h> | |
f367540c | 28 | #include <libgen.h> |
34090915 | 29 | #include <inttypes.h> |
b3f59d66 | 30 | #include <string.h> |
1b76c13e | 31 | #include <unistd.h> |
1a5e3fb1 WN |
32 | #include <fcntl.h> |
33 | #include <errno.h> | |
1b76c13e | 34 | #include <asm/unistd.h> |
e28ff1a8 | 35 | #include <linux/err.h> |
cb1e5e96 | 36 | #include <linux/kernel.h> |
1b76c13e | 37 | #include <linux/bpf.h> |
9a208eff | 38 | #include <linux/list.h> |
f367540c JS |
39 | #include <linux/limits.h> |
40 | #include <sys/stat.h> | |
41 | #include <sys/types.h> | |
42 | #include <sys/vfs.h> | |
1a5e3fb1 WN |
43 | #include <libelf.h> |
44 | #include <gelf.h> | |
1b76c13e WN |
45 | |
46 | #include "libbpf.h" | |
52d3352e | 47 | #include "bpf.h" |
b3f59d66 | 48 | |
9b16137a WN |
49 | #ifndef EM_BPF |
50 | #define EM_BPF 247 | |
51 | #endif | |
52 | ||
f367540c JS |
53 | #ifndef BPF_FS_MAGIC |
54 | #define BPF_FS_MAGIC 0xcafe4a11 | |
55 | #endif | |
56 | ||
b3f59d66 WN |
57 | #define __printf(a, b) __attribute__((format(printf, a, b))) |
58 | ||
59 | __printf(1, 2) | |
60 | static int __base_pr(const char *format, ...) | |
61 | { | |
62 | va_list args; | |
63 | int err; | |
64 | ||
65 | va_start(args, format); | |
66 | err = vfprintf(stderr, format, args); | |
67 | va_end(args); | |
68 | return err; | |
69 | } | |
70 | ||
71 | static __printf(1, 2) libbpf_print_fn_t __pr_warning = __base_pr; | |
72 | static __printf(1, 2) libbpf_print_fn_t __pr_info = __base_pr; | |
73 | static __printf(1, 2) libbpf_print_fn_t __pr_debug; | |
74 | ||
75 | #define __pr(func, fmt, ...) \ | |
76 | do { \ | |
77 | if ((func)) \ | |
78 | (func)("libbpf: " fmt, ##__VA_ARGS__); \ | |
79 | } while (0) | |
80 | ||
81 | #define pr_warning(fmt, ...) __pr(__pr_warning, fmt, ##__VA_ARGS__) | |
82 | #define pr_info(fmt, ...) __pr(__pr_info, fmt, ##__VA_ARGS__) | |
83 | #define pr_debug(fmt, ...) __pr(__pr_debug, fmt, ##__VA_ARGS__) | |
84 | ||
85 | void libbpf_set_print(libbpf_print_fn_t warn, | |
86 | libbpf_print_fn_t info, | |
87 | libbpf_print_fn_t debug) | |
88 | { | |
89 | __pr_warning = warn; | |
90 | __pr_info = info; | |
91 | __pr_debug = debug; | |
92 | } | |
1a5e3fb1 | 93 | |
6371ca3b WN |
94 | #define STRERR_BUFSIZE 128 |
95 | ||
96 | #define ERRNO_OFFSET(e) ((e) - __LIBBPF_ERRNO__START) | |
97 | #define ERRCODE_OFFSET(c) ERRNO_OFFSET(LIBBPF_ERRNO__##c) | |
98 | #define NR_ERRNO (__LIBBPF_ERRNO__END - __LIBBPF_ERRNO__START) | |
99 | ||
100 | static const char *libbpf_strerror_table[NR_ERRNO] = { | |
101 | [ERRCODE_OFFSET(LIBELF)] = "Something wrong in libelf", | |
102 | [ERRCODE_OFFSET(FORMAT)] = "BPF object format invalid", | |
103 | [ERRCODE_OFFSET(KVERSION)] = "'version' section incorrect or lost", | |
de8a63bd | 104 | [ERRCODE_OFFSET(ENDIAN)] = "Endian mismatch", |
6371ca3b WN |
105 | [ERRCODE_OFFSET(INTERNAL)] = "Internal error in libbpf", |
106 | [ERRCODE_OFFSET(RELOC)] = "Relocation failed", | |
107 | [ERRCODE_OFFSET(VERIFY)] = "Kernel verifier blocks program loading", | |
108 | [ERRCODE_OFFSET(PROG2BIG)] = "Program too big", | |
109 | [ERRCODE_OFFSET(KVER)] = "Incorrect kernel version", | |
705fa219 | 110 | [ERRCODE_OFFSET(PROGTYPE)] = "Kernel doesn't support this program type", |
949abbe8 EL |
111 | [ERRCODE_OFFSET(WRNGPID)] = "Wrong pid in netlink message", |
112 | [ERRCODE_OFFSET(INVSEQ)] = "Invalid netlink sequence", | |
6371ca3b WN |
113 | }; |
114 | ||
115 | int libbpf_strerror(int err, char *buf, size_t size) | |
116 | { | |
117 | if (!buf || !size) | |
118 | return -1; | |
119 | ||
120 | err = err > 0 ? err : -err; | |
121 | ||
122 | if (err < __LIBBPF_ERRNO__START) { | |
123 | int ret; | |
124 | ||
125 | ret = strerror_r(err, buf, size); | |
126 | buf[size - 1] = '\0'; | |
127 | return ret; | |
128 | } | |
129 | ||
130 | if (err < __LIBBPF_ERRNO__END) { | |
131 | const char *msg; | |
132 | ||
133 | msg = libbpf_strerror_table[ERRNO_OFFSET(err)]; | |
134 | snprintf(buf, size, "%s", msg); | |
135 | buf[size - 1] = '\0'; | |
136 | return 0; | |
137 | } | |
138 | ||
139 | snprintf(buf, size, "Unknown libbpf error %d", err); | |
140 | buf[size - 1] = '\0'; | |
141 | return -1; | |
142 | } | |
143 | ||
144 | #define CHECK_ERR(action, err, out) do { \ | |
145 | err = action; \ | |
146 | if (err) \ | |
147 | goto out; \ | |
148 | } while(0) | |
149 | ||
150 | ||
1a5e3fb1 WN |
151 | /* Copied from tools/perf/util/util.h */ |
152 | #ifndef zfree | |
153 | # define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) | |
154 | #endif | |
155 | ||
156 | #ifndef zclose | |
157 | # define zclose(fd) ({ \ | |
158 | int ___err = 0; \ | |
159 | if ((fd) >= 0) \ | |
160 | ___err = close((fd)); \ | |
161 | fd = -1; \ | |
162 | ___err; }) | |
163 | #endif | |
164 | ||
165 | #ifdef HAVE_LIBELF_MMAP_SUPPORT | |
166 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ_MMAP | |
167 | #else | |
168 | # define LIBBPF_ELF_C_READ_MMAP ELF_C_READ | |
169 | #endif | |
170 | ||
a5b8bd47 WN |
171 | /* |
172 | * bpf_prog should be a better name but it has been used in | |
173 | * linux/filter.h. | |
174 | */ | |
175 | struct bpf_program { | |
176 | /* Index in elf obj file, for relocation use. */ | |
177 | int idx; | |
88cda1c9 | 178 | char *name; |
a5b8bd47 WN |
179 | char *section_name; |
180 | struct bpf_insn *insns; | |
48cca7e4 | 181 | size_t insns_cnt, main_prog_cnt; |
5f44e4c8 | 182 | enum bpf_prog_type type; |
34090915 | 183 | |
48cca7e4 AS |
184 | struct reloc_desc { |
185 | enum { | |
186 | RELO_LD64, | |
187 | RELO_CALL, | |
188 | } type; | |
34090915 | 189 | int insn_idx; |
48cca7e4 AS |
190 | union { |
191 | int map_idx; | |
192 | int text_off; | |
193 | }; | |
34090915 WN |
194 | } *reloc_desc; |
195 | int nr_reloc; | |
55cffde2 | 196 | |
b580563e WN |
197 | struct { |
198 | int nr; | |
199 | int *fds; | |
200 | } instances; | |
201 | bpf_program_prep_t preprocessor; | |
aa9b1ac3 WN |
202 | |
203 | struct bpf_object *obj; | |
204 | void *priv; | |
205 | bpf_program_clear_priv_t clear_priv; | |
d7be143b AI |
206 | |
207 | enum bpf_attach_type expected_attach_type; | |
a5b8bd47 WN |
208 | }; |
209 | ||
9d759a9b WN |
210 | struct bpf_map { |
211 | int fd; | |
561bbcca | 212 | char *name; |
4708bbda | 213 | size_t offset; |
9d759a9b WN |
214 | struct bpf_map_def def; |
215 | void *priv; | |
216 | bpf_map_clear_priv_t clear_priv; | |
217 | }; | |
218 | ||
9a208eff WN |
219 | static LIST_HEAD(bpf_objects_list); |
220 | ||
1a5e3fb1 | 221 | struct bpf_object { |
cb1e5e96 WN |
222 | char license[64]; |
223 | u32 kern_version; | |
0b3d1efa | 224 | |
a5b8bd47 WN |
225 | struct bpf_program *programs; |
226 | size_t nr_programs; | |
9d759a9b WN |
227 | struct bpf_map *maps; |
228 | size_t nr_maps; | |
229 | ||
52d3352e | 230 | bool loaded; |
a5b8bd47 | 231 | |
1a5e3fb1 WN |
232 | /* |
233 | * Information when doing elf related work. Only valid if fd | |
234 | * is valid. | |
235 | */ | |
236 | struct { | |
237 | int fd; | |
6c956392 WN |
238 | void *obj_buf; |
239 | size_t obj_buf_sz; | |
1a5e3fb1 WN |
240 | Elf *elf; |
241 | GElf_Ehdr ehdr; | |
bec7d68c | 242 | Elf_Data *symbols; |
77ba9a5b | 243 | size_t strtabidx; |
b62f06e8 WN |
244 | struct { |
245 | GElf_Shdr shdr; | |
246 | Elf_Data *data; | |
247 | } *reloc; | |
248 | int nr_reloc; | |
666810e8 | 249 | int maps_shndx; |
48cca7e4 | 250 | int text_shndx; |
1a5e3fb1 | 251 | } efile; |
9a208eff WN |
252 | /* |
253 | * All loaded bpf_object is linked in a list, which is | |
254 | * hidden to caller. bpf_objects__<func> handlers deal with | |
255 | * all objects. | |
256 | */ | |
257 | struct list_head list; | |
10931d24 WN |
258 | |
259 | void *priv; | |
260 | bpf_object_clear_priv_t clear_priv; | |
261 | ||
1a5e3fb1 WN |
262 | char path[]; |
263 | }; | |
264 | #define obj_elf_valid(o) ((o)->efile.elf) | |
265 | ||
55cffde2 WN |
266 | static void bpf_program__unload(struct bpf_program *prog) |
267 | { | |
b580563e WN |
268 | int i; |
269 | ||
55cffde2 WN |
270 | if (!prog) |
271 | return; | |
272 | ||
b580563e WN |
273 | /* |
274 | * If the object is opened but the program was never loaded, | |
275 | * it is possible that prog->instances.nr == -1. | |
276 | */ | |
277 | if (prog->instances.nr > 0) { | |
278 | for (i = 0; i < prog->instances.nr; i++) | |
279 | zclose(prog->instances.fds[i]); | |
280 | } else if (prog->instances.nr != -1) { | |
281 | pr_warning("Internal error: instances.nr is %d\n", | |
282 | prog->instances.nr); | |
283 | } | |
284 | ||
285 | prog->instances.nr = -1; | |
286 | zfree(&prog->instances.fds); | |
55cffde2 WN |
287 | } |
288 | ||
a5b8bd47 WN |
289 | static void bpf_program__exit(struct bpf_program *prog) |
290 | { | |
291 | if (!prog) | |
292 | return; | |
293 | ||
aa9b1ac3 WN |
294 | if (prog->clear_priv) |
295 | prog->clear_priv(prog, prog->priv); | |
296 | ||
297 | prog->priv = NULL; | |
298 | prog->clear_priv = NULL; | |
299 | ||
55cffde2 | 300 | bpf_program__unload(prog); |
88cda1c9 | 301 | zfree(&prog->name); |
a5b8bd47 WN |
302 | zfree(&prog->section_name); |
303 | zfree(&prog->insns); | |
34090915 WN |
304 | zfree(&prog->reloc_desc); |
305 | ||
306 | prog->nr_reloc = 0; | |
a5b8bd47 WN |
307 | prog->insns_cnt = 0; |
308 | prog->idx = -1; | |
309 | } | |
310 | ||
311 | static int | |
88cda1c9 MKL |
312 | bpf_program__init(void *data, size_t size, char *section_name, int idx, |
313 | struct bpf_program *prog) | |
a5b8bd47 WN |
314 | { |
315 | if (size < sizeof(struct bpf_insn)) { | |
88cda1c9 | 316 | pr_warning("corrupted section '%s'\n", section_name); |
a5b8bd47 WN |
317 | return -EINVAL; |
318 | } | |
319 | ||
320 | bzero(prog, sizeof(*prog)); | |
321 | ||
88cda1c9 | 322 | prog->section_name = strdup(section_name); |
a5b8bd47 | 323 | if (!prog->section_name) { |
077c066a JDB |
324 | pr_warning("failed to alloc name for prog under section(%d) %s\n", |
325 | idx, section_name); | |
a5b8bd47 WN |
326 | goto errout; |
327 | } | |
328 | ||
329 | prog->insns = malloc(size); | |
330 | if (!prog->insns) { | |
88cda1c9 MKL |
331 | pr_warning("failed to alloc insns for prog under section %s\n", |
332 | section_name); | |
a5b8bd47 WN |
333 | goto errout; |
334 | } | |
335 | prog->insns_cnt = size / sizeof(struct bpf_insn); | |
336 | memcpy(prog->insns, data, | |
337 | prog->insns_cnt * sizeof(struct bpf_insn)); | |
338 | prog->idx = idx; | |
b580563e WN |
339 | prog->instances.fds = NULL; |
340 | prog->instances.nr = -1; | |
5f44e4c8 | 341 | prog->type = BPF_PROG_TYPE_KPROBE; |
a5b8bd47 WN |
342 | |
343 | return 0; | |
344 | errout: | |
345 | bpf_program__exit(prog); | |
346 | return -ENOMEM; | |
347 | } | |
348 | ||
349 | static int | |
350 | bpf_object__add_program(struct bpf_object *obj, void *data, size_t size, | |
88cda1c9 | 351 | char *section_name, int idx) |
a5b8bd47 WN |
352 | { |
353 | struct bpf_program prog, *progs; | |
354 | int nr_progs, err; | |
355 | ||
88cda1c9 | 356 | err = bpf_program__init(data, size, section_name, idx, &prog); |
a5b8bd47 WN |
357 | if (err) |
358 | return err; | |
359 | ||
360 | progs = obj->programs; | |
361 | nr_progs = obj->nr_programs; | |
362 | ||
363 | progs = realloc(progs, sizeof(progs[0]) * (nr_progs + 1)); | |
364 | if (!progs) { | |
365 | /* | |
366 | * In this case the original obj->programs | |
367 | * is still valid, so don't need special treat for | |
368 | * bpf_close_object(). | |
369 | */ | |
88cda1c9 MKL |
370 | pr_warning("failed to alloc a new program under section '%s'\n", |
371 | section_name); | |
a5b8bd47 WN |
372 | bpf_program__exit(&prog); |
373 | return -ENOMEM; | |
374 | } | |
375 | ||
376 | pr_debug("found program %s\n", prog.section_name); | |
377 | obj->programs = progs; | |
378 | obj->nr_programs = nr_progs + 1; | |
aa9b1ac3 | 379 | prog.obj = obj; |
a5b8bd47 WN |
380 | progs[nr_progs] = prog; |
381 | return 0; | |
382 | } | |
383 | ||
88cda1c9 MKL |
384 | static int |
385 | bpf_object__init_prog_names(struct bpf_object *obj) | |
386 | { | |
387 | Elf_Data *symbols = obj->efile.symbols; | |
388 | struct bpf_program *prog; | |
389 | size_t pi, si; | |
390 | ||
391 | for (pi = 0; pi < obj->nr_programs; pi++) { | |
48cca7e4 | 392 | const char *name = NULL; |
88cda1c9 MKL |
393 | |
394 | prog = &obj->programs[pi]; | |
48cca7e4 AS |
395 | if (prog->idx == obj->efile.text_shndx) { |
396 | name = ".text"; | |
397 | goto skip_search; | |
398 | } | |
88cda1c9 MKL |
399 | |
400 | for (si = 0; si < symbols->d_size / sizeof(GElf_Sym) && !name; | |
401 | si++) { | |
402 | GElf_Sym sym; | |
403 | ||
404 | if (!gelf_getsym(symbols, si, &sym)) | |
405 | continue; | |
406 | if (sym.st_shndx != prog->idx) | |
407 | continue; | |
fe4d44b2 RG |
408 | if (GELF_ST_BIND(sym.st_info) != STB_GLOBAL) |
409 | continue; | |
88cda1c9 MKL |
410 | |
411 | name = elf_strptr(obj->efile.elf, | |
412 | obj->efile.strtabidx, | |
413 | sym.st_name); | |
414 | if (!name) { | |
415 | pr_warning("failed to get sym name string for prog %s\n", | |
416 | prog->section_name); | |
417 | return -LIBBPF_ERRNO__LIBELF; | |
418 | } | |
419 | } | |
420 | ||
421 | if (!name) { | |
422 | pr_warning("failed to find sym for prog %s\n", | |
423 | prog->section_name); | |
424 | return -EINVAL; | |
425 | } | |
48cca7e4 | 426 | skip_search: |
88cda1c9 MKL |
427 | prog->name = strdup(name); |
428 | if (!prog->name) { | |
429 | pr_warning("failed to allocate memory for prog sym %s\n", | |
430 | name); | |
431 | return -ENOMEM; | |
432 | } | |
433 | } | |
434 | ||
435 | return 0; | |
436 | } | |
437 | ||
6c956392 WN |
438 | static struct bpf_object *bpf_object__new(const char *path, |
439 | void *obj_buf, | |
440 | size_t obj_buf_sz) | |
1a5e3fb1 WN |
441 | { |
442 | struct bpf_object *obj; | |
443 | ||
444 | obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); | |
445 | if (!obj) { | |
446 | pr_warning("alloc memory failed for %s\n", path); | |
6371ca3b | 447 | return ERR_PTR(-ENOMEM); |
1a5e3fb1 WN |
448 | } |
449 | ||
450 | strcpy(obj->path, path); | |
451 | obj->efile.fd = -1; | |
6c956392 WN |
452 | |
453 | /* | |
454 | * Caller of this function should also calls | |
455 | * bpf_object__elf_finish() after data collection to return | |
456 | * obj_buf to user. If not, we should duplicate the buffer to | |
457 | * avoid user freeing them before elf finish. | |
458 | */ | |
459 | obj->efile.obj_buf = obj_buf; | |
460 | obj->efile.obj_buf_sz = obj_buf_sz; | |
666810e8 | 461 | obj->efile.maps_shndx = -1; |
6c956392 | 462 | |
52d3352e | 463 | obj->loaded = false; |
9a208eff WN |
464 | |
465 | INIT_LIST_HEAD(&obj->list); | |
466 | list_add(&obj->list, &bpf_objects_list); | |
1a5e3fb1 WN |
467 | return obj; |
468 | } | |
469 | ||
470 | static void bpf_object__elf_finish(struct bpf_object *obj) | |
471 | { | |
472 | if (!obj_elf_valid(obj)) | |
473 | return; | |
474 | ||
475 | if (obj->efile.elf) { | |
476 | elf_end(obj->efile.elf); | |
477 | obj->efile.elf = NULL; | |
478 | } | |
bec7d68c | 479 | obj->efile.symbols = NULL; |
b62f06e8 WN |
480 | |
481 | zfree(&obj->efile.reloc); | |
482 | obj->efile.nr_reloc = 0; | |
1a5e3fb1 | 483 | zclose(obj->efile.fd); |
6c956392 WN |
484 | obj->efile.obj_buf = NULL; |
485 | obj->efile.obj_buf_sz = 0; | |
1a5e3fb1 WN |
486 | } |
487 | ||
488 | static int bpf_object__elf_init(struct bpf_object *obj) | |
489 | { | |
490 | int err = 0; | |
491 | GElf_Ehdr *ep; | |
492 | ||
493 | if (obj_elf_valid(obj)) { | |
494 | pr_warning("elf init: internal error\n"); | |
6371ca3b | 495 | return -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
496 | } |
497 | ||
6c956392 WN |
498 | if (obj->efile.obj_buf_sz > 0) { |
499 | /* | |
500 | * obj_buf should have been validated by | |
501 | * bpf_object__open_buffer(). | |
502 | */ | |
503 | obj->efile.elf = elf_memory(obj->efile.obj_buf, | |
504 | obj->efile.obj_buf_sz); | |
505 | } else { | |
506 | obj->efile.fd = open(obj->path, O_RDONLY); | |
507 | if (obj->efile.fd < 0) { | |
508 | pr_warning("failed to open %s: %s\n", obj->path, | |
509 | strerror(errno)); | |
510 | return -errno; | |
511 | } | |
512 | ||
513 | obj->efile.elf = elf_begin(obj->efile.fd, | |
514 | LIBBPF_ELF_C_READ_MMAP, | |
515 | NULL); | |
1a5e3fb1 WN |
516 | } |
517 | ||
1a5e3fb1 WN |
518 | if (!obj->efile.elf) { |
519 | pr_warning("failed to open %s as ELF file\n", | |
520 | obj->path); | |
6371ca3b | 521 | err = -LIBBPF_ERRNO__LIBELF; |
1a5e3fb1 WN |
522 | goto errout; |
523 | } | |
524 | ||
525 | if (!gelf_getehdr(obj->efile.elf, &obj->efile.ehdr)) { | |
526 | pr_warning("failed to get EHDR from %s\n", | |
527 | obj->path); | |
6371ca3b | 528 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
529 | goto errout; |
530 | } | |
531 | ep = &obj->efile.ehdr; | |
532 | ||
9b16137a WN |
533 | /* Old LLVM set e_machine to EM_NONE */ |
534 | if ((ep->e_type != ET_REL) || (ep->e_machine && (ep->e_machine != EM_BPF))) { | |
1a5e3fb1 WN |
535 | pr_warning("%s is not an eBPF object file\n", |
536 | obj->path); | |
6371ca3b | 537 | err = -LIBBPF_ERRNO__FORMAT; |
1a5e3fb1 WN |
538 | goto errout; |
539 | } | |
540 | ||
541 | return 0; | |
542 | errout: | |
543 | bpf_object__elf_finish(obj); | |
544 | return err; | |
545 | } | |
546 | ||
cc4228d5 WN |
547 | static int |
548 | bpf_object__check_endianness(struct bpf_object *obj) | |
549 | { | |
550 | static unsigned int const endian = 1; | |
551 | ||
552 | switch (obj->efile.ehdr.e_ident[EI_DATA]) { | |
553 | case ELFDATA2LSB: | |
554 | /* We are big endian, BPF obj is little endian. */ | |
555 | if (*(unsigned char const *)&endian != 1) | |
556 | goto mismatch; | |
557 | break; | |
558 | ||
559 | case ELFDATA2MSB: | |
560 | /* We are little endian, BPF obj is big endian. */ | |
561 | if (*(unsigned char const *)&endian != 0) | |
562 | goto mismatch; | |
563 | break; | |
564 | default: | |
6371ca3b | 565 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
566 | } |
567 | ||
568 | return 0; | |
569 | ||
570 | mismatch: | |
571 | pr_warning("Error: endianness mismatch.\n"); | |
6371ca3b | 572 | return -LIBBPF_ERRNO__ENDIAN; |
cc4228d5 WN |
573 | } |
574 | ||
cb1e5e96 WN |
575 | static int |
576 | bpf_object__init_license(struct bpf_object *obj, | |
577 | void *data, size_t size) | |
578 | { | |
579 | memcpy(obj->license, data, | |
580 | min(size, sizeof(obj->license) - 1)); | |
581 | pr_debug("license of %s is %s\n", obj->path, obj->license); | |
582 | return 0; | |
583 | } | |
584 | ||
585 | static int | |
586 | bpf_object__init_kversion(struct bpf_object *obj, | |
587 | void *data, size_t size) | |
588 | { | |
589 | u32 kver; | |
590 | ||
591 | if (size != sizeof(kver)) { | |
592 | pr_warning("invalid kver section in %s\n", obj->path); | |
6371ca3b | 593 | return -LIBBPF_ERRNO__FORMAT; |
cb1e5e96 WN |
594 | } |
595 | memcpy(&kver, data, sizeof(kver)); | |
596 | obj->kern_version = kver; | |
597 | pr_debug("kernel version of %s is %x\n", obj->path, | |
598 | obj->kern_version); | |
599 | return 0; | |
600 | } | |
601 | ||
4708bbda EL |
602 | static int compare_bpf_map(const void *_a, const void *_b) |
603 | { | |
604 | const struct bpf_map *a = _a; | |
605 | const struct bpf_map *b = _b; | |
9d759a9b | 606 | |
4708bbda | 607 | return a->offset - b->offset; |
0b3d1efa WN |
608 | } |
609 | ||
973170e6 | 610 | static int |
4708bbda | 611 | bpf_object__init_maps(struct bpf_object *obj) |
561bbcca | 612 | { |
b13c5c14 | 613 | int i, map_idx, map_def_sz, nr_maps = 0; |
4708bbda EL |
614 | Elf_Scn *scn; |
615 | Elf_Data *data; | |
561bbcca WN |
616 | Elf_Data *symbols = obj->efile.symbols; |
617 | ||
4708bbda EL |
618 | if (obj->efile.maps_shndx < 0) |
619 | return -EINVAL; | |
620 | if (!symbols) | |
621 | return -EINVAL; | |
622 | ||
623 | scn = elf_getscn(obj->efile.elf, obj->efile.maps_shndx); | |
624 | if (scn) | |
625 | data = elf_getdata(scn, NULL); | |
626 | if (!scn || !data) { | |
627 | pr_warning("failed to get Elf_Data from map section %d\n", | |
628 | obj->efile.maps_shndx); | |
973170e6 | 629 | return -EINVAL; |
4708bbda | 630 | } |
561bbcca | 631 | |
4708bbda EL |
632 | /* |
633 | * Count number of maps. Each map has a name. | |
634 | * Array of maps is not supported: only the first element is | |
635 | * considered. | |
636 | * | |
637 | * TODO: Detect array of map and report error. | |
638 | */ | |
561bbcca WN |
639 | for (i = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { |
640 | GElf_Sym sym; | |
4708bbda EL |
641 | |
642 | if (!gelf_getsym(symbols, i, &sym)) | |
643 | continue; | |
644 | if (sym.st_shndx != obj->efile.maps_shndx) | |
645 | continue; | |
646 | nr_maps++; | |
647 | } | |
648 | ||
649 | /* Alloc obj->maps and fill nr_maps. */ | |
650 | pr_debug("maps in %s: %d maps in %zd bytes\n", obj->path, | |
651 | nr_maps, data->d_size); | |
652 | ||
653 | if (!nr_maps) | |
654 | return 0; | |
655 | ||
b13c5c14 CG |
656 | /* Assume equally sized map definitions */ |
657 | map_def_sz = data->d_size / nr_maps; | |
658 | if (!data->d_size || (data->d_size % nr_maps) != 0) { | |
659 | pr_warning("unable to determine map definition size " | |
660 | "section %s, %d maps in %zd bytes\n", | |
661 | obj->path, nr_maps, data->d_size); | |
662 | return -EINVAL; | |
663 | } | |
664 | ||
4708bbda EL |
665 | obj->maps = calloc(nr_maps, sizeof(obj->maps[0])); |
666 | if (!obj->maps) { | |
667 | pr_warning("alloc maps for object failed\n"); | |
668 | return -ENOMEM; | |
669 | } | |
670 | obj->nr_maps = nr_maps; | |
671 | ||
672 | /* | |
673 | * fill all fd with -1 so won't close incorrect | |
674 | * fd (fd=0 is stdin) when failure (zclose won't close | |
675 | * negative fd)). | |
676 | */ | |
677 | for (i = 0; i < nr_maps; i++) | |
678 | obj->maps[i].fd = -1; | |
679 | ||
680 | /* | |
681 | * Fill obj->maps using data in "maps" section. | |
682 | */ | |
683 | for (i = 0, map_idx = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { | |
684 | GElf_Sym sym; | |
561bbcca | 685 | const char *map_name; |
4708bbda | 686 | struct bpf_map_def *def; |
561bbcca WN |
687 | |
688 | if (!gelf_getsym(symbols, i, &sym)) | |
689 | continue; | |
666810e8 | 690 | if (sym.st_shndx != obj->efile.maps_shndx) |
561bbcca WN |
691 | continue; |
692 | ||
693 | map_name = elf_strptr(obj->efile.elf, | |
77ba9a5b | 694 | obj->efile.strtabidx, |
561bbcca | 695 | sym.st_name); |
4708bbda | 696 | obj->maps[map_idx].offset = sym.st_value; |
b13c5c14 | 697 | if (sym.st_value + map_def_sz > data->d_size) { |
4708bbda EL |
698 | pr_warning("corrupted maps section in %s: last map \"%s\" too small\n", |
699 | obj->path, map_name); | |
700 | return -EINVAL; | |
561bbcca | 701 | } |
4708bbda | 702 | |
561bbcca | 703 | obj->maps[map_idx].name = strdup(map_name); |
973170e6 WN |
704 | if (!obj->maps[map_idx].name) { |
705 | pr_warning("failed to alloc map name\n"); | |
706 | return -ENOMEM; | |
707 | } | |
4708bbda | 708 | pr_debug("map %d is \"%s\"\n", map_idx, |
561bbcca | 709 | obj->maps[map_idx].name); |
4708bbda | 710 | def = (struct bpf_map_def *)(data->d_buf + sym.st_value); |
b13c5c14 CG |
711 | /* |
712 | * If the definition of the map in the object file fits in | |
713 | * bpf_map_def, copy it. Any extra fields in our version | |
714 | * of bpf_map_def will default to zero as a result of the | |
715 | * calloc above. | |
716 | */ | |
717 | if (map_def_sz <= sizeof(struct bpf_map_def)) { | |
718 | memcpy(&obj->maps[map_idx].def, def, map_def_sz); | |
719 | } else { | |
720 | /* | |
721 | * Here the map structure being read is bigger than what | |
722 | * we expect, truncate if the excess bits are all zero. | |
723 | * If they are not zero, reject this map as | |
724 | * incompatible. | |
725 | */ | |
726 | char *b; | |
727 | for (b = ((char *)def) + sizeof(struct bpf_map_def); | |
728 | b < ((char *)def) + map_def_sz; b++) { | |
729 | if (*b != 0) { | |
730 | pr_warning("maps section in %s: \"%s\" " | |
731 | "has unrecognized, non-zero " | |
732 | "options\n", | |
733 | obj->path, map_name); | |
734 | return -EINVAL; | |
735 | } | |
736 | } | |
737 | memcpy(&obj->maps[map_idx].def, def, | |
738 | sizeof(struct bpf_map_def)); | |
739 | } | |
4708bbda | 740 | map_idx++; |
561bbcca | 741 | } |
4708bbda EL |
742 | |
743 | qsort(obj->maps, obj->nr_maps, sizeof(obj->maps[0]), compare_bpf_map); | |
b13c5c14 | 744 | return 0; |
561bbcca WN |
745 | } |
746 | ||
e3d91b0c JDB |
747 | static bool section_have_execinstr(struct bpf_object *obj, int idx) |
748 | { | |
749 | Elf_Scn *scn; | |
750 | GElf_Shdr sh; | |
751 | ||
752 | scn = elf_getscn(obj->efile.elf, idx); | |
753 | if (!scn) | |
754 | return false; | |
755 | ||
756 | if (gelf_getshdr(scn, &sh) != &sh) | |
757 | return false; | |
758 | ||
759 | if (sh.sh_flags & SHF_EXECINSTR) | |
760 | return true; | |
761 | ||
762 | return false; | |
763 | } | |
764 | ||
29603665 WN |
765 | static int bpf_object__elf_collect(struct bpf_object *obj) |
766 | { | |
767 | Elf *elf = obj->efile.elf; | |
768 | GElf_Ehdr *ep = &obj->efile.ehdr; | |
769 | Elf_Scn *scn = NULL; | |
666810e8 | 770 | int idx = 0, err = 0; |
29603665 WN |
771 | |
772 | /* Elf is corrupted/truncated, avoid calling elf_strptr. */ | |
773 | if (!elf_rawdata(elf_getscn(elf, ep->e_shstrndx), NULL)) { | |
774 | pr_warning("failed to get e_shstrndx from %s\n", | |
775 | obj->path); | |
6371ca3b | 776 | return -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
777 | } |
778 | ||
779 | while ((scn = elf_nextscn(elf, scn)) != NULL) { | |
780 | char *name; | |
781 | GElf_Shdr sh; | |
782 | Elf_Data *data; | |
783 | ||
784 | idx++; | |
785 | if (gelf_getshdr(scn, &sh) != &sh) { | |
077c066a JDB |
786 | pr_warning("failed to get section(%d) header from %s\n", |
787 | idx, obj->path); | |
6371ca3b | 788 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
789 | goto out; |
790 | } | |
791 | ||
792 | name = elf_strptr(elf, ep->e_shstrndx, sh.sh_name); | |
793 | if (!name) { | |
077c066a JDB |
794 | pr_warning("failed to get section(%d) name from %s\n", |
795 | idx, obj->path); | |
6371ca3b | 796 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
797 | goto out; |
798 | } | |
799 | ||
800 | data = elf_getdata(scn, 0); | |
801 | if (!data) { | |
077c066a JDB |
802 | pr_warning("failed to get section(%d) data from %s(%s)\n", |
803 | idx, name, obj->path); | |
6371ca3b | 804 | err = -LIBBPF_ERRNO__FORMAT; |
29603665 WN |
805 | goto out; |
806 | } | |
077c066a JDB |
807 | pr_debug("section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", |
808 | idx, name, (unsigned long)data->d_size, | |
29603665 WN |
809 | (int)sh.sh_link, (unsigned long)sh.sh_flags, |
810 | (int)sh.sh_type); | |
cb1e5e96 WN |
811 | |
812 | if (strcmp(name, "license") == 0) | |
813 | err = bpf_object__init_license(obj, | |
814 | data->d_buf, | |
815 | data->d_size); | |
816 | else if (strcmp(name, "version") == 0) | |
817 | err = bpf_object__init_kversion(obj, | |
818 | data->d_buf, | |
819 | data->d_size); | |
4708bbda | 820 | else if (strcmp(name, "maps") == 0) |
666810e8 | 821 | obj->efile.maps_shndx = idx; |
4708bbda | 822 | else if (sh.sh_type == SHT_SYMTAB) { |
bec7d68c WN |
823 | if (obj->efile.symbols) { |
824 | pr_warning("bpf: multiple SYMTAB in %s\n", | |
825 | obj->path); | |
6371ca3b | 826 | err = -LIBBPF_ERRNO__FORMAT; |
77ba9a5b | 827 | } else { |
bec7d68c | 828 | obj->efile.symbols = data; |
77ba9a5b WN |
829 | obj->efile.strtabidx = sh.sh_link; |
830 | } | |
a5b8bd47 WN |
831 | } else if ((sh.sh_type == SHT_PROGBITS) && |
832 | (sh.sh_flags & SHF_EXECINSTR) && | |
833 | (data->d_size > 0)) { | |
48cca7e4 AS |
834 | if (strcmp(name, ".text") == 0) |
835 | obj->efile.text_shndx = idx; | |
a5b8bd47 WN |
836 | err = bpf_object__add_program(obj, data->d_buf, |
837 | data->d_size, name, idx); | |
838 | if (err) { | |
6371ca3b WN |
839 | char errmsg[STRERR_BUFSIZE]; |
840 | ||
a5b8bd47 WN |
841 | strerror_r(-err, errmsg, sizeof(errmsg)); |
842 | pr_warning("failed to alloc program %s (%s): %s", | |
843 | name, obj->path, errmsg); | |
844 | } | |
b62f06e8 WN |
845 | } else if (sh.sh_type == SHT_REL) { |
846 | void *reloc = obj->efile.reloc; | |
847 | int nr_reloc = obj->efile.nr_reloc + 1; | |
e3d91b0c JDB |
848 | int sec = sh.sh_info; /* points to other section */ |
849 | ||
850 | /* Only do relo for section with exec instructions */ | |
851 | if (!section_have_execinstr(obj, sec)) { | |
852 | pr_debug("skip relo %s(%d) for section(%d)\n", | |
853 | name, idx, sec); | |
854 | continue; | |
855 | } | |
b62f06e8 WN |
856 | |
857 | reloc = realloc(reloc, | |
858 | sizeof(*obj->efile.reloc) * nr_reloc); | |
859 | if (!reloc) { | |
860 | pr_warning("realloc failed\n"); | |
861 | err = -ENOMEM; | |
862 | } else { | |
863 | int n = nr_reloc - 1; | |
864 | ||
865 | obj->efile.reloc = reloc; | |
866 | obj->efile.nr_reloc = nr_reloc; | |
867 | ||
868 | obj->efile.reloc[n].shdr = sh; | |
869 | obj->efile.reloc[n].data = data; | |
870 | } | |
077c066a JDB |
871 | } else { |
872 | pr_debug("skip section(%d) %s\n", idx, name); | |
bec7d68c | 873 | } |
cb1e5e96 WN |
874 | if (err) |
875 | goto out; | |
29603665 | 876 | } |
561bbcca | 877 | |
77ba9a5b WN |
878 | if (!obj->efile.strtabidx || obj->efile.strtabidx >= idx) { |
879 | pr_warning("Corrupted ELF file: index of strtab invalid\n"); | |
880 | return LIBBPF_ERRNO__FORMAT; | |
881 | } | |
88cda1c9 | 882 | if (obj->efile.maps_shndx >= 0) { |
4708bbda | 883 | err = bpf_object__init_maps(obj); |
88cda1c9 MKL |
884 | if (err) |
885 | goto out; | |
886 | } | |
887 | err = bpf_object__init_prog_names(obj); | |
29603665 WN |
888 | out: |
889 | return err; | |
890 | } | |
891 | ||
34090915 WN |
892 | static struct bpf_program * |
893 | bpf_object__find_prog_by_idx(struct bpf_object *obj, int idx) | |
894 | { | |
895 | struct bpf_program *prog; | |
896 | size_t i; | |
897 | ||
898 | for (i = 0; i < obj->nr_programs; i++) { | |
899 | prog = &obj->programs[i]; | |
900 | if (prog->idx == idx) | |
901 | return prog; | |
902 | } | |
903 | return NULL; | |
904 | } | |
905 | ||
906 | static int | |
48cca7e4 AS |
907 | bpf_program__collect_reloc(struct bpf_program *prog, GElf_Shdr *shdr, |
908 | Elf_Data *data, struct bpf_object *obj) | |
34090915 | 909 | { |
48cca7e4 AS |
910 | Elf_Data *symbols = obj->efile.symbols; |
911 | int text_shndx = obj->efile.text_shndx; | |
912 | int maps_shndx = obj->efile.maps_shndx; | |
913 | struct bpf_map *maps = obj->maps; | |
914 | size_t nr_maps = obj->nr_maps; | |
34090915 WN |
915 | int i, nrels; |
916 | ||
917 | pr_debug("collecting relocating info for: '%s'\n", | |
918 | prog->section_name); | |
919 | nrels = shdr->sh_size / shdr->sh_entsize; | |
920 | ||
921 | prog->reloc_desc = malloc(sizeof(*prog->reloc_desc) * nrels); | |
922 | if (!prog->reloc_desc) { | |
923 | pr_warning("failed to alloc memory in relocation\n"); | |
924 | return -ENOMEM; | |
925 | } | |
926 | prog->nr_reloc = nrels; | |
927 | ||
928 | for (i = 0; i < nrels; i++) { | |
929 | GElf_Sym sym; | |
930 | GElf_Rel rel; | |
931 | unsigned int insn_idx; | |
932 | struct bpf_insn *insns = prog->insns; | |
933 | size_t map_idx; | |
934 | ||
935 | if (!gelf_getrel(data, i, &rel)) { | |
936 | pr_warning("relocation: failed to get %d reloc\n", i); | |
6371ca3b | 937 | return -LIBBPF_ERRNO__FORMAT; |
34090915 WN |
938 | } |
939 | ||
34090915 WN |
940 | if (!gelf_getsym(symbols, |
941 | GELF_R_SYM(rel.r_info), | |
942 | &sym)) { | |
943 | pr_warning("relocation: symbol %"PRIx64" not found\n", | |
944 | GELF_R_SYM(rel.r_info)); | |
6371ca3b | 945 | return -LIBBPF_ERRNO__FORMAT; |
34090915 | 946 | } |
7d9890ef DM |
947 | pr_debug("relo for %lld value %lld name %d\n", |
948 | (long long) (rel.r_info >> 32), | |
949 | (long long) sym.st_value, sym.st_name); | |
34090915 | 950 | |
48cca7e4 | 951 | if (sym.st_shndx != maps_shndx && sym.st_shndx != text_shndx) { |
666810e8 WN |
952 | pr_warning("Program '%s' contains non-map related relo data pointing to section %u\n", |
953 | prog->section_name, sym.st_shndx); | |
954 | return -LIBBPF_ERRNO__RELOC; | |
955 | } | |
956 | ||
957 | insn_idx = rel.r_offset / sizeof(struct bpf_insn); | |
958 | pr_debug("relocation: insn_idx=%u\n", insn_idx); | |
959 | ||
48cca7e4 AS |
960 | if (insns[insn_idx].code == (BPF_JMP | BPF_CALL)) { |
961 | if (insns[insn_idx].src_reg != BPF_PSEUDO_CALL) { | |
962 | pr_warning("incorrect bpf_call opcode\n"); | |
963 | return -LIBBPF_ERRNO__RELOC; | |
964 | } | |
965 | prog->reloc_desc[i].type = RELO_CALL; | |
966 | prog->reloc_desc[i].insn_idx = insn_idx; | |
967 | prog->reloc_desc[i].text_off = sym.st_value; | |
968 | continue; | |
969 | } | |
970 | ||
34090915 WN |
971 | if (insns[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { |
972 | pr_warning("bpf: relocation: invalid relo for insns[%d].code 0x%x\n", | |
973 | insn_idx, insns[insn_idx].code); | |
6371ca3b | 974 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
975 | } |
976 | ||
94e5adec JS |
977 | /* TODO: 'maps' is sorted. We can use bsearch to make it faster. */ |
978 | for (map_idx = 0; map_idx < nr_maps; map_idx++) { | |
979 | if (maps[map_idx].offset == sym.st_value) { | |
980 | pr_debug("relocation: find map %zd (%s) for insn %u\n", | |
981 | map_idx, maps[map_idx].name, insn_idx); | |
982 | break; | |
983 | } | |
984 | } | |
985 | ||
34090915 WN |
986 | if (map_idx >= nr_maps) { |
987 | pr_warning("bpf relocation: map_idx %d large than %d\n", | |
988 | (int)map_idx, (int)nr_maps - 1); | |
6371ca3b | 989 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
990 | } |
991 | ||
48cca7e4 | 992 | prog->reloc_desc[i].type = RELO_LD64; |
34090915 WN |
993 | prog->reloc_desc[i].insn_idx = insn_idx; |
994 | prog->reloc_desc[i].map_idx = map_idx; | |
995 | } | |
996 | return 0; | |
997 | } | |
998 | ||
52d3352e WN |
999 | static int |
1000 | bpf_object__create_maps(struct bpf_object *obj) | |
1001 | { | |
1002 | unsigned int i; | |
52d3352e | 1003 | |
9d759a9b WN |
1004 | for (i = 0; i < obj->nr_maps; i++) { |
1005 | struct bpf_map_def *def = &obj->maps[i].def; | |
1006 | int *pfd = &obj->maps[i].fd; | |
52d3352e | 1007 | |
88cda1c9 MKL |
1008 | *pfd = bpf_create_map_name(def->type, |
1009 | obj->maps[i].name, | |
1010 | def->key_size, | |
1011 | def->value_size, | |
1012 | def->max_entries, | |
fe9b5f77 | 1013 | def->map_flags); |
52d3352e WN |
1014 | if (*pfd < 0) { |
1015 | size_t j; | |
1016 | int err = *pfd; | |
1017 | ||
49bf4b36 EL |
1018 | pr_warning("failed to create map (name: '%s'): %s\n", |
1019 | obj->maps[i].name, | |
52d3352e WN |
1020 | strerror(errno)); |
1021 | for (j = 0; j < i; j++) | |
9d759a9b | 1022 | zclose(obj->maps[j].fd); |
52d3352e WN |
1023 | return err; |
1024 | } | |
4708bbda | 1025 | pr_debug("create map %s: fd=%d\n", obj->maps[i].name, *pfd); |
52d3352e WN |
1026 | } |
1027 | ||
52d3352e WN |
1028 | return 0; |
1029 | } | |
1030 | ||
48cca7e4 AS |
1031 | static int |
1032 | bpf_program__reloc_text(struct bpf_program *prog, struct bpf_object *obj, | |
1033 | struct reloc_desc *relo) | |
1034 | { | |
1035 | struct bpf_insn *insn, *new_insn; | |
1036 | struct bpf_program *text; | |
1037 | size_t new_cnt; | |
1038 | ||
1039 | if (relo->type != RELO_CALL) | |
1040 | return -LIBBPF_ERRNO__RELOC; | |
1041 | ||
1042 | if (prog->idx == obj->efile.text_shndx) { | |
1043 | pr_warning("relo in .text insn %d into off %d\n", | |
1044 | relo->insn_idx, relo->text_off); | |
1045 | return -LIBBPF_ERRNO__RELOC; | |
1046 | } | |
1047 | ||
1048 | if (prog->main_prog_cnt == 0) { | |
1049 | text = bpf_object__find_prog_by_idx(obj, obj->efile.text_shndx); | |
1050 | if (!text) { | |
1051 | pr_warning("no .text section found yet relo into text exist\n"); | |
1052 | return -LIBBPF_ERRNO__RELOC; | |
1053 | } | |
1054 | new_cnt = prog->insns_cnt + text->insns_cnt; | |
1055 | new_insn = realloc(prog->insns, new_cnt * sizeof(*insn)); | |
1056 | if (!new_insn) { | |
1057 | pr_warning("oom in prog realloc\n"); | |
1058 | return -ENOMEM; | |
1059 | } | |
1060 | memcpy(new_insn + prog->insns_cnt, text->insns, | |
1061 | text->insns_cnt * sizeof(*insn)); | |
1062 | prog->insns = new_insn; | |
1063 | prog->main_prog_cnt = prog->insns_cnt; | |
1064 | prog->insns_cnt = new_cnt; | |
b1a2ce82 JC |
1065 | pr_debug("added %zd insn from %s to prog %s\n", |
1066 | text->insns_cnt, text->section_name, | |
1067 | prog->section_name); | |
48cca7e4 AS |
1068 | } |
1069 | insn = &prog->insns[relo->insn_idx]; | |
1070 | insn->imm += prog->main_prog_cnt - relo->insn_idx; | |
48cca7e4 AS |
1071 | return 0; |
1072 | } | |
1073 | ||
8a47a6c5 | 1074 | static int |
9d759a9b | 1075 | bpf_program__relocate(struct bpf_program *prog, struct bpf_object *obj) |
8a47a6c5 | 1076 | { |
48cca7e4 | 1077 | int i, err; |
8a47a6c5 WN |
1078 | |
1079 | if (!prog || !prog->reloc_desc) | |
1080 | return 0; | |
1081 | ||
1082 | for (i = 0; i < prog->nr_reloc; i++) { | |
48cca7e4 AS |
1083 | if (prog->reloc_desc[i].type == RELO_LD64) { |
1084 | struct bpf_insn *insns = prog->insns; | |
1085 | int insn_idx, map_idx; | |
8a47a6c5 | 1086 | |
48cca7e4 AS |
1087 | insn_idx = prog->reloc_desc[i].insn_idx; |
1088 | map_idx = prog->reloc_desc[i].map_idx; | |
8a47a6c5 | 1089 | |
48cca7e4 AS |
1090 | if (insn_idx >= (int)prog->insns_cnt) { |
1091 | pr_warning("relocation out of range: '%s'\n", | |
1092 | prog->section_name); | |
1093 | return -LIBBPF_ERRNO__RELOC; | |
1094 | } | |
1095 | insns[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; | |
1096 | insns[insn_idx].imm = obj->maps[map_idx].fd; | |
1097 | } else { | |
1098 | err = bpf_program__reloc_text(prog, obj, | |
1099 | &prog->reloc_desc[i]); | |
1100 | if (err) | |
1101 | return err; | |
8a47a6c5 | 1102 | } |
8a47a6c5 WN |
1103 | } |
1104 | ||
1105 | zfree(&prog->reloc_desc); | |
1106 | prog->nr_reloc = 0; | |
1107 | return 0; | |
1108 | } | |
1109 | ||
1110 | ||
1111 | static int | |
1112 | bpf_object__relocate(struct bpf_object *obj) | |
1113 | { | |
1114 | struct bpf_program *prog; | |
1115 | size_t i; | |
1116 | int err; | |
1117 | ||
1118 | for (i = 0; i < obj->nr_programs; i++) { | |
1119 | prog = &obj->programs[i]; | |
1120 | ||
9d759a9b | 1121 | err = bpf_program__relocate(prog, obj); |
8a47a6c5 WN |
1122 | if (err) { |
1123 | pr_warning("failed to relocate '%s'\n", | |
1124 | prog->section_name); | |
1125 | return err; | |
1126 | } | |
1127 | } | |
1128 | return 0; | |
1129 | } | |
1130 | ||
34090915 WN |
1131 | static int bpf_object__collect_reloc(struct bpf_object *obj) |
1132 | { | |
1133 | int i, err; | |
1134 | ||
1135 | if (!obj_elf_valid(obj)) { | |
1136 | pr_warning("Internal error: elf object is closed\n"); | |
6371ca3b | 1137 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
1138 | } |
1139 | ||
1140 | for (i = 0; i < obj->efile.nr_reloc; i++) { | |
1141 | GElf_Shdr *shdr = &obj->efile.reloc[i].shdr; | |
1142 | Elf_Data *data = obj->efile.reloc[i].data; | |
1143 | int idx = shdr->sh_info; | |
1144 | struct bpf_program *prog; | |
34090915 WN |
1145 | |
1146 | if (shdr->sh_type != SHT_REL) { | |
1147 | pr_warning("internal error at %d\n", __LINE__); | |
6371ca3b | 1148 | return -LIBBPF_ERRNO__INTERNAL; |
34090915 WN |
1149 | } |
1150 | ||
1151 | prog = bpf_object__find_prog_by_idx(obj, idx); | |
1152 | if (!prog) { | |
077c066a | 1153 | pr_warning("relocation failed: no section(%d)\n", idx); |
6371ca3b | 1154 | return -LIBBPF_ERRNO__RELOC; |
34090915 WN |
1155 | } |
1156 | ||
48cca7e4 | 1157 | err = bpf_program__collect_reloc(prog, |
34090915 | 1158 | shdr, data, |
48cca7e4 | 1159 | obj); |
34090915 | 1160 | if (err) |
6371ca3b | 1161 | return err; |
34090915 WN |
1162 | } |
1163 | return 0; | |
1164 | } | |
1165 | ||
55cffde2 | 1166 | static int |
d7be143b AI |
1167 | load_program(enum bpf_prog_type type, enum bpf_attach_type expected_attach_type, |
1168 | const char *name, struct bpf_insn *insns, int insns_cnt, | |
1169 | char *license, u32 kern_version, int *pfd) | |
55cffde2 | 1170 | { |
d7be143b | 1171 | struct bpf_load_program_attr load_attr; |
55cffde2 | 1172 | char *log_buf; |
d7be143b | 1173 | int ret; |
55cffde2 | 1174 | |
d7be143b AI |
1175 | memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); |
1176 | load_attr.prog_type = type; | |
1177 | load_attr.expected_attach_type = expected_attach_type; | |
1178 | load_attr.name = name; | |
1179 | load_attr.insns = insns; | |
1180 | load_attr.insns_cnt = insns_cnt; | |
1181 | load_attr.license = license; | |
1182 | load_attr.kern_version = kern_version; | |
1183 | ||
1184 | if (!load_attr.insns || !load_attr.insns_cnt) | |
55cffde2 WN |
1185 | return -EINVAL; |
1186 | ||
1187 | log_buf = malloc(BPF_LOG_BUF_SIZE); | |
1188 | if (!log_buf) | |
1189 | pr_warning("Alloc log buffer for bpf loader error, continue without log\n"); | |
1190 | ||
d7be143b | 1191 | ret = bpf_load_program_xattr(&load_attr, log_buf, BPF_LOG_BUF_SIZE); |
55cffde2 WN |
1192 | |
1193 | if (ret >= 0) { | |
1194 | *pfd = ret; | |
1195 | ret = 0; | |
1196 | goto out; | |
1197 | } | |
1198 | ||
6371ca3b | 1199 | ret = -LIBBPF_ERRNO__LOAD; |
55cffde2 WN |
1200 | pr_warning("load bpf program failed: %s\n", strerror(errno)); |
1201 | ||
6371ca3b WN |
1202 | if (log_buf && log_buf[0] != '\0') { |
1203 | ret = -LIBBPF_ERRNO__VERIFY; | |
55cffde2 WN |
1204 | pr_warning("-- BEGIN DUMP LOG ---\n"); |
1205 | pr_warning("\n%s\n", log_buf); | |
1206 | pr_warning("-- END LOG --\n"); | |
d7be143b AI |
1207 | } else if (load_attr.insns_cnt >= BPF_MAXINSNS) { |
1208 | pr_warning("Program too large (%zu insns), at most %d insns\n", | |
1209 | load_attr.insns_cnt, BPF_MAXINSNS); | |
705fa219 | 1210 | ret = -LIBBPF_ERRNO__PROG2BIG; |
6371ca3b | 1211 | } else { |
705fa219 | 1212 | /* Wrong program type? */ |
d7be143b | 1213 | if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { |
705fa219 WN |
1214 | int fd; |
1215 | ||
d7be143b AI |
1216 | load_attr.prog_type = BPF_PROG_TYPE_KPROBE; |
1217 | load_attr.expected_attach_type = 0; | |
1218 | fd = bpf_load_program_xattr(&load_attr, NULL, 0); | |
705fa219 WN |
1219 | if (fd >= 0) { |
1220 | close(fd); | |
1221 | ret = -LIBBPF_ERRNO__PROGTYPE; | |
1222 | goto out; | |
1223 | } | |
6371ca3b | 1224 | } |
705fa219 WN |
1225 | |
1226 | if (log_buf) | |
1227 | ret = -LIBBPF_ERRNO__KVER; | |
55cffde2 WN |
1228 | } |
1229 | ||
1230 | out: | |
1231 | free(log_buf); | |
1232 | return ret; | |
1233 | } | |
1234 | ||
1235 | static int | |
1236 | bpf_program__load(struct bpf_program *prog, | |
1237 | char *license, u32 kern_version) | |
1238 | { | |
b580563e | 1239 | int err = 0, fd, i; |
55cffde2 | 1240 | |
b580563e WN |
1241 | if (prog->instances.nr < 0 || !prog->instances.fds) { |
1242 | if (prog->preprocessor) { | |
1243 | pr_warning("Internal error: can't load program '%s'\n", | |
1244 | prog->section_name); | |
1245 | return -LIBBPF_ERRNO__INTERNAL; | |
1246 | } | |
55cffde2 | 1247 | |
b580563e WN |
1248 | prog->instances.fds = malloc(sizeof(int)); |
1249 | if (!prog->instances.fds) { | |
1250 | pr_warning("Not enough memory for BPF fds\n"); | |
1251 | return -ENOMEM; | |
1252 | } | |
1253 | prog->instances.nr = 1; | |
1254 | prog->instances.fds[0] = -1; | |
1255 | } | |
1256 | ||
1257 | if (!prog->preprocessor) { | |
1258 | if (prog->instances.nr != 1) { | |
1259 | pr_warning("Program '%s' is inconsistent: nr(%d) != 1\n", | |
1260 | prog->section_name, prog->instances.nr); | |
1261 | } | |
d7be143b AI |
1262 | err = load_program(prog->type, prog->expected_attach_type, |
1263 | prog->name, prog->insns, prog->insns_cnt, | |
1264 | license, kern_version, &fd); | |
b580563e WN |
1265 | if (!err) |
1266 | prog->instances.fds[0] = fd; | |
1267 | goto out; | |
1268 | } | |
1269 | ||
1270 | for (i = 0; i < prog->instances.nr; i++) { | |
1271 | struct bpf_prog_prep_result result; | |
1272 | bpf_program_prep_t preprocessor = prog->preprocessor; | |
1273 | ||
1274 | bzero(&result, sizeof(result)); | |
1275 | err = preprocessor(prog, i, prog->insns, | |
1276 | prog->insns_cnt, &result); | |
1277 | if (err) { | |
1278 | pr_warning("Preprocessing the %dth instance of program '%s' failed\n", | |
1279 | i, prog->section_name); | |
1280 | goto out; | |
1281 | } | |
1282 | ||
1283 | if (!result.new_insn_ptr || !result.new_insn_cnt) { | |
1284 | pr_debug("Skip loading the %dth instance of program '%s'\n", | |
1285 | i, prog->section_name); | |
1286 | prog->instances.fds[i] = -1; | |
1287 | if (result.pfd) | |
1288 | *result.pfd = -1; | |
1289 | continue; | |
1290 | } | |
1291 | ||
d7be143b AI |
1292 | err = load_program(prog->type, prog->expected_attach_type, |
1293 | prog->name, result.new_insn_ptr, | |
b580563e WN |
1294 | result.new_insn_cnt, |
1295 | license, kern_version, &fd); | |
1296 | ||
1297 | if (err) { | |
1298 | pr_warning("Loading the %dth instance of program '%s' failed\n", | |
1299 | i, prog->section_name); | |
1300 | goto out; | |
1301 | } | |
1302 | ||
1303 | if (result.pfd) | |
1304 | *result.pfd = fd; | |
1305 | prog->instances.fds[i] = fd; | |
1306 | } | |
1307 | out: | |
55cffde2 WN |
1308 | if (err) |
1309 | pr_warning("failed to load program '%s'\n", | |
1310 | prog->section_name); | |
1311 | zfree(&prog->insns); | |
1312 | prog->insns_cnt = 0; | |
1313 | return err; | |
1314 | } | |
1315 | ||
1316 | static int | |
1317 | bpf_object__load_progs(struct bpf_object *obj) | |
1318 | { | |
1319 | size_t i; | |
1320 | int err; | |
1321 | ||
1322 | for (i = 0; i < obj->nr_programs; i++) { | |
48cca7e4 AS |
1323 | if (obj->programs[i].idx == obj->efile.text_shndx) |
1324 | continue; | |
55cffde2 WN |
1325 | err = bpf_program__load(&obj->programs[i], |
1326 | obj->license, | |
1327 | obj->kern_version); | |
1328 | if (err) | |
1329 | return err; | |
1330 | } | |
1331 | return 0; | |
1332 | } | |
1333 | ||
cb1e5e96 WN |
1334 | static int bpf_object__validate(struct bpf_object *obj) |
1335 | { | |
1336 | if (obj->kern_version == 0) { | |
1337 | pr_warning("%s doesn't provide kernel version\n", | |
1338 | obj->path); | |
6371ca3b | 1339 | return -LIBBPF_ERRNO__KVERSION; |
cb1e5e96 WN |
1340 | } |
1341 | return 0; | |
1342 | } | |
1343 | ||
1a5e3fb1 | 1344 | static struct bpf_object * |
6c956392 | 1345 | __bpf_object__open(const char *path, void *obj_buf, size_t obj_buf_sz) |
1a5e3fb1 WN |
1346 | { |
1347 | struct bpf_object *obj; | |
6371ca3b | 1348 | int err; |
1a5e3fb1 WN |
1349 | |
1350 | if (elf_version(EV_CURRENT) == EV_NONE) { | |
1351 | pr_warning("failed to init libelf for %s\n", path); | |
6371ca3b | 1352 | return ERR_PTR(-LIBBPF_ERRNO__LIBELF); |
1a5e3fb1 WN |
1353 | } |
1354 | ||
6c956392 | 1355 | obj = bpf_object__new(path, obj_buf, obj_buf_sz); |
6371ca3b WN |
1356 | if (IS_ERR(obj)) |
1357 | return obj; | |
1a5e3fb1 | 1358 | |
6371ca3b WN |
1359 | CHECK_ERR(bpf_object__elf_init(obj), err, out); |
1360 | CHECK_ERR(bpf_object__check_endianness(obj), err, out); | |
1361 | CHECK_ERR(bpf_object__elf_collect(obj), err, out); | |
1362 | CHECK_ERR(bpf_object__collect_reloc(obj), err, out); | |
1363 | CHECK_ERR(bpf_object__validate(obj), err, out); | |
1a5e3fb1 WN |
1364 | |
1365 | bpf_object__elf_finish(obj); | |
1366 | return obj; | |
1367 | out: | |
1368 | bpf_object__close(obj); | |
6371ca3b | 1369 | return ERR_PTR(err); |
1a5e3fb1 WN |
1370 | } |
1371 | ||
1372 | struct bpf_object *bpf_object__open(const char *path) | |
1373 | { | |
1374 | /* param validation */ | |
1375 | if (!path) | |
1376 | return NULL; | |
1377 | ||
1378 | pr_debug("loading %s\n", path); | |
1379 | ||
6c956392 WN |
1380 | return __bpf_object__open(path, NULL, 0); |
1381 | } | |
1382 | ||
1383 | struct bpf_object *bpf_object__open_buffer(void *obj_buf, | |
acf860ae WN |
1384 | size_t obj_buf_sz, |
1385 | const char *name) | |
6c956392 | 1386 | { |
acf860ae WN |
1387 | char tmp_name[64]; |
1388 | ||
6c956392 WN |
1389 | /* param validation */ |
1390 | if (!obj_buf || obj_buf_sz <= 0) | |
1391 | return NULL; | |
1392 | ||
acf860ae WN |
1393 | if (!name) { |
1394 | snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", | |
1395 | (unsigned long)obj_buf, | |
1396 | (unsigned long)obj_buf_sz); | |
1397 | tmp_name[sizeof(tmp_name) - 1] = '\0'; | |
1398 | name = tmp_name; | |
1399 | } | |
1400 | pr_debug("loading object '%s' from buffer\n", | |
1401 | name); | |
6c956392 | 1402 | |
acf860ae | 1403 | return __bpf_object__open(name, obj_buf, obj_buf_sz); |
1a5e3fb1 WN |
1404 | } |
1405 | ||
52d3352e WN |
1406 | int bpf_object__unload(struct bpf_object *obj) |
1407 | { | |
1408 | size_t i; | |
1409 | ||
1410 | if (!obj) | |
1411 | return -EINVAL; | |
1412 | ||
9d759a9b WN |
1413 | for (i = 0; i < obj->nr_maps; i++) |
1414 | zclose(obj->maps[i].fd); | |
52d3352e | 1415 | |
55cffde2 WN |
1416 | for (i = 0; i < obj->nr_programs; i++) |
1417 | bpf_program__unload(&obj->programs[i]); | |
1418 | ||
52d3352e WN |
1419 | return 0; |
1420 | } | |
1421 | ||
1422 | int bpf_object__load(struct bpf_object *obj) | |
1423 | { | |
6371ca3b WN |
1424 | int err; |
1425 | ||
52d3352e WN |
1426 | if (!obj) |
1427 | return -EINVAL; | |
1428 | ||
1429 | if (obj->loaded) { | |
1430 | pr_warning("object should not be loaded twice\n"); | |
1431 | return -EINVAL; | |
1432 | } | |
1433 | ||
1434 | obj->loaded = true; | |
6371ca3b WN |
1435 | |
1436 | CHECK_ERR(bpf_object__create_maps(obj), err, out); | |
1437 | CHECK_ERR(bpf_object__relocate(obj), err, out); | |
1438 | CHECK_ERR(bpf_object__load_progs(obj), err, out); | |
52d3352e WN |
1439 | |
1440 | return 0; | |
1441 | out: | |
1442 | bpf_object__unload(obj); | |
1443 | pr_warning("failed to load object '%s'\n", obj->path); | |
6371ca3b | 1444 | return err; |
52d3352e WN |
1445 | } |
1446 | ||
f367540c JS |
1447 | static int check_path(const char *path) |
1448 | { | |
1449 | struct statfs st_fs; | |
1450 | char *dname, *dir; | |
1451 | int err = 0; | |
1452 | ||
1453 | if (path == NULL) | |
1454 | return -EINVAL; | |
1455 | ||
1456 | dname = strdup(path); | |
1457 | if (dname == NULL) | |
1458 | return -ENOMEM; | |
1459 | ||
1460 | dir = dirname(dname); | |
1461 | if (statfs(dir, &st_fs)) { | |
1462 | pr_warning("failed to statfs %s: %s\n", dir, strerror(errno)); | |
1463 | err = -errno; | |
1464 | } | |
1465 | free(dname); | |
1466 | ||
1467 | if (!err && st_fs.f_type != BPF_FS_MAGIC) { | |
1468 | pr_warning("specified path %s is not on BPF FS\n", path); | |
1469 | err = -EINVAL; | |
1470 | } | |
1471 | ||
1472 | return err; | |
1473 | } | |
1474 | ||
1475 | int bpf_program__pin_instance(struct bpf_program *prog, const char *path, | |
1476 | int instance) | |
1477 | { | |
1478 | int err; | |
1479 | ||
1480 | err = check_path(path); | |
1481 | if (err) | |
1482 | return err; | |
1483 | ||
1484 | if (prog == NULL) { | |
1485 | pr_warning("invalid program pointer\n"); | |
1486 | return -EINVAL; | |
1487 | } | |
1488 | ||
1489 | if (instance < 0 || instance >= prog->instances.nr) { | |
1490 | pr_warning("invalid prog instance %d of prog %s (max %d)\n", | |
1491 | instance, prog->section_name, prog->instances.nr); | |
1492 | return -EINVAL; | |
1493 | } | |
1494 | ||
1495 | if (bpf_obj_pin(prog->instances.fds[instance], path)) { | |
1496 | pr_warning("failed to pin program: %s\n", strerror(errno)); | |
1497 | return -errno; | |
1498 | } | |
1499 | pr_debug("pinned program '%s'\n", path); | |
1500 | ||
1501 | return 0; | |
1502 | } | |
1503 | ||
1504 | static int make_dir(const char *path) | |
1505 | { | |
1506 | int err = 0; | |
1507 | ||
1508 | if (mkdir(path, 0700) && errno != EEXIST) | |
1509 | err = -errno; | |
1510 | ||
1511 | if (err) | |
1512 | pr_warning("failed to mkdir %s: %s\n", path, strerror(-err)); | |
1513 | return err; | |
1514 | } | |
1515 | ||
1516 | int bpf_program__pin(struct bpf_program *prog, const char *path) | |
1517 | { | |
1518 | int i, err; | |
1519 | ||
1520 | err = check_path(path); | |
1521 | if (err) | |
1522 | return err; | |
1523 | ||
1524 | if (prog == NULL) { | |
1525 | pr_warning("invalid program pointer\n"); | |
1526 | return -EINVAL; | |
1527 | } | |
1528 | ||
1529 | if (prog->instances.nr <= 0) { | |
1530 | pr_warning("no instances of prog %s to pin\n", | |
1531 | prog->section_name); | |
1532 | return -EINVAL; | |
1533 | } | |
1534 | ||
1535 | err = make_dir(path); | |
1536 | if (err) | |
1537 | return err; | |
1538 | ||
1539 | for (i = 0; i < prog->instances.nr; i++) { | |
1540 | char buf[PATH_MAX]; | |
1541 | int len; | |
1542 | ||
1543 | len = snprintf(buf, PATH_MAX, "%s/%d", path, i); | |
1544 | if (len < 0) | |
1545 | return -EINVAL; | |
1546 | else if (len >= PATH_MAX) | |
1547 | return -ENAMETOOLONG; | |
1548 | ||
1549 | err = bpf_program__pin_instance(prog, buf, i); | |
1550 | if (err) | |
1551 | return err; | |
1552 | } | |
1553 | ||
1554 | return 0; | |
1555 | } | |
1556 | ||
b6989f35 JS |
1557 | int bpf_map__pin(struct bpf_map *map, const char *path) |
1558 | { | |
1559 | int err; | |
1560 | ||
1561 | err = check_path(path); | |
1562 | if (err) | |
1563 | return err; | |
1564 | ||
1565 | if (map == NULL) { | |
1566 | pr_warning("invalid map pointer\n"); | |
1567 | return -EINVAL; | |
1568 | } | |
1569 | ||
1570 | if (bpf_obj_pin(map->fd, path)) { | |
1571 | pr_warning("failed to pin map: %s\n", strerror(errno)); | |
1572 | return -errno; | |
1573 | } | |
1574 | ||
1575 | pr_debug("pinned map '%s'\n", path); | |
1576 | return 0; | |
1577 | } | |
1578 | ||
d5148d85 JS |
1579 | int bpf_object__pin(struct bpf_object *obj, const char *path) |
1580 | { | |
1581 | struct bpf_program *prog; | |
1582 | struct bpf_map *map; | |
1583 | int err; | |
1584 | ||
1585 | if (!obj) | |
1586 | return -ENOENT; | |
1587 | ||
1588 | if (!obj->loaded) { | |
1589 | pr_warning("object not yet loaded; load it first\n"); | |
1590 | return -ENOENT; | |
1591 | } | |
1592 | ||
1593 | err = make_dir(path); | |
1594 | if (err) | |
1595 | return err; | |
1596 | ||
1597 | bpf_map__for_each(map, obj) { | |
1598 | char buf[PATH_MAX]; | |
1599 | int len; | |
1600 | ||
1601 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
1602 | bpf_map__name(map)); | |
1603 | if (len < 0) | |
1604 | return -EINVAL; | |
1605 | else if (len >= PATH_MAX) | |
1606 | return -ENAMETOOLONG; | |
1607 | ||
1608 | err = bpf_map__pin(map, buf); | |
1609 | if (err) | |
1610 | return err; | |
1611 | } | |
1612 | ||
1613 | bpf_object__for_each_program(prog, obj) { | |
1614 | char buf[PATH_MAX]; | |
1615 | int len; | |
1616 | ||
1617 | len = snprintf(buf, PATH_MAX, "%s/%s", path, | |
1618 | prog->section_name); | |
1619 | if (len < 0) | |
1620 | return -EINVAL; | |
1621 | else if (len >= PATH_MAX) | |
1622 | return -ENAMETOOLONG; | |
1623 | ||
1624 | err = bpf_program__pin(prog, buf); | |
1625 | if (err) | |
1626 | return err; | |
1627 | } | |
1628 | ||
1629 | return 0; | |
1630 | } | |
1631 | ||
1a5e3fb1 WN |
1632 | void bpf_object__close(struct bpf_object *obj) |
1633 | { | |
a5b8bd47 WN |
1634 | size_t i; |
1635 | ||
1a5e3fb1 WN |
1636 | if (!obj) |
1637 | return; | |
1638 | ||
10931d24 WN |
1639 | if (obj->clear_priv) |
1640 | obj->clear_priv(obj, obj->priv); | |
1641 | ||
1a5e3fb1 | 1642 | bpf_object__elf_finish(obj); |
52d3352e | 1643 | bpf_object__unload(obj); |
1a5e3fb1 | 1644 | |
9d759a9b | 1645 | for (i = 0; i < obj->nr_maps; i++) { |
561bbcca | 1646 | zfree(&obj->maps[i].name); |
9d759a9b WN |
1647 | if (obj->maps[i].clear_priv) |
1648 | obj->maps[i].clear_priv(&obj->maps[i], | |
1649 | obj->maps[i].priv); | |
1650 | obj->maps[i].priv = NULL; | |
1651 | obj->maps[i].clear_priv = NULL; | |
1652 | } | |
1653 | zfree(&obj->maps); | |
1654 | obj->nr_maps = 0; | |
a5b8bd47 WN |
1655 | |
1656 | if (obj->programs && obj->nr_programs) { | |
1657 | for (i = 0; i < obj->nr_programs; i++) | |
1658 | bpf_program__exit(&obj->programs[i]); | |
1659 | } | |
1660 | zfree(&obj->programs); | |
1661 | ||
9a208eff | 1662 | list_del(&obj->list); |
1a5e3fb1 WN |
1663 | free(obj); |
1664 | } | |
aa9b1ac3 | 1665 | |
9a208eff WN |
1666 | struct bpf_object * |
1667 | bpf_object__next(struct bpf_object *prev) | |
1668 | { | |
1669 | struct bpf_object *next; | |
1670 | ||
1671 | if (!prev) | |
1672 | next = list_first_entry(&bpf_objects_list, | |
1673 | struct bpf_object, | |
1674 | list); | |
1675 | else | |
1676 | next = list_next_entry(prev, list); | |
1677 | ||
1678 | /* Empty list is noticed here so don't need checking on entry. */ | |
1679 | if (&next->list == &bpf_objects_list) | |
1680 | return NULL; | |
1681 | ||
1682 | return next; | |
1683 | } | |
1684 | ||
a7fe0450 | 1685 | const char *bpf_object__name(struct bpf_object *obj) |
acf860ae | 1686 | { |
a7fe0450 | 1687 | return obj ? obj->path : ERR_PTR(-EINVAL); |
acf860ae WN |
1688 | } |
1689 | ||
a7fe0450 | 1690 | unsigned int bpf_object__kversion(struct bpf_object *obj) |
45825d8a | 1691 | { |
a7fe0450 | 1692 | return obj ? obj->kern_version : 0; |
45825d8a WN |
1693 | } |
1694 | ||
10931d24 WN |
1695 | int bpf_object__set_priv(struct bpf_object *obj, void *priv, |
1696 | bpf_object_clear_priv_t clear_priv) | |
1697 | { | |
1698 | if (obj->priv && obj->clear_priv) | |
1699 | obj->clear_priv(obj, obj->priv); | |
1700 | ||
1701 | obj->priv = priv; | |
1702 | obj->clear_priv = clear_priv; | |
1703 | return 0; | |
1704 | } | |
1705 | ||
1706 | void *bpf_object__priv(struct bpf_object *obj) | |
1707 | { | |
1708 | return obj ? obj->priv : ERR_PTR(-EINVAL); | |
1709 | } | |
1710 | ||
aa9b1ac3 WN |
1711 | struct bpf_program * |
1712 | bpf_program__next(struct bpf_program *prev, struct bpf_object *obj) | |
1713 | { | |
1714 | size_t idx; | |
1715 | ||
1716 | if (!obj->programs) | |
1717 | return NULL; | |
1718 | /* First handler */ | |
1719 | if (prev == NULL) | |
1720 | return &obj->programs[0]; | |
1721 | ||
1722 | if (prev->obj != obj) { | |
1723 | pr_warning("error: program handler doesn't match object\n"); | |
1724 | return NULL; | |
1725 | } | |
1726 | ||
1727 | idx = (prev - obj->programs) + 1; | |
1728 | if (idx >= obj->nr_programs) | |
1729 | return NULL; | |
1730 | return &obj->programs[idx]; | |
1731 | } | |
1732 | ||
edb13ed4 ACM |
1733 | int bpf_program__set_priv(struct bpf_program *prog, void *priv, |
1734 | bpf_program_clear_priv_t clear_priv) | |
aa9b1ac3 WN |
1735 | { |
1736 | if (prog->priv && prog->clear_priv) | |
1737 | prog->clear_priv(prog, prog->priv); | |
1738 | ||
1739 | prog->priv = priv; | |
1740 | prog->clear_priv = clear_priv; | |
1741 | return 0; | |
1742 | } | |
1743 | ||
be834ffb | 1744 | void *bpf_program__priv(struct bpf_program *prog) |
aa9b1ac3 | 1745 | { |
be834ffb | 1746 | return prog ? prog->priv : ERR_PTR(-EINVAL); |
aa9b1ac3 WN |
1747 | } |
1748 | ||
715f8db9 | 1749 | const char *bpf_program__title(struct bpf_program *prog, bool needs_copy) |
aa9b1ac3 WN |
1750 | { |
1751 | const char *title; | |
1752 | ||
1753 | title = prog->section_name; | |
715f8db9 | 1754 | if (needs_copy) { |
aa9b1ac3 WN |
1755 | title = strdup(title); |
1756 | if (!title) { | |
1757 | pr_warning("failed to strdup program title\n"); | |
6371ca3b | 1758 | return ERR_PTR(-ENOMEM); |
aa9b1ac3 WN |
1759 | } |
1760 | } | |
1761 | ||
1762 | return title; | |
1763 | } | |
1764 | ||
1765 | int bpf_program__fd(struct bpf_program *prog) | |
1766 | { | |
b580563e WN |
1767 | return bpf_program__nth_fd(prog, 0); |
1768 | } | |
1769 | ||
1770 | int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, | |
1771 | bpf_program_prep_t prep) | |
1772 | { | |
1773 | int *instances_fds; | |
1774 | ||
1775 | if (nr_instances <= 0 || !prep) | |
1776 | return -EINVAL; | |
1777 | ||
1778 | if (prog->instances.nr > 0 || prog->instances.fds) { | |
1779 | pr_warning("Can't set pre-processor after loading\n"); | |
1780 | return -EINVAL; | |
1781 | } | |
1782 | ||
1783 | instances_fds = malloc(sizeof(int) * nr_instances); | |
1784 | if (!instances_fds) { | |
1785 | pr_warning("alloc memory failed for fds\n"); | |
1786 | return -ENOMEM; | |
1787 | } | |
1788 | ||
1789 | /* fill all fd with -1 */ | |
1790 | memset(instances_fds, -1, sizeof(int) * nr_instances); | |
1791 | ||
1792 | prog->instances.nr = nr_instances; | |
1793 | prog->instances.fds = instances_fds; | |
1794 | prog->preprocessor = prep; | |
1795 | return 0; | |
1796 | } | |
1797 | ||
1798 | int bpf_program__nth_fd(struct bpf_program *prog, int n) | |
1799 | { | |
1800 | int fd; | |
1801 | ||
1802 | if (n >= prog->instances.nr || n < 0) { | |
1803 | pr_warning("Can't get the %dth fd from program %s: only %d instances\n", | |
1804 | n, prog->section_name, prog->instances.nr); | |
1805 | return -EINVAL; | |
1806 | } | |
1807 | ||
1808 | fd = prog->instances.fds[n]; | |
1809 | if (fd < 0) { | |
1810 | pr_warning("%dth instance of program '%s' is invalid\n", | |
1811 | n, prog->section_name); | |
1812 | return -ENOENT; | |
1813 | } | |
1814 | ||
1815 | return fd; | |
aa9b1ac3 | 1816 | } |
9d759a9b | 1817 | |
dd26b7f5 | 1818 | void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) |
5f44e4c8 WN |
1819 | { |
1820 | prog->type = type; | |
1821 | } | |
1822 | ||
5f44e4c8 WN |
1823 | static bool bpf_program__is_type(struct bpf_program *prog, |
1824 | enum bpf_prog_type type) | |
1825 | { | |
1826 | return prog ? (prog->type == type) : false; | |
1827 | } | |
1828 | ||
ed794073 JS |
1829 | #define BPF_PROG_TYPE_FNS(NAME, TYPE) \ |
1830 | int bpf_program__set_##NAME(struct bpf_program *prog) \ | |
1831 | { \ | |
1832 | if (!prog) \ | |
1833 | return -EINVAL; \ | |
1834 | bpf_program__set_type(prog, TYPE); \ | |
1835 | return 0; \ | |
1836 | } \ | |
1837 | \ | |
1838 | bool bpf_program__is_##NAME(struct bpf_program *prog) \ | |
1839 | { \ | |
1840 | return bpf_program__is_type(prog, TYPE); \ | |
1841 | } \ | |
1842 | ||
7803ba73 | 1843 | BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); |
ed794073 | 1844 | BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); |
7803ba73 JS |
1845 | BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); |
1846 | BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); | |
ed794073 | 1847 | BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); |
7803ba73 JS |
1848 | BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); |
1849 | BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); | |
5f44e4c8 | 1850 | |
d7be143b AI |
1851 | static void bpf_program__set_expected_attach_type(struct bpf_program *prog, |
1852 | enum bpf_attach_type type) | |
1853 | { | |
1854 | prog->expected_attach_type = type; | |
1855 | } | |
1856 | ||
1857 | #define BPF_PROG_SEC_FULL(string, ptype, atype) \ | |
1858 | { string, sizeof(string) - 1, ptype, atype } | |
1859 | ||
1860 | #define BPF_PROG_SEC(string, ptype) BPF_PROG_SEC_FULL(string, ptype, 0) | |
1861 | ||
e50b0a6f AI |
1862 | #define BPF_SA_PROG_SEC(string, ptype) \ |
1863 | BPF_PROG_SEC_FULL(string, BPF_PROG_TYPE_CGROUP_SOCK_ADDR, ptype) | |
1864 | ||
583c9009 RG |
1865 | static const struct { |
1866 | const char *sec; | |
1867 | size_t len; | |
1868 | enum bpf_prog_type prog_type; | |
d7be143b | 1869 | enum bpf_attach_type expected_attach_type; |
583c9009 RG |
1870 | } section_names[] = { |
1871 | BPF_PROG_SEC("socket", BPF_PROG_TYPE_SOCKET_FILTER), | |
1872 | BPF_PROG_SEC("kprobe/", BPF_PROG_TYPE_KPROBE), | |
1873 | BPF_PROG_SEC("kretprobe/", BPF_PROG_TYPE_KPROBE), | |
0badd331 QM |
1874 | BPF_PROG_SEC("classifier", BPF_PROG_TYPE_SCHED_CLS), |
1875 | BPF_PROG_SEC("action", BPF_PROG_TYPE_SCHED_ACT), | |
583c9009 RG |
1876 | BPF_PROG_SEC("tracepoint/", BPF_PROG_TYPE_TRACEPOINT), |
1877 | BPF_PROG_SEC("xdp", BPF_PROG_TYPE_XDP), | |
1878 | BPF_PROG_SEC("perf_event", BPF_PROG_TYPE_PERF_EVENT), | |
1879 | BPF_PROG_SEC("cgroup/skb", BPF_PROG_TYPE_CGROUP_SKB), | |
1880 | BPF_PROG_SEC("cgroup/sock", BPF_PROG_TYPE_CGROUP_SOCK), | |
1881 | BPF_PROG_SEC("cgroup/dev", BPF_PROG_TYPE_CGROUP_DEVICE), | |
0badd331 QM |
1882 | BPF_PROG_SEC("lwt_in", BPF_PROG_TYPE_LWT_IN), |
1883 | BPF_PROG_SEC("lwt_out", BPF_PROG_TYPE_LWT_OUT), | |
1884 | BPF_PROG_SEC("lwt_xmit", BPF_PROG_TYPE_LWT_XMIT), | |
583c9009 RG |
1885 | BPF_PROG_SEC("sockops", BPF_PROG_TYPE_SOCK_OPS), |
1886 | BPF_PROG_SEC("sk_skb", BPF_PROG_TYPE_SK_SKB), | |
4c4c3c27 | 1887 | BPF_PROG_SEC("sk_msg", BPF_PROG_TYPE_SK_MSG), |
e50b0a6f AI |
1888 | BPF_SA_PROG_SEC("cgroup/bind4", BPF_CGROUP_INET4_BIND), |
1889 | BPF_SA_PROG_SEC("cgroup/bind6", BPF_CGROUP_INET6_BIND), | |
622adafb AI |
1890 | BPF_SA_PROG_SEC("cgroup/connect4", BPF_CGROUP_INET4_CONNECT), |
1891 | BPF_SA_PROG_SEC("cgroup/connect6", BPF_CGROUP_INET6_CONNECT), | |
583c9009 | 1892 | }; |
d7be143b | 1893 | |
583c9009 | 1894 | #undef BPF_PROG_SEC |
d7be143b | 1895 | #undef BPF_PROG_SEC_FULL |
e50b0a6f | 1896 | #undef BPF_SA_PROG_SEC |
583c9009 | 1897 | |
d7be143b | 1898 | static int bpf_program__identify_section(struct bpf_program *prog) |
583c9009 RG |
1899 | { |
1900 | int i; | |
1901 | ||
1902 | if (!prog->section_name) | |
1903 | goto err; | |
1904 | ||
1905 | for (i = 0; i < ARRAY_SIZE(section_names); i++) | |
1906 | if (strncmp(prog->section_name, section_names[i].sec, | |
1907 | section_names[i].len) == 0) | |
d7be143b | 1908 | return i; |
583c9009 RG |
1909 | |
1910 | err: | |
1911 | pr_warning("failed to guess program type based on section name %s\n", | |
1912 | prog->section_name); | |
1913 | ||
d7be143b | 1914 | return -1; |
583c9009 RG |
1915 | } |
1916 | ||
6e009e65 | 1917 | int bpf_map__fd(struct bpf_map *map) |
9d759a9b | 1918 | { |
6e009e65 | 1919 | return map ? map->fd : -EINVAL; |
9d759a9b WN |
1920 | } |
1921 | ||
53897a78 | 1922 | const struct bpf_map_def *bpf_map__def(struct bpf_map *map) |
9d759a9b | 1923 | { |
53897a78 | 1924 | return map ? &map->def : ERR_PTR(-EINVAL); |
9d759a9b WN |
1925 | } |
1926 | ||
009ad5d5 | 1927 | const char *bpf_map__name(struct bpf_map *map) |
561bbcca | 1928 | { |
009ad5d5 | 1929 | return map ? map->name : NULL; |
561bbcca WN |
1930 | } |
1931 | ||
edb13ed4 ACM |
1932 | int bpf_map__set_priv(struct bpf_map *map, void *priv, |
1933 | bpf_map_clear_priv_t clear_priv) | |
9d759a9b WN |
1934 | { |
1935 | if (!map) | |
1936 | return -EINVAL; | |
1937 | ||
1938 | if (map->priv) { | |
1939 | if (map->clear_priv) | |
1940 | map->clear_priv(map, map->priv); | |
1941 | } | |
1942 | ||
1943 | map->priv = priv; | |
1944 | map->clear_priv = clear_priv; | |
1945 | return 0; | |
1946 | } | |
1947 | ||
b4cbfa56 | 1948 | void *bpf_map__priv(struct bpf_map *map) |
9d759a9b | 1949 | { |
b4cbfa56 | 1950 | return map ? map->priv : ERR_PTR(-EINVAL); |
9d759a9b WN |
1951 | } |
1952 | ||
1953 | struct bpf_map * | |
1954 | bpf_map__next(struct bpf_map *prev, struct bpf_object *obj) | |
1955 | { | |
1956 | size_t idx; | |
1957 | struct bpf_map *s, *e; | |
1958 | ||
1959 | if (!obj || !obj->maps) | |
1960 | return NULL; | |
1961 | ||
1962 | s = obj->maps; | |
1963 | e = obj->maps + obj->nr_maps; | |
1964 | ||
1965 | if (prev == NULL) | |
1966 | return s; | |
1967 | ||
1968 | if ((prev < s) || (prev >= e)) { | |
1969 | pr_warning("error in %s: map handler doesn't belong to object\n", | |
1970 | __func__); | |
1971 | return NULL; | |
1972 | } | |
1973 | ||
1974 | idx = (prev - obj->maps) + 1; | |
1975 | if (idx >= obj->nr_maps) | |
1976 | return NULL; | |
1977 | return &obj->maps[idx]; | |
1978 | } | |
561bbcca WN |
1979 | |
1980 | struct bpf_map * | |
a7fe0450 | 1981 | bpf_object__find_map_by_name(struct bpf_object *obj, const char *name) |
561bbcca WN |
1982 | { |
1983 | struct bpf_map *pos; | |
1984 | ||
1985 | bpf_map__for_each(pos, obj) { | |
973170e6 | 1986 | if (pos->name && !strcmp(pos->name, name)) |
561bbcca WN |
1987 | return pos; |
1988 | } | |
1989 | return NULL; | |
1990 | } | |
5a6acad1 WN |
1991 | |
1992 | struct bpf_map * | |
1993 | bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) | |
1994 | { | |
1995 | int i; | |
1996 | ||
1997 | for (i = 0; i < obj->nr_maps; i++) { | |
1998 | if (obj->maps[i].offset == offset) | |
1999 | return &obj->maps[i]; | |
2000 | } | |
2001 | return ERR_PTR(-ENOENT); | |
2002 | } | |
e28ff1a8 JS |
2003 | |
2004 | long libbpf_get_error(const void *ptr) | |
2005 | { | |
2006 | if (IS_ERR(ptr)) | |
2007 | return PTR_ERR(ptr); | |
2008 | return 0; | |
2009 | } | |
6f6d33f3 JF |
2010 | |
2011 | int bpf_prog_load(const char *file, enum bpf_prog_type type, | |
2012 | struct bpf_object **pobj, int *prog_fd) | |
d7be143b AI |
2013 | { |
2014 | struct bpf_prog_load_attr attr; | |
2015 | ||
2016 | memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); | |
2017 | attr.file = file; | |
2018 | attr.prog_type = type; | |
2019 | attr.expected_attach_type = 0; | |
2020 | ||
2021 | return bpf_prog_load_xattr(&attr, pobj, prog_fd); | |
2022 | } | |
2023 | ||
2024 | int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, | |
2025 | struct bpf_object **pobj, int *prog_fd) | |
6f6d33f3 | 2026 | { |
48cca7e4 | 2027 | struct bpf_program *prog, *first_prog = NULL; |
d7be143b AI |
2028 | enum bpf_attach_type expected_attach_type; |
2029 | enum bpf_prog_type prog_type; | |
6f6d33f3 | 2030 | struct bpf_object *obj; |
d7be143b | 2031 | int section_idx; |
6f6d33f3 JF |
2032 | int err; |
2033 | ||
d7be143b AI |
2034 | if (!attr) |
2035 | return -EINVAL; | |
2036 | ||
2037 | obj = bpf_object__open(attr->file); | |
6f6d33f3 JF |
2038 | if (IS_ERR(obj)) |
2039 | return -ENOENT; | |
2040 | ||
48cca7e4 AS |
2041 | bpf_object__for_each_program(prog, obj) { |
2042 | /* | |
2043 | * If type is not specified, try to guess it based on | |
2044 | * section name. | |
2045 | */ | |
d7be143b AI |
2046 | prog_type = attr->prog_type; |
2047 | expected_attach_type = attr->expected_attach_type; | |
2048 | if (prog_type == BPF_PROG_TYPE_UNSPEC) { | |
2049 | section_idx = bpf_program__identify_section(prog); | |
2050 | if (section_idx < 0) { | |
48cca7e4 AS |
2051 | bpf_object__close(obj); |
2052 | return -EINVAL; | |
2053 | } | |
d7be143b AI |
2054 | prog_type = section_names[section_idx].prog_type; |
2055 | expected_attach_type = | |
2056 | section_names[section_idx].expected_attach_type; | |
583c9009 | 2057 | } |
48cca7e4 | 2058 | |
d7be143b AI |
2059 | bpf_program__set_type(prog, prog_type); |
2060 | bpf_program__set_expected_attach_type(prog, | |
2061 | expected_attach_type); | |
2062 | ||
48cca7e4 AS |
2063 | if (prog->idx != obj->efile.text_shndx && !first_prog) |
2064 | first_prog = prog; | |
2065 | } | |
2066 | ||
2067 | if (!first_prog) { | |
2068 | pr_warning("object file doesn't contain bpf program\n"); | |
2069 | bpf_object__close(obj); | |
2070 | return -ENOENT; | |
583c9009 RG |
2071 | } |
2072 | ||
6f6d33f3 JF |
2073 | err = bpf_object__load(obj); |
2074 | if (err) { | |
2075 | bpf_object__close(obj); | |
2076 | return -EINVAL; | |
2077 | } | |
2078 | ||
2079 | *pobj = obj; | |
48cca7e4 | 2080 | *prog_fd = bpf_program__fd(first_prog); |
6f6d33f3 JF |
2081 | return 0; |
2082 | } |