Commit | Line | Data |
---|---|---|
2e4913e0 AN |
1 | // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) |
2 | /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ | |
3 | #include <ctype.h> | |
4 | #include <stdio.h> | |
5 | #include <stdlib.h> | |
6 | #include <string.h> | |
7 | #include <libelf.h> | |
8 | #include <gelf.h> | |
9 | #include <unistd.h> | |
10 | #include <linux/ptrace.h> | |
11 | #include <linux/kernel.h> | |
12 | ||
58ca8b05 PL |
13 | /* s8 will be marked as poison while it's a reg of riscv */ |
14 | #if defined(__riscv) | |
15 | #define rv_s8 s8 | |
16 | #endif | |
17 | ||
2e4913e0 AN |
18 | #include "bpf.h" |
19 | #include "libbpf.h" | |
20 | #include "libbpf_common.h" | |
21 | #include "libbpf_internal.h" | |
22 | #include "hashmap.h" | |
23 | ||
24 | /* libbpf's USDT support consists of BPF-side state/code and user-space | |
25 | * state/code working together in concert. BPF-side parts are defined in | |
26 | * usdt.bpf.h header library. User-space state is encapsulated by struct | |
27 | * usdt_manager and all the supporting code centered around usdt_manager. | |
28 | * | |
29 | * usdt.bpf.h defines two BPF maps that usdt_manager expects: USDT spec map | |
30 | * and IP-to-spec-ID map, which is auxiliary map necessary for kernels that | |
31 | * don't support BPF cookie (see below). These two maps are implicitly | |
32 | * embedded into user's end BPF object file when user's code included | |
33 | * usdt.bpf.h. This means that libbpf doesn't do anything special to create | |
34 | * these USDT support maps. They are created by normal libbpf logic of | |
35 | * instantiating BPF maps when opening and loading BPF object. | |
36 | * | |
37 | * As such, libbpf is basically unaware of the need to do anything | |
38 | * USDT-related until the very first call to bpf_program__attach_usdt(), which | |
39 | * can be called by user explicitly or happen automatically during skeleton | |
40 | * attach (or, equivalently, through generic bpf_program__attach() call). At | |
41 | * this point, libbpf will instantiate and initialize struct usdt_manager and | |
42 | * store it in bpf_object. USDT manager is per-BPF object construct, as each | |
43 | * independent BPF object might or might not have USDT programs, and thus all | |
44 | * the expected USDT-related state. There is no coordination between two | |
45 | * bpf_object in parts of USDT attachment, they are oblivious of each other's | |
46 | * existence and libbpf is just oblivious, dealing with bpf_object-specific | |
47 | * USDT state. | |
48 | * | |
49 | * Quick crash course on USDTs. | |
50 | * | |
51 | * From user-space application's point of view, USDT is essentially just | |
52 | * a slightly special function call that normally has zero overhead, unless it | |
53 | * is being traced by some external entity (e.g, BPF-based tool). Here's how | |
54 | * a typical application can trigger USDT probe: | |
55 | * | |
56 | * #include <sys/sdt.h> // provided by systemtap-sdt-devel package | |
57 | * // folly also provide similar functionality in folly/tracing/StaticTracepoint.h | |
58 | * | |
59 | * STAP_PROBE3(my_usdt_provider, my_usdt_probe_name, 123, x, &y); | |
60 | * | |
61 | * USDT is identified by it's <provider-name>:<probe-name> pair of names. Each | |
62 | * individual USDT has a fixed number of arguments (3 in the above example) | |
63 | * and specifies values of each argument as if it was a function call. | |
64 | * | |
65 | * USDT call is actually not a function call, but is instead replaced by | |
66 | * a single NOP instruction (thus zero overhead, effectively). But in addition | |
67 | * to that, those USDT macros generate special SHT_NOTE ELF records in | |
68 | * .note.stapsdt ELF section. Here's an example USDT definition as emitted by | |
69 | * `readelf -n <binary>`: | |
70 | * | |
71 | * stapsdt 0x00000089 NT_STAPSDT (SystemTap probe descriptors) | |
72 | * Provider: test | |
73 | * Name: usdt12 | |
74 | * Location: 0x0000000000549df3, Base: 0x00000000008effa4, Semaphore: 0x0000000000a4606e | |
75 | * Arguments: -4@-1204(%rbp) -4@%edi -8@-1216(%rbp) -8@%r8 -4@$5 -8@%r9 8@%rdx 8@%r10 -4@$-9 -2@%cx -2@%ax -1@%sil | |
76 | * | |
77 | * In this case we have USDT test:usdt12 with 12 arguments. | |
78 | * | |
79 | * Location and base are offsets used to calculate absolute IP address of that | |
80 | * NOP instruction that kernel can replace with an interrupt instruction to | |
81 | * trigger instrumentation code (BPF program for all that we care about). | |
82 | * | |
83 | * Semaphore above is and optional feature. It records an address of a 2-byte | |
84 | * refcount variable (normally in '.probes' ELF section) used for signaling if | |
85 | * there is anything that is attached to USDT. This is useful for user | |
86 | * applications if, for example, they need to prepare some arguments that are | |
87 | * passed only to USDTs and preparation is expensive. By checking if USDT is | |
88 | * "activated", an application can avoid paying those costs unnecessarily. | |
89 | * Recent enough kernel has built-in support for automatically managing this | |
90 | * refcount, which libbpf expects and relies on. If USDT is defined without | |
91 | * associated semaphore, this value will be zero. See selftests for semaphore | |
92 | * examples. | |
93 | * | |
94 | * Arguments is the most interesting part. This USDT specification string is | |
95 | * providing information about all the USDT arguments and their locations. The | |
96 | * part before @ sign defined byte size of the argument (1, 2, 4, or 8) and | |
97 | * whether the argument is signed or unsigned (negative size means signed). | |
98 | * The part after @ sign is assembly-like definition of argument location | |
99 | * (see [0] for more details). Technically, assembler can provide some pretty | |
100 | * advanced definitions, but libbpf is currently supporting three most common | |
101 | * cases: | |
102 | * 1) immediate constant, see 5th and 9th args above (-4@$5 and -4@-9); | |
103 | * 2) register value, e.g., 8@%rdx, which means "unsigned 8-byte integer | |
104 | * whose value is in register %rdx"; | |
105 | * 3) memory dereference addressed by register, e.g., -4@-1204(%rbp), which | |
106 | * specifies signed 32-bit integer stored at offset -1204 bytes from | |
107 | * memory address stored in %rbp. | |
108 | * | |
109 | * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation | |
110 | * | |
111 | * During attachment, libbpf parses all the relevant USDT specifications and | |
112 | * prepares `struct usdt_spec` (USDT spec), which is then provided to BPF-side | |
113 | * code through spec map. This allows BPF applications to quickly fetch the | |
114 | * actual value at runtime using a simple BPF-side code. | |
115 | * | |
e1b6df59 | 116 | * With basics out of the way, let's go over less immediately obvious aspects |
2e4913e0 AN |
117 | * of supporting USDTs. |
118 | * | |
119 | * First, there is no special USDT BPF program type. It is actually just | |
120 | * a uprobe BPF program (which for kernel, at least currently, is just a kprobe | |
121 | * program, so BPF_PROG_TYPE_KPROBE program type). With the only difference | |
122 | * that uprobe is usually attached at the function entry, while USDT will | |
123 | * normally will be somewhere inside the function. But it should always be | |
124 | * pointing to NOP instruction, which makes such uprobes the fastest uprobe | |
125 | * kind. | |
126 | * | |
127 | * Second, it's important to realize that such STAP_PROBEn(provider, name, ...) | |
128 | * macro invocations can end up being inlined many-many times, depending on | |
129 | * specifics of each individual user application. So single conceptual USDT | |
130 | * (identified by provider:name pair of identifiers) is, generally speaking, | |
131 | * multiple uprobe locations (USDT call sites) in different places in user | |
132 | * application. Further, again due to inlining, each USDT call site might end | |
133 | * up having the same argument #N be located in a different place. In one call | |
134 | * site it could be a constant, in another will end up in a register, and in | |
135 | * yet another could be some other register or even somewhere on the stack. | |
136 | * | |
137 | * As such, "attaching to USDT" means (in general case) attaching the same | |
138 | * uprobe BPF program to multiple target locations in user application, each | |
139 | * potentially having a completely different USDT spec associated with it. | |
140 | * To wire all this up together libbpf allocates a unique integer spec ID for | |
141 | * each unique USDT spec. Spec IDs are allocated as sequential small integers | |
142 | * so that they can be used as keys in array BPF map (for performance reasons). | |
143 | * Spec ID allocation and accounting is big part of what usdt_manager is | |
144 | * about. This state has to be maintained per-BPF object and coordinate | |
145 | * between different USDT attachments within the same BPF object. | |
146 | * | |
147 | * Spec ID is the key in spec BPF map, value is the actual USDT spec layed out | |
148 | * as struct usdt_spec. Each invocation of BPF program at runtime needs to | |
149 | * know its associated spec ID. It gets it either through BPF cookie, which | |
150 | * libbpf sets to spec ID during attach time, or, if kernel is too old to | |
151 | * support BPF cookie, through IP-to-spec-ID map that libbpf maintains in such | |
152 | * case. The latter means that some modes of operation can't be supported | |
153 | * without BPF cookie. Such mode is attaching to shared library "generically", | |
154 | * without specifying target process. In such case, it's impossible to | |
155 | * calculate absolute IP addresses for IP-to-spec-ID map, and thus such mode | |
156 | * is not supported without BPF cookie support. | |
157 | * | |
158 | * Note that libbpf is using BPF cookie functionality for its own internal | |
159 | * needs, so user itself can't rely on BPF cookie feature. To that end, libbpf | |
160 | * provides conceptually equivalent USDT cookie support. It's still u64 | |
161 | * user-provided value that can be associated with USDT attachment. Note that | |
162 | * this will be the same value for all USDT call sites within the same single | |
163 | * *logical* USDT attachment. This makes sense because to user attaching to | |
164 | * USDT is a single BPF program triggered for singular USDT probe. The fact | |
165 | * that this is done at multiple actual locations is a mostly hidden | |
166 | * implementation details. This USDT cookie value can be fetched with | |
167 | * bpf_usdt_cookie(ctx) API provided by usdt.bpf.h | |
168 | * | |
169 | * Lastly, while single USDT can have tons of USDT call sites, it doesn't | |
170 | * necessarily have that many different USDT specs. It very well might be | |
171 | * that 1000 USDT call sites only need 5 different USDT specs, because all the | |
172 | * arguments are typically contained in a small set of registers or stack | |
173 | * locations. As such, it's wasteful to allocate as many USDT spec IDs as | |
174 | * there are USDT call sites. So libbpf tries to be frugal and performs | |
175 | * on-the-fly deduplication during a single USDT attachment to only allocate | |
176 | * the minimal required amount of unique USDT specs (and thus spec IDs). This | |
177 | * is trivially achieved by using USDT spec string (Arguments string from USDT | |
178 | * note) as a lookup key in a hashmap. USDT spec string uniquely defines | |
179 | * everything about how to fetch USDT arguments, so two USDT call sites | |
180 | * sharing USDT spec string can safely share the same USDT spec and spec ID. | |
181 | * Note, this spec string deduplication is happening only during the same USDT | |
182 | * attachment, so each USDT spec shares the same USDT cookie value. This is | |
183 | * not generally true for other USDT attachments within the same BPF object, | |
184 | * as even if USDT spec string is the same, USDT cookie value can be | |
185 | * different. It was deemed excessive to try to deduplicate across independent | |
186 | * USDT attachments by taking into account USDT spec string *and* USDT cookie | |
187 | * value, which would complicated spec ID accounting significantly for little | |
188 | * gain. | |
189 | */ | |
190 | ||
74cc6311 AN |
191 | #define USDT_BASE_SEC ".stapsdt.base" |
192 | #define USDT_SEMA_SEC ".probes" | |
193 | #define USDT_NOTE_SEC ".note.stapsdt" | |
194 | #define USDT_NOTE_TYPE 3 | |
195 | #define USDT_NOTE_NAME "stapsdt" | |
196 | ||
e1b6df59 | 197 | /* should match exactly enum __bpf_usdt_arg_type from usdt.bpf.h */ |
74cc6311 AN |
198 | enum usdt_arg_type { |
199 | USDT_ARG_CONST, | |
200 | USDT_ARG_REG, | |
201 | USDT_ARG_REG_DEREF, | |
202 | }; | |
203 | ||
e1b6df59 | 204 | /* should match exactly struct __bpf_usdt_arg_spec from usdt.bpf.h */ |
74cc6311 AN |
205 | struct usdt_arg_spec { |
206 | __u64 val_off; | |
207 | enum usdt_arg_type arg_type; | |
208 | short reg_off; | |
209 | bool arg_signed; | |
210 | char arg_bitshift; | |
211 | }; | |
212 | ||
213 | /* should match BPF_USDT_MAX_ARG_CNT in usdt.bpf.h */ | |
214 | #define USDT_MAX_ARG_CNT 12 | |
215 | ||
216 | /* should match struct __bpf_usdt_spec from usdt.bpf.h */ | |
217 | struct usdt_spec { | |
218 | struct usdt_arg_spec args[USDT_MAX_ARG_CNT]; | |
219 | __u64 usdt_cookie; | |
220 | short arg_cnt; | |
221 | }; | |
222 | ||
223 | struct usdt_note { | |
224 | const char *provider; | |
225 | const char *name; | |
226 | /* USDT args specification string, e.g.: | |
227 | * "-4@%esi -4@-24(%rbp) -4@%ecx 2@%ax 8@%rdx" | |
228 | */ | |
229 | const char *args; | |
230 | long loc_addr; | |
231 | long base_addr; | |
232 | long sema_addr; | |
233 | }; | |
234 | ||
2e4913e0 AN |
235 | struct usdt_target { |
236 | long abs_ip; | |
237 | long rel_ip; | |
238 | long sema_off; | |
74cc6311 AN |
239 | struct usdt_spec spec; |
240 | const char *spec_str; | |
2e4913e0 AN |
241 | }; |
242 | ||
243 | struct usdt_manager { | |
244 | struct bpf_map *specs_map; | |
245 | struct bpf_map *ip_to_spec_id_map; | |
246 | ||
999783c8 AN |
247 | int *free_spec_ids; |
248 | size_t free_spec_cnt; | |
249 | size_t next_free_spec_id; | |
250 | ||
2e4913e0 AN |
251 | bool has_bpf_cookie; |
252 | bool has_sema_refcnt; | |
253 | }; | |
254 | ||
255 | struct usdt_manager *usdt_manager_new(struct bpf_object *obj) | |
256 | { | |
257 | static const char *ref_ctr_sysfs_path = "/sys/bus/event_source/devices/uprobe/format/ref_ctr_offset"; | |
258 | struct usdt_manager *man; | |
259 | struct bpf_map *specs_map, *ip_to_spec_id_map; | |
260 | ||
261 | specs_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_specs"); | |
262 | ip_to_spec_id_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_ip_to_spec_id"); | |
263 | if (!specs_map || !ip_to_spec_id_map) { | |
264 | pr_warn("usdt: failed to find USDT support BPF maps, did you forget to include bpf/usdt.bpf.h?\n"); | |
265 | return ERR_PTR(-ESRCH); | |
266 | } | |
267 | ||
268 | man = calloc(1, sizeof(*man)); | |
269 | if (!man) | |
270 | return ERR_PTR(-ENOMEM); | |
271 | ||
272 | man->specs_map = specs_map; | |
273 | man->ip_to_spec_id_map = ip_to_spec_id_map; | |
274 | ||
275 | /* Detect if BPF cookie is supported for kprobes. | |
276 | * We don't need IP-to-ID mapping if we can use BPF cookies. | |
277 | * Added in: 7adfc6c9b315 ("bpf: Add bpf_get_attach_cookie() BPF helper to access bpf_cookie value") | |
278 | */ | |
279 | man->has_bpf_cookie = kernel_supports(obj, FEAT_BPF_COOKIE); | |
280 | ||
281 | /* Detect kernel support for automatic refcounting of USDT semaphore. | |
282 | * If this is not supported, USDTs with semaphores will not be supported. | |
283 | * Added in: a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe") | |
284 | */ | |
285 | man->has_sema_refcnt = access(ref_ctr_sysfs_path, F_OK) == 0; | |
286 | ||
287 | return man; | |
288 | } | |
289 | ||
290 | void usdt_manager_free(struct usdt_manager *man) | |
291 | { | |
292 | if (IS_ERR_OR_NULL(man)) | |
293 | return; | |
294 | ||
999783c8 | 295 | free(man->free_spec_ids); |
2e4913e0 AN |
296 | free(man); |
297 | } | |
298 | ||
299 | static int sanity_check_usdt_elf(Elf *elf, const char *path) | |
300 | { | |
301 | GElf_Ehdr ehdr; | |
302 | int endianness; | |
303 | ||
304 | if (elf_kind(elf) != ELF_K_ELF) { | |
305 | pr_warn("usdt: unrecognized ELF kind %d for '%s'\n", elf_kind(elf), path); | |
306 | return -EBADF; | |
307 | } | |
308 | ||
309 | switch (gelf_getclass(elf)) { | |
310 | case ELFCLASS64: | |
311 | if (sizeof(void *) != 8) { | |
312 | pr_warn("usdt: attaching to 64-bit ELF binary '%s' is not supported\n", path); | |
313 | return -EBADF; | |
314 | } | |
315 | break; | |
316 | case ELFCLASS32: | |
317 | if (sizeof(void *) != 4) { | |
318 | pr_warn("usdt: attaching to 32-bit ELF binary '%s' is not supported\n", path); | |
319 | return -EBADF; | |
320 | } | |
321 | break; | |
322 | default: | |
323 | pr_warn("usdt: unsupported ELF class for '%s'\n", path); | |
324 | return -EBADF; | |
325 | } | |
326 | ||
327 | if (!gelf_getehdr(elf, &ehdr)) | |
328 | return -EINVAL; | |
329 | ||
330 | if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN) { | |
331 | pr_warn("usdt: unsupported type of ELF binary '%s' (%d), only ET_EXEC and ET_DYN are supported\n", | |
332 | path, ehdr.e_type); | |
333 | return -EBADF; | |
334 | } | |
335 | ||
e1b6df59 | 336 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ |
2e4913e0 | 337 | endianness = ELFDATA2LSB; |
e1b6df59 | 338 | #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ |
2e4913e0 AN |
339 | endianness = ELFDATA2MSB; |
340 | #else | |
341 | # error "Unrecognized __BYTE_ORDER__" | |
342 | #endif | |
343 | if (endianness != ehdr.e_ident[EI_DATA]) { | |
344 | pr_warn("usdt: ELF endianness mismatch for '%s'\n", path); | |
345 | return -EBADF; | |
346 | } | |
347 | ||
348 | return 0; | |
349 | } | |
350 | ||
74cc6311 AN |
351 | static int find_elf_sec_by_name(Elf *elf, const char *sec_name, GElf_Shdr *shdr, Elf_Scn **scn) |
352 | { | |
353 | Elf_Scn *sec = NULL; | |
354 | size_t shstrndx; | |
355 | ||
356 | if (elf_getshdrstrndx(elf, &shstrndx)) | |
357 | return -EINVAL; | |
358 | ||
359 | /* check if ELF is corrupted and avoid calling elf_strptr if yes */ | |
360 | if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL)) | |
361 | return -EINVAL; | |
362 | ||
363 | while ((sec = elf_nextscn(elf, sec)) != NULL) { | |
364 | char *name; | |
365 | ||
366 | if (!gelf_getshdr(sec, shdr)) | |
367 | return -EINVAL; | |
368 | ||
369 | name = elf_strptr(elf, shstrndx, shdr->sh_name); | |
370 | if (name && strcmp(sec_name, name) == 0) { | |
371 | *scn = sec; | |
372 | return 0; | |
373 | } | |
374 | } | |
375 | ||
376 | return -ENOENT; | |
377 | } | |
378 | ||
379 | struct elf_seg { | |
380 | long start; | |
381 | long end; | |
382 | long offset; | |
383 | bool is_exec; | |
384 | }; | |
385 | ||
386 | static int cmp_elf_segs(const void *_a, const void *_b) | |
387 | { | |
388 | const struct elf_seg *a = _a; | |
389 | const struct elf_seg *b = _b; | |
390 | ||
391 | return a->start < b->start ? -1 : 1; | |
392 | } | |
393 | ||
394 | static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, size_t *seg_cnt) | |
395 | { | |
396 | GElf_Phdr phdr; | |
397 | size_t n; | |
398 | int i, err; | |
399 | struct elf_seg *seg; | |
400 | void *tmp; | |
401 | ||
402 | *seg_cnt = 0; | |
403 | ||
404 | if (elf_getphdrnum(elf, &n)) { | |
405 | err = -errno; | |
406 | return err; | |
407 | } | |
408 | ||
409 | for (i = 0; i < n; i++) { | |
410 | if (!gelf_getphdr(elf, i, &phdr)) { | |
411 | err = -errno; | |
412 | return err; | |
413 | } | |
414 | ||
415 | pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n", | |
416 | i, path, (long)phdr.p_vaddr, (long)phdr.p_memsz, (long)phdr.p_offset, | |
417 | (long)phdr.p_type, (long)phdr.p_flags); | |
418 | if (phdr.p_type != PT_LOAD) | |
419 | continue; | |
420 | ||
421 | tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs)); | |
422 | if (!tmp) | |
423 | return -ENOMEM; | |
424 | ||
425 | *segs = tmp; | |
426 | seg = *segs + *seg_cnt; | |
427 | (*seg_cnt)++; | |
428 | ||
429 | seg->start = phdr.p_vaddr; | |
430 | seg->end = phdr.p_vaddr + phdr.p_memsz; | |
431 | seg->offset = phdr.p_offset; | |
432 | seg->is_exec = phdr.p_flags & PF_X; | |
433 | } | |
434 | ||
435 | if (*seg_cnt == 0) { | |
436 | pr_warn("usdt: failed to find PT_LOAD program headers in '%s'\n", path); | |
437 | return -ESRCH; | |
438 | } | |
439 | ||
440 | qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs); | |
441 | return 0; | |
442 | } | |
443 | ||
444 | static int parse_lib_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt) | |
445 | { | |
446 | char path[PATH_MAX], line[PATH_MAX], mode[16]; | |
447 | size_t seg_start, seg_end, seg_off; | |
448 | struct elf_seg *seg; | |
449 | int tmp_pid, i, err; | |
450 | FILE *f; | |
451 | ||
452 | *seg_cnt = 0; | |
453 | ||
454 | /* Handle containerized binaries only accessible from | |
455 | * /proc/<pid>/root/<path>. They will be reported as just /<path> in | |
456 | * /proc/<pid>/maps. | |
457 | */ | |
458 | if (sscanf(lib_path, "/proc/%d/root%s", &tmp_pid, path) == 2 && pid == tmp_pid) | |
459 | goto proceed; | |
460 | ||
461 | if (!realpath(lib_path, path)) { | |
462 | pr_warn("usdt: failed to get absolute path of '%s' (err %d), using path as is...\n", | |
463 | lib_path, -errno); | |
3c0dfe6e | 464 | libbpf_strlcpy(path, lib_path, sizeof(path)); |
74cc6311 AN |
465 | } |
466 | ||
467 | proceed: | |
468 | sprintf(line, "/proc/%d/maps", pid); | |
469 | f = fopen(line, "r"); | |
470 | if (!f) { | |
471 | err = -errno; | |
472 | pr_warn("usdt: failed to open '%s' to get base addr of '%s': %d\n", | |
473 | line, lib_path, err); | |
474 | return err; | |
475 | } | |
476 | ||
477 | /* We need to handle lines with no path at the end: | |
478 | * | |
479 | * 7f5c6f5d1000-7f5c6f5d3000 rw-p 001c7000 08:04 21238613 /usr/lib64/libc-2.17.so | |
480 | * 7f5c6f5d3000-7f5c6f5d8000 rw-p 00000000 00:00 0 | |
481 | * 7f5c6f5d8000-7f5c6f5d9000 r-xp 00000000 103:01 362990598 /data/users/andriin/linux/tools/bpf/usdt/libhello_usdt.so | |
482 | */ | |
483 | while (fscanf(f, "%zx-%zx %s %zx %*s %*d%[^\n]\n", | |
484 | &seg_start, &seg_end, mode, &seg_off, line) == 5) { | |
485 | void *tmp; | |
486 | ||
487 | /* to handle no path case (see above) we need to capture line | |
488 | * without skipping any whitespaces. So we need to strip | |
489 | * leading whitespaces manually here | |
490 | */ | |
491 | i = 0; | |
492 | while (isblank(line[i])) | |
493 | i++; | |
494 | if (strcmp(line + i, path) != 0) | |
495 | continue; | |
496 | ||
497 | pr_debug("usdt: discovered segment for lib '%s': addrs %zx-%zx mode %s offset %zx\n", | |
498 | path, seg_start, seg_end, mode, seg_off); | |
499 | ||
500 | /* ignore non-executable sections for shared libs */ | |
501 | if (mode[2] != 'x') | |
502 | continue; | |
503 | ||
504 | tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs)); | |
505 | if (!tmp) { | |
506 | err = -ENOMEM; | |
507 | goto err_out; | |
508 | } | |
509 | ||
510 | *segs = tmp; | |
511 | seg = *segs + *seg_cnt; | |
512 | *seg_cnt += 1; | |
513 | ||
514 | seg->start = seg_start; | |
515 | seg->end = seg_end; | |
516 | seg->offset = seg_off; | |
517 | seg->is_exec = true; | |
518 | } | |
519 | ||
520 | if (*seg_cnt == 0) { | |
521 | pr_warn("usdt: failed to find '%s' (resolved to '%s') within PID %d memory mappings\n", | |
522 | lib_path, path, pid); | |
523 | err = -ESRCH; | |
524 | goto err_out; | |
525 | } | |
526 | ||
527 | qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs); | |
528 | err = 0; | |
529 | err_out: | |
530 | fclose(f); | |
531 | return err; | |
532 | } | |
533 | ||
534 | static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long addr, bool relative) | |
535 | { | |
536 | struct elf_seg *seg; | |
537 | int i; | |
538 | ||
539 | if (relative) { | |
540 | /* for shared libraries, address is relative offset and thus | |
541 | * should be fall within logical offset-based range of | |
542 | * [offset_start, offset_end) | |
543 | */ | |
544 | for (i = 0, seg = segs; i < seg_cnt; i++, seg++) { | |
545 | if (seg->offset <= addr && addr < seg->offset + (seg->end - seg->start)) | |
546 | return seg; | |
547 | } | |
548 | } else { | |
549 | /* for binaries, address is absolute and thus should be within | |
550 | * absolute address range of [seg_start, seg_end) | |
551 | */ | |
552 | for (i = 0, seg = segs; i < seg_cnt; i++, seg++) { | |
553 | if (seg->start <= addr && addr < seg->end) | |
554 | return seg; | |
555 | } | |
556 | } | |
557 | ||
558 | return NULL; | |
559 | } | |
560 | ||
561 | static int parse_usdt_note(Elf *elf, const char *path, long base_addr, | |
562 | GElf_Nhdr *nhdr, const char *data, size_t name_off, size_t desc_off, | |
563 | struct usdt_note *usdt_note); | |
564 | ||
5af25a41 | 565 | static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie); |
74cc6311 | 566 | |
2e4913e0 | 567 | static int collect_usdt_targets(struct usdt_manager *man, Elf *elf, const char *path, pid_t pid, |
5af25a41 | 568 | const char *usdt_provider, const char *usdt_name, __u64 usdt_cookie, |
2e4913e0 AN |
569 | struct usdt_target **out_targets, size_t *out_target_cnt) |
570 | { | |
74cc6311 AN |
571 | size_t off, name_off, desc_off, seg_cnt = 0, lib_seg_cnt = 0, target_cnt = 0; |
572 | struct elf_seg *segs = NULL, *lib_segs = NULL; | |
573 | struct usdt_target *targets = NULL, *target; | |
574 | long base_addr = 0; | |
575 | Elf_Scn *notes_scn, *base_scn; | |
576 | GElf_Shdr base_shdr, notes_shdr; | |
577 | GElf_Ehdr ehdr; | |
578 | GElf_Nhdr nhdr; | |
579 | Elf_Data *data; | |
580 | int err; | |
581 | ||
582 | *out_targets = NULL; | |
583 | *out_target_cnt = 0; | |
584 | ||
585 | err = find_elf_sec_by_name(elf, USDT_NOTE_SEC, ¬es_shdr, ¬es_scn); | |
586 | if (err) { | |
587 | pr_warn("usdt: no USDT notes section (%s) found in '%s'\n", USDT_NOTE_SEC, path); | |
588 | return err; | |
589 | } | |
590 | ||
591 | if (notes_shdr.sh_type != SHT_NOTE || !gelf_getehdr(elf, &ehdr)) { | |
592 | pr_warn("usdt: invalid USDT notes section (%s) in '%s'\n", USDT_NOTE_SEC, path); | |
593 | return -EINVAL; | |
594 | } | |
595 | ||
596 | err = parse_elf_segs(elf, path, &segs, &seg_cnt); | |
597 | if (err) { | |
598 | pr_warn("usdt: failed to process ELF program segments for '%s': %d\n", path, err); | |
599 | goto err_out; | |
600 | } | |
601 | ||
602 | /* .stapsdt.base ELF section is optional, but is used for prelink | |
603 | * offset compensation (see a big comment further below) | |
604 | */ | |
605 | if (find_elf_sec_by_name(elf, USDT_BASE_SEC, &base_shdr, &base_scn) == 0) | |
606 | base_addr = base_shdr.sh_addr; | |
607 | ||
608 | data = elf_getdata(notes_scn, 0); | |
609 | off = 0; | |
610 | while ((off = gelf_getnote(data, off, &nhdr, &name_off, &desc_off)) > 0) { | |
611 | long usdt_abs_ip, usdt_rel_ip, usdt_sema_off = 0; | |
612 | struct usdt_note note; | |
613 | struct elf_seg *seg = NULL; | |
614 | void *tmp; | |
615 | ||
616 | err = parse_usdt_note(elf, path, base_addr, &nhdr, | |
617 | data->d_buf, name_off, desc_off, ¬e); | |
618 | if (err) | |
619 | goto err_out; | |
620 | ||
621 | if (strcmp(note.provider, usdt_provider) != 0 || strcmp(note.name, usdt_name) != 0) | |
622 | continue; | |
623 | ||
624 | /* We need to compensate "prelink effect". See [0] for details, | |
625 | * relevant parts quoted here: | |
626 | * | |
627 | * Each SDT probe also expands into a non-allocated ELF note. You can | |
628 | * find this by looking at SHT_NOTE sections and decoding the format; | |
629 | * see below for details. Because the note is non-allocated, it means | |
630 | * there is no runtime cost, and also preserved in both stripped files | |
631 | * and .debug files. | |
632 | * | |
633 | * However, this means that prelink won't adjust the note's contents | |
634 | * for address offsets. Instead, this is done via the .stapsdt.base | |
635 | * section. This is a special section that is added to the text. We | |
636 | * will only ever have one of these sections in a final link and it | |
637 | * will only ever be one byte long. Nothing about this section itself | |
638 | * matters, we just use it as a marker to detect prelink address | |
639 | * adjustments. | |
640 | * | |
641 | * Each probe note records the link-time address of the .stapsdt.base | |
642 | * section alongside the probe PC address. The decoder compares the | |
643 | * base address stored in the note with the .stapsdt.base section's | |
644 | * sh_addr. Initially these are the same, but the section header will | |
645 | * be adjusted by prelink. So the decoder applies the difference to | |
646 | * the probe PC address to get the correct prelinked PC address; the | |
647 | * same adjustment is applied to the semaphore address, if any. | |
648 | * | |
649 | * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation | |
650 | */ | |
651 | usdt_rel_ip = usdt_abs_ip = note.loc_addr; | |
652 | if (base_addr) { | |
653 | usdt_abs_ip += base_addr - note.base_addr; | |
654 | usdt_rel_ip += base_addr - note.base_addr; | |
655 | } | |
656 | ||
657 | if (ehdr.e_type == ET_EXEC) { | |
658 | /* When attaching uprobes (which what USDTs basically | |
659 | * are) kernel expects a relative IP to be specified, | |
660 | * so if we are attaching to an executable ELF binary | |
661 | * (i.e., not a shared library), we need to calculate | |
662 | * proper relative IP based on ELF's load address | |
663 | */ | |
664 | seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip, false /* relative */); | |
665 | if (!seg) { | |
666 | err = -ESRCH; | |
667 | pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n", | |
668 | usdt_provider, usdt_name, path, usdt_abs_ip); | |
669 | goto err_out; | |
670 | } | |
671 | if (!seg->is_exec) { | |
672 | err = -ESRCH; | |
673 | pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n", | |
674 | path, seg->start, seg->end, usdt_provider, usdt_name, | |
675 | usdt_abs_ip); | |
676 | goto err_out; | |
677 | } | |
678 | ||
679 | usdt_rel_ip = usdt_abs_ip - (seg->start - seg->offset); | |
680 | } else if (!man->has_bpf_cookie) { /* ehdr.e_type == ET_DYN */ | |
681 | /* If we don't have BPF cookie support but need to | |
682 | * attach to a shared library, we'll need to know and | |
683 | * record absolute addresses of attach points due to | |
684 | * the need to lookup USDT spec by absolute IP of | |
685 | * triggered uprobe. Doing this resolution is only | |
686 | * possible when we have a specific PID of the process | |
687 | * that's using specified shared library. BPF cookie | |
688 | * removes the absolute address limitation as we don't | |
689 | * need to do this lookup (we just use BPF cookie as | |
690 | * an index of USDT spec), so for newer kernels with | |
691 | * BPF cookie support libbpf supports USDT attachment | |
692 | * to shared libraries with no PID filter. | |
693 | */ | |
694 | if (pid < 0) { | |
a8d600f6 | 695 | pr_warn("usdt: attaching to shared libraries without specific PID is not supported on current kernel\n"); |
74cc6311 AN |
696 | err = -ENOTSUP; |
697 | goto err_out; | |
698 | } | |
699 | ||
700 | /* lib_segs are lazily initialized only if necessary */ | |
701 | if (lib_seg_cnt == 0) { | |
702 | err = parse_lib_segs(pid, path, &lib_segs, &lib_seg_cnt); | |
703 | if (err) { | |
704 | pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %d\n", | |
705 | pid, path, err); | |
706 | goto err_out; | |
707 | } | |
708 | } | |
709 | ||
710 | seg = find_elf_seg(lib_segs, lib_seg_cnt, usdt_rel_ip, true /* relative */); | |
711 | if (!seg) { | |
712 | err = -ESRCH; | |
713 | pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n", | |
714 | usdt_provider, usdt_name, path, usdt_rel_ip); | |
715 | goto err_out; | |
716 | } | |
717 | ||
718 | usdt_abs_ip = seg->start + (usdt_rel_ip - seg->offset); | |
719 | } | |
720 | ||
721 | pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n", | |
722 | usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path, | |
723 | note.loc_addr, note.base_addr, usdt_abs_ip, usdt_rel_ip, note.args, | |
724 | seg ? seg->start : 0, seg ? seg->end : 0, seg ? seg->offset : 0); | |
725 | ||
726 | /* Adjust semaphore address to be a relative offset */ | |
727 | if (note.sema_addr) { | |
728 | if (!man->has_sema_refcnt) { | |
729 | pr_warn("usdt: kernel doesn't support USDT semaphore refcounting for '%s:%s' in '%s'\n", | |
730 | usdt_provider, usdt_name, path); | |
731 | err = -ENOTSUP; | |
732 | goto err_out; | |
733 | } | |
734 | ||
735 | seg = find_elf_seg(segs, seg_cnt, note.sema_addr, false /* relative */); | |
736 | if (!seg) { | |
737 | err = -ESRCH; | |
738 | pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n", | |
739 | usdt_provider, usdt_name, path, note.sema_addr); | |
740 | goto err_out; | |
741 | } | |
742 | if (seg->is_exec) { | |
743 | err = -ESRCH; | |
744 | pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n", | |
745 | path, seg->start, seg->end, usdt_provider, usdt_name, | |
746 | note.sema_addr); | |
747 | goto err_out; | |
748 | } | |
749 | ||
750 | usdt_sema_off = note.sema_addr - (seg->start - seg->offset); | |
751 | ||
752 | pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n", | |
753 | usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", | |
754 | path, note.sema_addr, note.base_addr, usdt_sema_off, | |
755 | seg->start, seg->end, seg->offset); | |
756 | } | |
757 | ||
758 | /* Record adjusted addresses and offsets and parse USDT spec */ | |
759 | tmp = libbpf_reallocarray(targets, target_cnt + 1, sizeof(*targets)); | |
760 | if (!tmp) { | |
761 | err = -ENOMEM; | |
762 | goto err_out; | |
763 | } | |
764 | targets = tmp; | |
765 | ||
766 | target = &targets[target_cnt]; | |
767 | memset(target, 0, sizeof(*target)); | |
768 | ||
769 | target->abs_ip = usdt_abs_ip; | |
770 | target->rel_ip = usdt_rel_ip; | |
771 | target->sema_off = usdt_sema_off; | |
772 | ||
773 | /* notes->args references strings from Elf itself, so they can | |
774 | * be referenced safely until elf_end() call | |
775 | */ | |
776 | target->spec_str = note.args; | |
777 | ||
778 | err = parse_usdt_spec(&target->spec, ¬e, usdt_cookie); | |
779 | if (err) | |
780 | goto err_out; | |
781 | ||
782 | target_cnt++; | |
783 | } | |
784 | ||
785 | *out_targets = targets; | |
786 | *out_target_cnt = target_cnt; | |
787 | err = target_cnt; | |
788 | ||
789 | err_out: | |
790 | free(segs); | |
791 | free(lib_segs); | |
792 | if (err < 0) | |
793 | free(targets); | |
794 | return err; | |
2e4913e0 AN |
795 | } |
796 | ||
797 | struct bpf_link_usdt { | |
798 | struct bpf_link link; | |
799 | ||
800 | struct usdt_manager *usdt_man; | |
801 | ||
999783c8 AN |
802 | size_t spec_cnt; |
803 | int *spec_ids; | |
804 | ||
2e4913e0 AN |
805 | size_t uprobe_cnt; |
806 | struct { | |
807 | long abs_ip; | |
808 | struct bpf_link *link; | |
809 | } *uprobes; | |
810 | }; | |
811 | ||
812 | static int bpf_link_usdt_detach(struct bpf_link *link) | |
813 | { | |
814 | struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); | |
999783c8 | 815 | struct usdt_manager *man = usdt_link->usdt_man; |
2e4913e0 AN |
816 | int i; |
817 | ||
818 | for (i = 0; i < usdt_link->uprobe_cnt; i++) { | |
819 | /* detach underlying uprobe link */ | |
820 | bpf_link__destroy(usdt_link->uprobes[i].link); | |
999783c8 AN |
821 | /* there is no need to update specs map because it will be |
822 | * unconditionally overwritten on subsequent USDT attaches, | |
823 | * but if BPF cookies are not used we need to remove entry | |
824 | * from ip_to_spec_id map, otherwise we'll run into false | |
825 | * conflicting IP errors | |
826 | */ | |
827 | if (!man->has_bpf_cookie) { | |
828 | /* not much we can do about errors here */ | |
829 | (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map), | |
830 | &usdt_link->uprobes[i].abs_ip); | |
831 | } | |
832 | } | |
833 | ||
834 | /* try to return the list of previously used spec IDs to usdt_manager | |
835 | * for future reuse for subsequent USDT attaches | |
836 | */ | |
837 | if (!man->free_spec_ids) { | |
838 | /* if there were no free spec IDs yet, just transfer our IDs */ | |
839 | man->free_spec_ids = usdt_link->spec_ids; | |
840 | man->free_spec_cnt = usdt_link->spec_cnt; | |
841 | usdt_link->spec_ids = NULL; | |
842 | } else { | |
843 | /* otherwise concat IDs */ | |
844 | size_t new_cnt = man->free_spec_cnt + usdt_link->spec_cnt; | |
845 | int *new_free_ids; | |
846 | ||
847 | new_free_ids = libbpf_reallocarray(man->free_spec_ids, new_cnt, | |
848 | sizeof(*new_free_ids)); | |
849 | /* If we couldn't resize free_spec_ids, we'll just leak | |
850 | * a bunch of free IDs; this is very unlikely to happen and if | |
e1b6df59 | 851 | * system is so exhausted on memory, it's the least of user's |
999783c8 AN |
852 | * concerns, probably. |
853 | * So just do our best here to return those IDs to usdt_manager. | |
854 | */ | |
855 | if (new_free_ids) { | |
856 | memcpy(new_free_ids + man->free_spec_cnt, usdt_link->spec_ids, | |
857 | usdt_link->spec_cnt * sizeof(*usdt_link->spec_ids)); | |
858 | man->free_spec_ids = new_free_ids; | |
859 | man->free_spec_cnt = new_cnt; | |
860 | } | |
2e4913e0 AN |
861 | } |
862 | ||
863 | return 0; | |
864 | } | |
865 | ||
866 | static void bpf_link_usdt_dealloc(struct bpf_link *link) | |
867 | { | |
868 | struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link); | |
869 | ||
999783c8 | 870 | free(usdt_link->spec_ids); |
2e4913e0 AN |
871 | free(usdt_link->uprobes); |
872 | free(usdt_link); | |
873 | } | |
874 | ||
999783c8 AN |
875 | static size_t specs_hash_fn(const void *key, void *ctx) |
876 | { | |
877 | const char *s = key; | |
878 | ||
879 | return str_hash(s); | |
880 | } | |
881 | ||
882 | static bool specs_equal_fn(const void *key1, const void *key2, void *ctx) | |
883 | { | |
884 | const char *s1 = key1; | |
885 | const char *s2 = key2; | |
886 | ||
887 | return strcmp(s1, s2) == 0; | |
888 | } | |
889 | ||
890 | static int allocate_spec_id(struct usdt_manager *man, struct hashmap *specs_hash, | |
891 | struct bpf_link_usdt *link, struct usdt_target *target, | |
892 | int *spec_id, bool *is_new) | |
893 | { | |
894 | void *tmp; | |
895 | int err; | |
896 | ||
897 | /* check if we already allocated spec ID for this spec string */ | |
898 | if (hashmap__find(specs_hash, target->spec_str, &tmp)) { | |
899 | *spec_id = (long)tmp; | |
900 | *is_new = false; | |
901 | return 0; | |
902 | } | |
903 | ||
904 | /* otherwise it's a new ID that needs to be set up in specs map and | |
905 | * returned back to usdt_manager when USDT link is detached | |
906 | */ | |
907 | tmp = libbpf_reallocarray(link->spec_ids, link->spec_cnt + 1, sizeof(*link->spec_ids)); | |
908 | if (!tmp) | |
909 | return -ENOMEM; | |
910 | link->spec_ids = tmp; | |
911 | ||
912 | /* get next free spec ID, giving preference to free list, if not empty */ | |
913 | if (man->free_spec_cnt) { | |
914 | *spec_id = man->free_spec_ids[man->free_spec_cnt - 1]; | |
915 | ||
916 | /* cache spec ID for current spec string for future lookups */ | |
917 | err = hashmap__add(specs_hash, target->spec_str, (void *)(long)*spec_id); | |
918 | if (err) | |
919 | return err; | |
920 | ||
921 | man->free_spec_cnt--; | |
922 | } else { | |
923 | /* don't allocate spec ID bigger than what fits in specs map */ | |
924 | if (man->next_free_spec_id >= bpf_map__max_entries(man->specs_map)) | |
925 | return -E2BIG; | |
926 | ||
927 | *spec_id = man->next_free_spec_id; | |
928 | ||
929 | /* cache spec ID for current spec string for future lookups */ | |
930 | err = hashmap__add(specs_hash, target->spec_str, (void *)(long)*spec_id); | |
931 | if (err) | |
932 | return err; | |
933 | ||
934 | man->next_free_spec_id++; | |
935 | } | |
936 | ||
937 | /* remember new spec ID in the link for later return back to free list on detach */ | |
938 | link->spec_ids[link->spec_cnt] = *spec_id; | |
939 | link->spec_cnt++; | |
940 | *is_new = true; | |
941 | return 0; | |
942 | } | |
943 | ||
2e4913e0 AN |
944 | struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct bpf_program *prog, |
945 | pid_t pid, const char *path, | |
946 | const char *usdt_provider, const char *usdt_name, | |
5af25a41 | 947 | __u64 usdt_cookie) |
2e4913e0 | 948 | { |
999783c8 | 949 | int i, fd, err, spec_map_fd, ip_map_fd; |
2e4913e0 | 950 | LIBBPF_OPTS(bpf_uprobe_opts, opts); |
999783c8 | 951 | struct hashmap *specs_hash = NULL; |
2e4913e0 AN |
952 | struct bpf_link_usdt *link = NULL; |
953 | struct usdt_target *targets = NULL; | |
954 | size_t target_cnt; | |
2e4913e0 AN |
955 | Elf *elf; |
956 | ||
999783c8 AN |
957 | spec_map_fd = bpf_map__fd(man->specs_map); |
958 | ip_map_fd = bpf_map__fd(man->ip_to_spec_id_map); | |
959 | ||
2e4913e0 AN |
960 | /* TODO: perform path resolution similar to uprobe's */ |
961 | fd = open(path, O_RDONLY); | |
962 | if (fd < 0) { | |
963 | err = -errno; | |
964 | pr_warn("usdt: failed to open ELF binary '%s': %d\n", path, err); | |
965 | return libbpf_err_ptr(err); | |
966 | } | |
967 | ||
968 | elf = elf_begin(fd, ELF_C_READ_MMAP, NULL); | |
969 | if (!elf) { | |
970 | err = -EBADF; | |
971 | pr_warn("usdt: failed to parse ELF binary '%s': %s\n", path, elf_errmsg(-1)); | |
972 | goto err_out; | |
973 | } | |
974 | ||
975 | err = sanity_check_usdt_elf(elf, path); | |
976 | if (err) | |
977 | goto err_out; | |
978 | ||
979 | /* normalize PID filter */ | |
980 | if (pid < 0) | |
981 | pid = -1; | |
982 | else if (pid == 0) | |
983 | pid = getpid(); | |
984 | ||
985 | /* discover USDT in given binary, optionally limiting | |
986 | * activations to a given PID, if pid > 0 | |
987 | */ | |
988 | err = collect_usdt_targets(man, elf, path, pid, usdt_provider, usdt_name, | |
989 | usdt_cookie, &targets, &target_cnt); | |
990 | if (err <= 0) { | |
991 | err = (err == 0) ? -ENOENT : err; | |
992 | goto err_out; | |
993 | } | |
994 | ||
999783c8 AN |
995 | specs_hash = hashmap__new(specs_hash_fn, specs_equal_fn, NULL); |
996 | if (IS_ERR(specs_hash)) { | |
997 | err = PTR_ERR(specs_hash); | |
998 | goto err_out; | |
999 | } | |
1000 | ||
2e4913e0 AN |
1001 | link = calloc(1, sizeof(*link)); |
1002 | if (!link) { | |
1003 | err = -ENOMEM; | |
1004 | goto err_out; | |
1005 | } | |
1006 | ||
1007 | link->usdt_man = man; | |
1008 | link->link.detach = &bpf_link_usdt_detach; | |
1009 | link->link.dealloc = &bpf_link_usdt_dealloc; | |
1010 | ||
1011 | link->uprobes = calloc(target_cnt, sizeof(*link->uprobes)); | |
1012 | if (!link->uprobes) { | |
1013 | err = -ENOMEM; | |
1014 | goto err_out; | |
1015 | } | |
1016 | ||
1017 | for (i = 0; i < target_cnt; i++) { | |
1018 | struct usdt_target *target = &targets[i]; | |
1019 | struct bpf_link *uprobe_link; | |
999783c8 AN |
1020 | bool is_new; |
1021 | int spec_id; | |
1022 | ||
1023 | /* Spec ID can be either reused or newly allocated. If it is | |
1024 | * newly allocated, we'll need to fill out spec map, otherwise | |
1025 | * entire spec should be valid and can be just used by a new | |
1026 | * uprobe. We reuse spec when USDT arg spec is identical. We | |
1027 | * also never share specs between two different USDT | |
1028 | * attachments ("links"), so all the reused specs already | |
1029 | * share USDT cookie value implicitly. | |
1030 | */ | |
1031 | err = allocate_spec_id(man, specs_hash, link, target, &spec_id, &is_new); | |
1032 | if (err) | |
1033 | goto err_out; | |
1034 | ||
1035 | if (is_new && bpf_map_update_elem(spec_map_fd, &spec_id, &target->spec, BPF_ANY)) { | |
1036 | err = -errno; | |
1037 | pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %d\n", | |
1038 | spec_id, usdt_provider, usdt_name, path, err); | |
1039 | goto err_out; | |
1040 | } | |
1041 | if (!man->has_bpf_cookie && | |
1042 | bpf_map_update_elem(ip_map_fd, &target->abs_ip, &spec_id, BPF_NOEXIST)) { | |
1043 | err = -errno; | |
1044 | if (err == -EEXIST) { | |
1045 | pr_warn("usdt: IP collision detected for spec #%d for '%s:%s' in '%s'\n", | |
1046 | spec_id, usdt_provider, usdt_name, path); | |
1047 | } else { | |
1048 | pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %d\n", | |
1049 | target->abs_ip, spec_id, usdt_provider, usdt_name, | |
1050 | path, err); | |
1051 | } | |
1052 | goto err_out; | |
1053 | } | |
2e4913e0 AN |
1054 | |
1055 | opts.ref_ctr_offset = target->sema_off; | |
999783c8 | 1056 | opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0; |
2e4913e0 AN |
1057 | uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path, |
1058 | target->rel_ip, &opts); | |
1059 | err = libbpf_get_error(uprobe_link); | |
1060 | if (err) { | |
1061 | pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %d\n", | |
1062 | i, usdt_provider, usdt_name, path, err); | |
1063 | goto err_out; | |
1064 | } | |
1065 | ||
1066 | link->uprobes[i].link = uprobe_link; | |
1067 | link->uprobes[i].abs_ip = target->abs_ip; | |
1068 | link->uprobe_cnt++; | |
1069 | } | |
1070 | ||
74cc6311 | 1071 | free(targets); |
999783c8 | 1072 | hashmap__free(specs_hash); |
2e4913e0 AN |
1073 | elf_end(elf); |
1074 | close(fd); | |
1075 | ||
1076 | return &link->link; | |
1077 | ||
1078 | err_out: | |
e58c5c97 HB |
1079 | if (link) |
1080 | bpf_link__destroy(&link->link); | |
74cc6311 | 1081 | free(targets); |
999783c8 | 1082 | hashmap__free(specs_hash); |
2e4913e0 AN |
1083 | if (elf) |
1084 | elf_end(elf); | |
1085 | close(fd); | |
1086 | return libbpf_err_ptr(err); | |
1087 | } | |
74cc6311 AN |
1088 | |
1089 | /* Parse out USDT ELF note from '.note.stapsdt' section. | |
1090 | * Logic inspired by perf's code. | |
1091 | */ | |
1092 | static int parse_usdt_note(Elf *elf, const char *path, long base_addr, | |
1093 | GElf_Nhdr *nhdr, const char *data, size_t name_off, size_t desc_off, | |
1094 | struct usdt_note *note) | |
1095 | { | |
1096 | const char *provider, *name, *args; | |
1097 | long addrs[3]; | |
1098 | size_t len; | |
1099 | ||
1100 | /* sanity check USDT note name and type first */ | |
1101 | if (strncmp(data + name_off, USDT_NOTE_NAME, nhdr->n_namesz) != 0) | |
1102 | return -EINVAL; | |
1103 | if (nhdr->n_type != USDT_NOTE_TYPE) | |
1104 | return -EINVAL; | |
1105 | ||
1106 | /* sanity check USDT note contents ("description" in ELF terminology) */ | |
1107 | len = nhdr->n_descsz; | |
1108 | data = data + desc_off; | |
1109 | ||
1110 | /* +3 is the very minimum required to store three empty strings */ | |
1111 | if (len < sizeof(addrs) + 3) | |
1112 | return -EINVAL; | |
1113 | ||
1114 | /* get location, base, and semaphore addrs */ | |
1115 | memcpy(&addrs, data, sizeof(addrs)); | |
1116 | ||
1117 | /* parse string fields: provider, name, args */ | |
1118 | provider = data + sizeof(addrs); | |
1119 | ||
1120 | name = (const char *)memchr(provider, '\0', data + len - provider); | |
1121 | if (!name) /* non-zero-terminated provider */ | |
1122 | return -EINVAL; | |
1123 | name++; | |
1124 | if (name >= data + len || *name == '\0') /* missing or empty name */ | |
1125 | return -EINVAL; | |
1126 | ||
1127 | args = memchr(name, '\0', data + len - name); | |
1128 | if (!args) /* non-zero-terminated name */ | |
1129 | return -EINVAL; | |
1130 | ++args; | |
1131 | if (args >= data + len) /* missing arguments spec */ | |
1132 | return -EINVAL; | |
1133 | ||
1134 | note->provider = provider; | |
1135 | note->name = name; | |
1136 | if (*args == '\0' || *args == ':') | |
1137 | note->args = ""; | |
1138 | else | |
1139 | note->args = args; | |
1140 | note->loc_addr = addrs[0]; | |
1141 | note->base_addr = addrs[1]; | |
1142 | note->sema_addr = addrs[2]; | |
1143 | ||
1144 | return 0; | |
1145 | } | |
1146 | ||
1147 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg); | |
1148 | ||
5af25a41 | 1149 | static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie) |
74cc6311 AN |
1150 | { |
1151 | const char *s; | |
1152 | int len; | |
1153 | ||
1154 | spec->usdt_cookie = usdt_cookie; | |
1155 | spec->arg_cnt = 0; | |
1156 | ||
1157 | s = note->args; | |
1158 | while (s[0]) { | |
1159 | if (spec->arg_cnt >= USDT_MAX_ARG_CNT) { | |
1160 | pr_warn("usdt: too many USDT arguments (> %d) for '%s:%s' with args spec '%s'\n", | |
1161 | USDT_MAX_ARG_CNT, note->provider, note->name, note->args); | |
1162 | return -E2BIG; | |
1163 | } | |
1164 | ||
1165 | len = parse_usdt_arg(s, spec->arg_cnt, &spec->args[spec->arg_cnt]); | |
1166 | if (len < 0) | |
1167 | return len; | |
1168 | ||
1169 | s += len; | |
1170 | spec->arg_cnt++; | |
1171 | } | |
1172 | ||
1173 | return 0; | |
1174 | } | |
1175 | ||
4c59e584 AN |
1176 | /* Architecture-specific logic for parsing USDT argument location specs */ |
1177 | ||
1178 | #if defined(__x86_64__) || defined(__i386__) | |
1179 | ||
1180 | static int calc_pt_regs_off(const char *reg_name) | |
1181 | { | |
1182 | static struct { | |
1183 | const char *names[4]; | |
1184 | size_t pt_regs_off; | |
1185 | } reg_map[] = { | |
ded6dffa | 1186 | #ifdef __x86_64__ |
4c59e584 AN |
1187 | #define reg_off(reg64, reg32) offsetof(struct pt_regs, reg64) |
1188 | #else | |
1189 | #define reg_off(reg64, reg32) offsetof(struct pt_regs, reg32) | |
1190 | #endif | |
1191 | { {"rip", "eip", "", ""}, reg_off(rip, eip) }, | |
1192 | { {"rax", "eax", "ax", "al"}, reg_off(rax, eax) }, | |
1193 | { {"rbx", "ebx", "bx", "bl"}, reg_off(rbx, ebx) }, | |
1194 | { {"rcx", "ecx", "cx", "cl"}, reg_off(rcx, ecx) }, | |
1195 | { {"rdx", "edx", "dx", "dl"}, reg_off(rdx, edx) }, | |
1196 | { {"rsi", "esi", "si", "sil"}, reg_off(rsi, esi) }, | |
1197 | { {"rdi", "edi", "di", "dil"}, reg_off(rdi, edi) }, | |
1198 | { {"rbp", "ebp", "bp", "bpl"}, reg_off(rbp, ebp) }, | |
1199 | { {"rsp", "esp", "sp", "spl"}, reg_off(rsp, esp) }, | |
1200 | #undef reg_off | |
ded6dffa | 1201 | #ifdef __x86_64__ |
4c59e584 AN |
1202 | { {"r8", "r8d", "r8w", "r8b"}, offsetof(struct pt_regs, r8) }, |
1203 | { {"r9", "r9d", "r9w", "r9b"}, offsetof(struct pt_regs, r9) }, | |
1204 | { {"r10", "r10d", "r10w", "r10b"}, offsetof(struct pt_regs, r10) }, | |
1205 | { {"r11", "r11d", "r11w", "r11b"}, offsetof(struct pt_regs, r11) }, | |
1206 | { {"r12", "r12d", "r12w", "r12b"}, offsetof(struct pt_regs, r12) }, | |
1207 | { {"r13", "r13d", "r13w", "r13b"}, offsetof(struct pt_regs, r13) }, | |
1208 | { {"r14", "r14d", "r14w", "r14b"}, offsetof(struct pt_regs, r14) }, | |
1209 | { {"r15", "r15d", "r15w", "r15b"}, offsetof(struct pt_regs, r15) }, | |
1210 | #endif | |
1211 | }; | |
1212 | int i, j; | |
1213 | ||
1214 | for (i = 0; i < ARRAY_SIZE(reg_map); i++) { | |
1215 | for (j = 0; j < ARRAY_SIZE(reg_map[i].names); j++) { | |
1216 | if (strcmp(reg_name, reg_map[i].names[j]) == 0) | |
1217 | return reg_map[i].pt_regs_off; | |
1218 | } | |
1219 | } | |
1220 | ||
1221 | pr_warn("usdt: unrecognized register '%s'\n", reg_name); | |
1222 | return -ENOENT; | |
1223 | } | |
1224 | ||
1225 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg) | |
1226 | { | |
1227 | char *reg_name = NULL; | |
1228 | int arg_sz, len, reg_off; | |
1229 | long off; | |
1230 | ||
1231 | if (sscanf(arg_str, " %d @ %ld ( %%%m[^)] ) %n", &arg_sz, &off, ®_name, &len) == 3) { | |
1232 | /* Memory dereference case, e.g., -4@-20(%rbp) */ | |
1233 | arg->arg_type = USDT_ARG_REG_DEREF; | |
1234 | arg->val_off = off; | |
1235 | reg_off = calc_pt_regs_off(reg_name); | |
1236 | free(reg_name); | |
1237 | if (reg_off < 0) | |
1238 | return reg_off; | |
1239 | arg->reg_off = reg_off; | |
1240 | } else if (sscanf(arg_str, " %d @ %%%ms %n", &arg_sz, ®_name, &len) == 2) { | |
1241 | /* Register read case, e.g., -4@%eax */ | |
1242 | arg->arg_type = USDT_ARG_REG; | |
1243 | arg->val_off = 0; | |
1244 | ||
1245 | reg_off = calc_pt_regs_off(reg_name); | |
1246 | free(reg_name); | |
1247 | if (reg_off < 0) | |
1248 | return reg_off; | |
1249 | arg->reg_off = reg_off; | |
1250 | } else if (sscanf(arg_str, " %d @ $%ld %n", &arg_sz, &off, &len) == 2) { | |
1251 | /* Constant value case, e.g., 4@$71 */ | |
1252 | arg->arg_type = USDT_ARG_CONST; | |
1253 | arg->val_off = off; | |
1254 | arg->reg_off = 0; | |
1255 | } else { | |
1256 | pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); | |
1257 | return -EINVAL; | |
1258 | } | |
1259 | ||
1260 | arg->arg_signed = arg_sz < 0; | |
1261 | if (arg_sz < 0) | |
1262 | arg_sz = -arg_sz; | |
1263 | ||
1264 | switch (arg_sz) { | |
1265 | case 1: case 2: case 4: case 8: | |
1266 | arg->arg_bitshift = 64 - arg_sz * 8; | |
1267 | break; | |
1268 | default: | |
1269 | pr_warn("usdt: unsupported arg #%d (spec '%s') size: %d\n", | |
1270 | arg_num, arg_str, arg_sz); | |
1271 | return -EINVAL; | |
1272 | } | |
1273 | ||
1274 | return len; | |
1275 | } | |
1276 | ||
bd022685 IL |
1277 | #elif defined(__s390x__) |
1278 | ||
1279 | /* Do not support __s390__ for now, since user_pt_regs is broken with -m31. */ | |
1280 | ||
1281 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg) | |
1282 | { | |
1283 | unsigned int reg; | |
1284 | int arg_sz, len; | |
1285 | long off; | |
1286 | ||
1287 | if (sscanf(arg_str, " %d @ %ld ( %%r%u ) %n", &arg_sz, &off, ®, &len) == 3) { | |
1288 | /* Memory dereference case, e.g., -2@-28(%r15) */ | |
1289 | arg->arg_type = USDT_ARG_REG_DEREF; | |
1290 | arg->val_off = off; | |
1291 | if (reg > 15) { | |
1292 | pr_warn("usdt: unrecognized register '%%r%u'\n", reg); | |
1293 | return -EINVAL; | |
1294 | } | |
1295 | arg->reg_off = offsetof(user_pt_regs, gprs[reg]); | |
1296 | } else if (sscanf(arg_str, " %d @ %%r%u %n", &arg_sz, ®, &len) == 2) { | |
1297 | /* Register read case, e.g., -8@%r0 */ | |
1298 | arg->arg_type = USDT_ARG_REG; | |
1299 | arg->val_off = 0; | |
1300 | if (reg > 15) { | |
1301 | pr_warn("usdt: unrecognized register '%%r%u'\n", reg); | |
1302 | return -EINVAL; | |
1303 | } | |
1304 | arg->reg_off = offsetof(user_pt_regs, gprs[reg]); | |
1305 | } else if (sscanf(arg_str, " %d @ %ld %n", &arg_sz, &off, &len) == 2) { | |
1306 | /* Constant value case, e.g., 4@71 */ | |
1307 | arg->arg_type = USDT_ARG_CONST; | |
1308 | arg->val_off = off; | |
1309 | arg->reg_off = 0; | |
1310 | } else { | |
1311 | pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); | |
1312 | return -EINVAL; | |
1313 | } | |
1314 | ||
1315 | arg->arg_signed = arg_sz < 0; | |
1316 | if (arg_sz < 0) | |
1317 | arg_sz = -arg_sz; | |
1318 | ||
1319 | switch (arg_sz) { | |
1320 | case 1: case 2: case 4: case 8: | |
1321 | arg->arg_bitshift = 64 - arg_sz * 8; | |
1322 | break; | |
1323 | default: | |
1324 | pr_warn("usdt: unsupported arg #%d (spec '%s') size: %d\n", | |
1325 | arg_num, arg_str, arg_sz); | |
1326 | return -EINVAL; | |
1327 | } | |
1328 | ||
1329 | return len; | |
1330 | } | |
1331 | ||
0f861992 AM |
1332 | #elif defined(__aarch64__) |
1333 | ||
1334 | static int calc_pt_regs_off(const char *reg_name) | |
1335 | { | |
1336 | int reg_num; | |
1337 | ||
1338 | if (sscanf(reg_name, "x%d", ®_num) == 1) { | |
1339 | if (reg_num >= 0 && reg_num < 31) | |
1340 | return offsetof(struct user_pt_regs, regs[reg_num]); | |
1341 | } else if (strcmp(reg_name, "sp") == 0) { | |
1342 | return offsetof(struct user_pt_regs, sp); | |
1343 | } | |
1344 | pr_warn("usdt: unrecognized register '%s'\n", reg_name); | |
1345 | return -ENOENT; | |
1346 | } | |
1347 | ||
1348 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg) | |
1349 | { | |
1350 | char *reg_name = NULL; | |
1351 | int arg_sz, len, reg_off; | |
1352 | long off; | |
1353 | ||
1354 | if (sscanf(arg_str, " %d @ \[ %m[a-z0-9], %ld ] %n", &arg_sz, ®_name, &off, &len) == 3) { | |
1355 | /* Memory dereference case, e.g., -4@[sp, 96] */ | |
1356 | arg->arg_type = USDT_ARG_REG_DEREF; | |
1357 | arg->val_off = off; | |
1358 | reg_off = calc_pt_regs_off(reg_name); | |
1359 | free(reg_name); | |
1360 | if (reg_off < 0) | |
1361 | return reg_off; | |
1362 | arg->reg_off = reg_off; | |
1363 | } else if (sscanf(arg_str, " %d @ \[ %m[a-z0-9] ] %n", &arg_sz, ®_name, &len) == 2) { | |
1364 | /* Memory dereference case, e.g., -4@[sp] */ | |
1365 | arg->arg_type = USDT_ARG_REG_DEREF; | |
1366 | arg->val_off = 0; | |
1367 | reg_off = calc_pt_regs_off(reg_name); | |
1368 | free(reg_name); | |
1369 | if (reg_off < 0) | |
1370 | return reg_off; | |
1371 | arg->reg_off = reg_off; | |
1372 | } else if (sscanf(arg_str, " %d @ %ld %n", &arg_sz, &off, &len) == 2) { | |
1373 | /* Constant value case, e.g., 4@5 */ | |
1374 | arg->arg_type = USDT_ARG_CONST; | |
1375 | arg->val_off = off; | |
1376 | arg->reg_off = 0; | |
1377 | } else if (sscanf(arg_str, " %d @ %m[a-z0-9] %n", &arg_sz, ®_name, &len) == 2) { | |
1378 | /* Register read case, e.g., -8@x4 */ | |
1379 | arg->arg_type = USDT_ARG_REG; | |
1380 | arg->val_off = 0; | |
1381 | reg_off = calc_pt_regs_off(reg_name); | |
1382 | free(reg_name); | |
1383 | if (reg_off < 0) | |
1384 | return reg_off; | |
1385 | arg->reg_off = reg_off; | |
1386 | } else { | |
1387 | pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); | |
1388 | return -EINVAL; | |
1389 | } | |
1390 | ||
1391 | arg->arg_signed = arg_sz < 0; | |
1392 | if (arg_sz < 0) | |
1393 | arg_sz = -arg_sz; | |
1394 | ||
1395 | switch (arg_sz) { | |
1396 | case 1: case 2: case 4: case 8: | |
1397 | arg->arg_bitshift = 64 - arg_sz * 8; | |
1398 | break; | |
1399 | default: | |
1400 | pr_warn("usdt: unsupported arg #%d (spec '%s') size: %d\n", | |
1401 | arg_num, arg_str, arg_sz); | |
1402 | return -EINVAL; | |
1403 | } | |
1404 | ||
1405 | return len; | |
1406 | } | |
1407 | ||
58ca8b05 PL |
1408 | #elif defined(__riscv) |
1409 | ||
1410 | static int calc_pt_regs_off(const char *reg_name) | |
1411 | { | |
1412 | static struct { | |
1413 | const char *name; | |
1414 | size_t pt_regs_off; | |
1415 | } reg_map[] = { | |
1416 | { "ra", offsetof(struct user_regs_struct, ra) }, | |
1417 | { "sp", offsetof(struct user_regs_struct, sp) }, | |
1418 | { "gp", offsetof(struct user_regs_struct, gp) }, | |
1419 | { "tp", offsetof(struct user_regs_struct, tp) }, | |
1420 | { "a0", offsetof(struct user_regs_struct, a0) }, | |
1421 | { "a1", offsetof(struct user_regs_struct, a1) }, | |
1422 | { "a2", offsetof(struct user_regs_struct, a2) }, | |
1423 | { "a3", offsetof(struct user_regs_struct, a3) }, | |
1424 | { "a4", offsetof(struct user_regs_struct, a4) }, | |
1425 | { "a5", offsetof(struct user_regs_struct, a5) }, | |
1426 | { "a6", offsetof(struct user_regs_struct, a6) }, | |
1427 | { "a7", offsetof(struct user_regs_struct, a7) }, | |
1428 | { "s0", offsetof(struct user_regs_struct, s0) }, | |
1429 | { "s1", offsetof(struct user_regs_struct, s1) }, | |
1430 | { "s2", offsetof(struct user_regs_struct, s2) }, | |
1431 | { "s3", offsetof(struct user_regs_struct, s3) }, | |
1432 | { "s4", offsetof(struct user_regs_struct, s4) }, | |
1433 | { "s5", offsetof(struct user_regs_struct, s5) }, | |
1434 | { "s6", offsetof(struct user_regs_struct, s6) }, | |
1435 | { "s7", offsetof(struct user_regs_struct, s7) }, | |
1436 | { "s8", offsetof(struct user_regs_struct, rv_s8) }, | |
1437 | { "s9", offsetof(struct user_regs_struct, s9) }, | |
1438 | { "s10", offsetof(struct user_regs_struct, s10) }, | |
1439 | { "s11", offsetof(struct user_regs_struct, s11) }, | |
1440 | { "t0", offsetof(struct user_regs_struct, t0) }, | |
1441 | { "t1", offsetof(struct user_regs_struct, t1) }, | |
1442 | { "t2", offsetof(struct user_regs_struct, t2) }, | |
1443 | { "t3", offsetof(struct user_regs_struct, t3) }, | |
1444 | { "t4", offsetof(struct user_regs_struct, t4) }, | |
1445 | { "t5", offsetof(struct user_regs_struct, t5) }, | |
1446 | { "t6", offsetof(struct user_regs_struct, t6) }, | |
1447 | }; | |
1448 | int i; | |
1449 | ||
1450 | for (i = 0; i < ARRAY_SIZE(reg_map); i++) { | |
1451 | if (strcmp(reg_name, reg_map[i].name) == 0) | |
1452 | return reg_map[i].pt_regs_off; | |
1453 | } | |
1454 | ||
1455 | pr_warn("usdt: unrecognized register '%s'\n", reg_name); | |
1456 | return -ENOENT; | |
1457 | } | |
1458 | ||
1459 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg) | |
1460 | { | |
1461 | char *reg_name = NULL; | |
1462 | int arg_sz, len, reg_off; | |
1463 | long off; | |
1464 | ||
1465 | if (sscanf(arg_str, " %d @ %ld ( %m[a-z0-9] ) %n", &arg_sz, &off, ®_name, &len) == 3) { | |
1466 | /* Memory dereference case, e.g., -8@-88(s0) */ | |
1467 | arg->arg_type = USDT_ARG_REG_DEREF; | |
1468 | arg->val_off = off; | |
1469 | reg_off = calc_pt_regs_off(reg_name); | |
1470 | free(reg_name); | |
1471 | if (reg_off < 0) | |
1472 | return reg_off; | |
1473 | arg->reg_off = reg_off; | |
1474 | } else if (sscanf(arg_str, " %d @ %ld %n", &arg_sz, &off, &len) == 2) { | |
1475 | /* Constant value case, e.g., 4@5 */ | |
1476 | arg->arg_type = USDT_ARG_CONST; | |
1477 | arg->val_off = off; | |
1478 | arg->reg_off = 0; | |
1479 | } else if (sscanf(arg_str, " %d @ %m[a-z0-9] %n", &arg_sz, ®_name, &len) == 2) { | |
1480 | /* Register read case, e.g., -8@a1 */ | |
1481 | arg->arg_type = USDT_ARG_REG; | |
1482 | arg->val_off = 0; | |
1483 | reg_off = calc_pt_regs_off(reg_name); | |
1484 | free(reg_name); | |
1485 | if (reg_off < 0) | |
1486 | return reg_off; | |
1487 | arg->reg_off = reg_off; | |
1488 | } else { | |
1489 | pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str); | |
1490 | return -EINVAL; | |
1491 | } | |
1492 | ||
1493 | arg->arg_signed = arg_sz < 0; | |
1494 | if (arg_sz < 0) | |
1495 | arg_sz = -arg_sz; | |
1496 | ||
1497 | switch (arg_sz) { | |
1498 | case 1: case 2: case 4: case 8: | |
1499 | arg->arg_bitshift = 64 - arg_sz * 8; | |
1500 | break; | |
1501 | default: | |
1502 | pr_warn("usdt: unsupported arg #%d (spec '%s') size: %d\n", | |
1503 | arg_num, arg_str, arg_sz); | |
1504 | return -EINVAL; | |
1505 | } | |
1506 | ||
1507 | return len; | |
1508 | } | |
1509 | ||
4c59e584 AN |
1510 | #else |
1511 | ||
74cc6311 AN |
1512 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg) |
1513 | { | |
1514 | pr_warn("usdt: libbpf doesn't support USDTs on current architecture\n"); | |
1515 | return -ENOTSUP; | |
1516 | } | |
4c59e584 AN |
1517 | |
1518 | #endif |