Commit | Line | Data |
---|---|---|
fa853c4b SL |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | /* Copyright (c) 2019 Facebook */ | |
4 | ||
5 | #include <assert.h> | |
6 | #include <limits.h> | |
7 | #include <unistd.h> | |
7fac83aa | 8 | #include <sys/file.h> |
fa853c4b | 9 | #include <sys/time.h> |
fa853c4b SL |
10 | #include <linux/err.h> |
11 | #include <linux/zalloc.h> | |
7fac83aa | 12 | #include <api/fs/fs.h> |
ec8149fb | 13 | #include <perf/bpf_perf.h> |
fa853c4b SL |
14 | |
15 | #include "bpf_counter.h" | |
6ac22d03 | 16 | #include "bpf-utils.h" |
fa853c4b SL |
17 | #include "counts.h" |
18 | #include "debug.h" | |
19 | #include "evsel.h" | |
7fac83aa | 20 | #include "evlist.h" |
fa853c4b | 21 | #include "target.h" |
944138f0 | 22 | #include "cgroup.h" |
7fac83aa SL |
23 | #include "cpumap.h" |
24 | #include "thread_map.h" | |
fa853c4b SL |
25 | |
26 | #include "bpf_skel/bpf_prog_profiler.skel.h" | |
7fac83aa SL |
27 | #include "bpf_skel/bperf_u.h" |
28 | #include "bpf_skel/bperf_leader.skel.h" | |
29 | #include "bpf_skel/bperf_follower.skel.h" | |
30 | ||
7fac83aa | 31 | #define ATTR_MAP_SIZE 16 |
fa853c4b SL |
32 | |
33 | static inline void *u64_to_ptr(__u64 ptr) | |
34 | { | |
35 | return (void *)(unsigned long)ptr; | |
36 | } | |
37 | ||
fa853c4b SL |
38 | static struct bpf_counter *bpf_counter_alloc(void) |
39 | { | |
40 | struct bpf_counter *counter; | |
41 | ||
42 | counter = zalloc(sizeof(*counter)); | |
43 | if (counter) | |
44 | INIT_LIST_HEAD(&counter->list); | |
45 | return counter; | |
46 | } | |
47 | ||
48 | static int bpf_program_profiler__destroy(struct evsel *evsel) | |
49 | { | |
50 | struct bpf_counter *counter, *tmp; | |
51 | ||
52 | list_for_each_entry_safe(counter, tmp, | |
53 | &evsel->bpf_counter_list, list) { | |
54 | list_del_init(&counter->list); | |
55 | bpf_prog_profiler_bpf__destroy(counter->skel); | |
56 | free(counter); | |
57 | } | |
58 | assert(list_empty(&evsel->bpf_counter_list)); | |
59 | ||
60 | return 0; | |
61 | } | |
62 | ||
63 | static char *bpf_target_prog_name(int tgt_fd) | |
64 | { | |
fa853c4b | 65 | struct bpf_func_info *func_info; |
6ac22d03 | 66 | struct perf_bpil *info_linear; |
fa853c4b | 67 | const struct btf_type *t; |
369e955b | 68 | struct btf *btf = NULL; |
fa853c4b | 69 | char *name = NULL; |
fa853c4b | 70 | |
6ac22d03 | 71 | info_linear = get_bpf_prog_info_linear(tgt_fd, 1UL << PERF_BPIL_FUNC_INFO); |
fa853c4b SL |
72 | if (IS_ERR_OR_NULL(info_linear)) { |
73 | pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd); | |
74 | return NULL; | |
75 | } | |
76 | ||
86f4b7f2 | 77 | if (info_linear->info.btf_id == 0) { |
fa853c4b SL |
78 | pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd); |
79 | goto out; | |
80 | } | |
81 | ||
86f4b7f2 QM |
82 | btf = btf__load_from_kernel_by_id(info_linear->info.btf_id); |
83 | if (libbpf_get_error(btf)) { | |
84 | pr_debug("failed to load btf for prog FD %d\n", tgt_fd); | |
85 | goto out; | |
86 | } | |
87 | ||
fa853c4b SL |
88 | func_info = u64_to_ptr(info_linear->info.func_info); |
89 | t = btf__type_by_id(btf, func_info[0].type_id); | |
90 | if (!t) { | |
91 | pr_debug("btf %d doesn't have type %d\n", | |
92 | info_linear->info.btf_id, func_info[0].type_id); | |
93 | goto out; | |
94 | } | |
95 | name = strdup(btf__name_by_offset(btf, t->name_off)); | |
96 | out: | |
369e955b | 97 | btf__free(btf); |
fa853c4b SL |
98 | free(info_linear); |
99 | return name; | |
100 | } | |
101 | ||
102 | static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id) | |
103 | { | |
104 | struct bpf_prog_profiler_bpf *skel; | |
105 | struct bpf_counter *counter; | |
106 | struct bpf_program *prog; | |
b84b3f47 | 107 | char *prog_name = NULL; |
fa853c4b SL |
108 | int prog_fd; |
109 | int err; | |
110 | ||
111 | prog_fd = bpf_prog_get_fd_by_id(prog_id); | |
112 | if (prog_fd < 0) { | |
113 | pr_err("Failed to open fd for bpf prog %u\n", prog_id); | |
114 | return -1; | |
115 | } | |
116 | counter = bpf_counter_alloc(); | |
117 | if (!counter) { | |
118 | close(prog_fd); | |
119 | return -1; | |
120 | } | |
121 | ||
122 | skel = bpf_prog_profiler_bpf__open(); | |
123 | if (!skel) { | |
124 | pr_err("Failed to open bpf skeleton\n"); | |
125 | goto err_out; | |
126 | } | |
127 | ||
128 | skel->rodata->num_cpu = evsel__nr_cpus(evsel); | |
129 | ||
ddf0d4de MFW |
130 | bpf_map__set_max_entries(skel->maps.events, evsel__nr_cpus(evsel)); |
131 | bpf_map__set_max_entries(skel->maps.fentry_readings, 1); | |
132 | bpf_map__set_max_entries(skel->maps.accum_readings, 1); | |
fa853c4b SL |
133 | |
134 | prog_name = bpf_target_prog_name(prog_fd); | |
135 | if (!prog_name) { | |
136 | pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id); | |
137 | goto err_out; | |
138 | } | |
139 | ||
140 | bpf_object__for_each_program(prog, skel->obj) { | |
141 | err = bpf_program__set_attach_target(prog, prog_fd, prog_name); | |
142 | if (err) { | |
143 | pr_err("bpf_program__set_attach_target failed.\n" | |
144 | "Does bpf prog %u have BTF?\n", prog_id); | |
145 | goto err_out; | |
146 | } | |
147 | } | |
148 | set_max_rlimit(); | |
149 | err = bpf_prog_profiler_bpf__load(skel); | |
150 | if (err) { | |
151 | pr_err("bpf_prog_profiler_bpf__load failed\n"); | |
152 | goto err_out; | |
153 | } | |
154 | ||
155 | assert(skel != NULL); | |
156 | counter->skel = skel; | |
157 | list_add(&counter->list, &evsel->bpf_counter_list); | |
b84b3f47 | 158 | free(prog_name); |
fa853c4b SL |
159 | close(prog_fd); |
160 | return 0; | |
161 | err_out: | |
162 | bpf_prog_profiler_bpf__destroy(skel); | |
b84b3f47 | 163 | free(prog_name); |
fa853c4b SL |
164 | free(counter); |
165 | close(prog_fd); | |
166 | return -1; | |
167 | } | |
168 | ||
169 | static int bpf_program_profiler__load(struct evsel *evsel, struct target *target) | |
170 | { | |
171 | char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p; | |
172 | u32 prog_id; | |
173 | int ret; | |
174 | ||
175 | bpf_str_ = bpf_str = strdup(target->bpf_str); | |
176 | if (!bpf_str) | |
177 | return -1; | |
178 | ||
179 | while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) { | |
180 | prog_id = strtoul(tok, &p, 10); | |
181 | if (prog_id == 0 || prog_id == UINT_MAX || | |
182 | (*p != '\0' && *p != ',')) { | |
183 | pr_err("Failed to parse bpf prog ids %s\n", | |
184 | target->bpf_str); | |
b84b3f47 | 185 | free(bpf_str_); |
fa853c4b SL |
186 | return -1; |
187 | } | |
188 | ||
189 | ret = bpf_program_profiler_load_one(evsel, prog_id); | |
190 | if (ret) { | |
191 | bpf_program_profiler__destroy(evsel); | |
192 | free(bpf_str_); | |
193 | return -1; | |
194 | } | |
195 | bpf_str = NULL; | |
196 | } | |
197 | free(bpf_str_); | |
198 | return 0; | |
199 | } | |
200 | ||
201 | static int bpf_program_profiler__enable(struct evsel *evsel) | |
202 | { | |
203 | struct bpf_counter *counter; | |
204 | int ret; | |
205 | ||
206 | list_for_each_entry(counter, &evsel->bpf_counter_list, list) { | |
207 | assert(counter->skel != NULL); | |
208 | ret = bpf_prog_profiler_bpf__attach(counter->skel); | |
209 | if (ret) { | |
210 | bpf_program_profiler__destroy(evsel); | |
211 | return ret; | |
212 | } | |
213 | } | |
214 | return 0; | |
215 | } | |
216 | ||
5508c9da SL |
217 | static int bpf_program_profiler__disable(struct evsel *evsel) |
218 | { | |
219 | struct bpf_counter *counter; | |
220 | ||
221 | list_for_each_entry(counter, &evsel->bpf_counter_list, list) { | |
222 | assert(counter->skel != NULL); | |
223 | bpf_prog_profiler_bpf__detach(counter->skel); | |
224 | } | |
225 | return 0; | |
226 | } | |
227 | ||
fa853c4b SL |
228 | static int bpf_program_profiler__read(struct evsel *evsel) |
229 | { | |
fa853c4b SL |
230 | // BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible |
231 | // Sometimes possible > online, like on a Ryzen 3900X that has 24 | |
232 | // threads but its possible showed 0-31 -acme | |
233 | int num_cpu_bpf = libbpf_num_possible_cpus(); | |
234 | struct bpf_perf_event_value values[num_cpu_bpf]; | |
235 | struct bpf_counter *counter; | |
54668a4e | 236 | struct perf_counts_values *counts; |
fa853c4b SL |
237 | int reading_map_fd; |
238 | __u32 key = 0; | |
54668a4e | 239 | int err, idx, bpf_cpu; |
fa853c4b SL |
240 | |
241 | if (list_empty(&evsel->bpf_counter_list)) | |
242 | return -EAGAIN; | |
243 | ||
54668a4e IR |
244 | perf_cpu_map__for_each_idx(idx, evsel__cpus(evsel)) { |
245 | counts = perf_counts(evsel->counts, idx, 0); | |
246 | counts->val = 0; | |
247 | counts->ena = 0; | |
248 | counts->run = 0; | |
fa853c4b SL |
249 | } |
250 | list_for_each_entry(counter, &evsel->bpf_counter_list, list) { | |
251 | struct bpf_prog_profiler_bpf *skel = counter->skel; | |
252 | ||
253 | assert(skel != NULL); | |
254 | reading_map_fd = bpf_map__fd(skel->maps.accum_readings); | |
255 | ||
256 | err = bpf_map_lookup_elem(reading_map_fd, &key, values); | |
257 | if (err) { | |
258 | pr_err("failed to read value\n"); | |
259 | return err; | |
260 | } | |
261 | ||
54668a4e IR |
262 | for (bpf_cpu = 0; bpf_cpu < num_cpu_bpf; bpf_cpu++) { |
263 | idx = perf_cpu_map__idx(evsel__cpus(evsel), | |
264 | (struct perf_cpu){.cpu = bpf_cpu}); | |
265 | if (idx == -1) | |
266 | continue; | |
267 | counts = perf_counts(evsel->counts, idx, 0); | |
268 | counts->val += values[bpf_cpu].counter; | |
269 | counts->ena += values[bpf_cpu].enabled; | |
270 | counts->run += values[bpf_cpu].running; | |
fa853c4b SL |
271 | } |
272 | } | |
273 | return 0; | |
274 | } | |
275 | ||
7263f349 | 276 | static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu_map_idx, |
fa853c4b SL |
277 | int fd) |
278 | { | |
279 | struct bpf_prog_profiler_bpf *skel; | |
280 | struct bpf_counter *counter; | |
281 | int ret; | |
282 | ||
283 | list_for_each_entry(counter, &evsel->bpf_counter_list, list) { | |
284 | skel = counter->skel; | |
285 | assert(skel != NULL); | |
286 | ||
287 | ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events), | |
7263f349 | 288 | &cpu_map_idx, &fd, BPF_ANY); |
fa853c4b SL |
289 | if (ret) |
290 | return ret; | |
291 | } | |
292 | return 0; | |
293 | } | |
294 | ||
295 | struct bpf_counter_ops bpf_program_profiler_ops = { | |
296 | .load = bpf_program_profiler__load, | |
297 | .enable = bpf_program_profiler__enable, | |
5508c9da | 298 | .disable = bpf_program_profiler__disable, |
fa853c4b SL |
299 | .read = bpf_program_profiler__read, |
300 | .destroy = bpf_program_profiler__destroy, | |
301 | .install_pe = bpf_program_profiler__install_pe, | |
302 | }; | |
303 | ||
fe3dd826 SL |
304 | static bool bperf_attr_map_compatible(int attr_map_fd) |
305 | { | |
306 | struct bpf_map_info map_info = {0}; | |
307 | __u32 map_info_len = sizeof(map_info); | |
308 | int err; | |
309 | ||
310 | err = bpf_obj_get_info_by_fd(attr_map_fd, &map_info, &map_info_len); | |
311 | ||
312 | if (err) | |
313 | return false; | |
314 | return (map_info.key_size == sizeof(struct perf_event_attr)) && | |
315 | (map_info.value_size == sizeof(struct perf_event_attr_map_entry)); | |
316 | } | |
317 | ||
7fac83aa SL |
318 | static int bperf_lock_attr_map(struct target *target) |
319 | { | |
320 | char path[PATH_MAX]; | |
321 | int map_fd, err; | |
322 | ||
323 | if (target->attr_map) { | |
324 | scnprintf(path, PATH_MAX, "%s", target->attr_map); | |
325 | } else { | |
ec8149fb SL |
326 | scnprintf(path, PATH_MAX, "%s/fs/bpf/%s", sysfs__mountpoint(), |
327 | BPF_PERF_DEFAULT_ATTR_MAP_PATH); | |
7fac83aa SL |
328 | } |
329 | ||
330 | if (access(path, F_OK)) { | |
8d0f9e73 | 331 | map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, |
7fac83aa SL |
332 | sizeof(struct perf_event_attr), |
333 | sizeof(struct perf_event_attr_map_entry), | |
8d0f9e73 | 334 | ATTR_MAP_SIZE, NULL); |
7fac83aa SL |
335 | if (map_fd < 0) |
336 | return -1; | |
337 | ||
338 | err = bpf_obj_pin(map_fd, path); | |
339 | if (err) { | |
340 | /* someone pinned the map in parallel? */ | |
341 | close(map_fd); | |
342 | map_fd = bpf_obj_get(path); | |
343 | if (map_fd < 0) | |
344 | return -1; | |
345 | } | |
346 | } else { | |
347 | map_fd = bpf_obj_get(path); | |
348 | if (map_fd < 0) | |
349 | return -1; | |
350 | } | |
351 | ||
fe3dd826 SL |
352 | if (!bperf_attr_map_compatible(map_fd)) { |
353 | close(map_fd); | |
354 | return -1; | |
355 | ||
356 | } | |
7fac83aa SL |
357 | err = flock(map_fd, LOCK_EX); |
358 | if (err) { | |
359 | close(map_fd); | |
360 | return -1; | |
361 | } | |
362 | return map_fd; | |
363 | } | |
364 | ||
7fac83aa SL |
365 | static int bperf_check_target(struct evsel *evsel, |
366 | struct target *target, | |
367 | enum bperf_filter_type *filter_type, | |
368 | __u32 *filter_entry_cnt) | |
369 | { | |
fba7c866 | 370 | if (evsel->core.leader->nr_members > 1) { |
7fac83aa SL |
371 | pr_err("bpf managed perf events do not yet support groups.\n"); |
372 | return -1; | |
373 | } | |
374 | ||
375 | /* determine filter type based on target */ | |
376 | if (target->system_wide) { | |
377 | *filter_type = BPERF_FILTER_GLOBAL; | |
378 | *filter_entry_cnt = 1; | |
379 | } else if (target->cpu_list) { | |
380 | *filter_type = BPERF_FILTER_CPU; | |
381 | *filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel)); | |
382 | } else if (target->tid) { | |
383 | *filter_type = BPERF_FILTER_PID; | |
384 | *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); | |
385 | } else if (target->pid || evsel->evlist->workload.pid != -1) { | |
386 | *filter_type = BPERF_FILTER_TGID; | |
387 | *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); | |
388 | } else { | |
389 | pr_err("bpf managed perf events do not yet support these targets.\n"); | |
390 | return -1; | |
391 | } | |
392 | ||
393 | return 0; | |
394 | } | |
395 | ||
396 | static struct perf_cpu_map *all_cpu_map; | |
397 | ||
398 | static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd, | |
399 | struct perf_event_attr_map_entry *entry) | |
400 | { | |
401 | struct bperf_leader_bpf *skel = bperf_leader_bpf__open(); | |
402 | int link_fd, diff_map_fd, err; | |
403 | struct bpf_link *link = NULL; | |
404 | ||
405 | if (!skel) { | |
406 | pr_err("Failed to open leader skeleton\n"); | |
407 | return -1; | |
408 | } | |
409 | ||
ddf0d4de | 410 | bpf_map__set_max_entries(skel->maps.events, libbpf_num_possible_cpus()); |
7fac83aa SL |
411 | err = bperf_leader_bpf__load(skel); |
412 | if (err) { | |
413 | pr_err("Failed to load leader skeleton\n"); | |
414 | goto out; | |
415 | } | |
416 | ||
7fac83aa | 417 | link = bpf_program__attach(skel->progs.on_switch); |
c673b7f5 | 418 | if (IS_ERR(link)) { |
7fac83aa | 419 | pr_err("Failed to attach leader program\n"); |
c673b7f5 | 420 | err = PTR_ERR(link); |
7fac83aa SL |
421 | goto out; |
422 | } | |
423 | ||
424 | link_fd = bpf_link__fd(link); | |
425 | diff_map_fd = bpf_map__fd(skel->maps.diff_readings); | |
426 | entry->link_id = bpf_link_get_id(link_fd); | |
427 | entry->diff_map_id = bpf_map_get_id(diff_map_fd); | |
428 | err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY); | |
429 | assert(err == 0); | |
430 | ||
431 | evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id); | |
432 | assert(evsel->bperf_leader_link_fd >= 0); | |
433 | ||
434 | /* | |
435 | * save leader_skel for install_pe, which is called within | |
436 | * following evsel__open_per_cpu call | |
437 | */ | |
438 | evsel->leader_skel = skel; | |
439 | evsel__open_per_cpu(evsel, all_cpu_map, -1); | |
440 | ||
441 | out: | |
442 | bperf_leader_bpf__destroy(skel); | |
443 | bpf_link__destroy(link); | |
444 | return err; | |
445 | } | |
446 | ||
447 | static int bperf__load(struct evsel *evsel, struct target *target) | |
448 | { | |
449 | struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff}; | |
450 | int attr_map_fd, diff_map_fd = -1, err; | |
451 | enum bperf_filter_type filter_type; | |
452 | __u32 filter_entry_cnt, i; | |
453 | ||
454 | if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt)) | |
455 | return -1; | |
456 | ||
457 | if (!all_cpu_map) { | |
effe957c | 458 | all_cpu_map = perf_cpu_map__new_online_cpus(); |
7fac83aa SL |
459 | if (!all_cpu_map) |
460 | return -1; | |
461 | } | |
462 | ||
463 | evsel->bperf_leader_prog_fd = -1; | |
464 | evsel->bperf_leader_link_fd = -1; | |
465 | ||
466 | /* | |
467 | * Step 1: hold a fd on the leader program and the bpf_link, if | |
468 | * the program is not already gone, reload the program. | |
469 | * Use flock() to ensure exclusive access to the perf_event_attr | |
470 | * map. | |
471 | */ | |
472 | attr_map_fd = bperf_lock_attr_map(target); | |
473 | if (attr_map_fd < 0) { | |
474 | pr_err("Failed to lock perf_event_attr map\n"); | |
475 | return -1; | |
476 | } | |
477 | ||
478 | err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry); | |
479 | if (err) { | |
480 | err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY); | |
481 | if (err) | |
482 | goto out; | |
483 | } | |
484 | ||
485 | evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id); | |
486 | if (evsel->bperf_leader_link_fd < 0 && | |
d3fddc35 YK |
487 | bperf_reload_leader_program(evsel, attr_map_fd, &entry)) { |
488 | err = -1; | |
7fac83aa | 489 | goto out; |
d3fddc35 | 490 | } |
7fac83aa SL |
491 | /* |
492 | * The bpf_link holds reference to the leader program, and the | |
493 | * leader program holds reference to the maps. Therefore, if | |
494 | * link_id is valid, diff_map_id should also be valid. | |
495 | */ | |
496 | evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id( | |
497 | bpf_link_get_prog_id(evsel->bperf_leader_link_fd)); | |
498 | assert(evsel->bperf_leader_prog_fd >= 0); | |
499 | ||
500 | diff_map_fd = bpf_map_get_fd_by_id(entry.diff_map_id); | |
501 | assert(diff_map_fd >= 0); | |
502 | ||
503 | /* | |
504 | * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check | |
505 | * whether the kernel support it | |
506 | */ | |
507 | err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0); | |
508 | if (err) { | |
509 | pr_err("The kernel does not support test_run for raw_tp BPF programs.\n" | |
510 | "Therefore, --use-bpf might show inaccurate readings\n"); | |
511 | goto out; | |
512 | } | |
513 | ||
514 | /* Step 2: load the follower skeleton */ | |
515 | evsel->follower_skel = bperf_follower_bpf__open(); | |
516 | if (!evsel->follower_skel) { | |
d3fddc35 | 517 | err = -1; |
7fac83aa SL |
518 | pr_err("Failed to open follower skeleton\n"); |
519 | goto out; | |
520 | } | |
521 | ||
522 | /* attach fexit program to the leader program */ | |
523 | bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX, | |
524 | evsel->bperf_leader_prog_fd, "on_switch"); | |
525 | ||
526 | /* connect to leader diff_reading map */ | |
527 | bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd); | |
528 | ||
529 | /* set up reading map */ | |
530 | bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings, | |
531 | filter_entry_cnt); | |
532 | /* set up follower filter based on target */ | |
533 | bpf_map__set_max_entries(evsel->follower_skel->maps.filter, | |
534 | filter_entry_cnt); | |
535 | err = bperf_follower_bpf__load(evsel->follower_skel); | |
536 | if (err) { | |
537 | pr_err("Failed to load follower skeleton\n"); | |
538 | bperf_follower_bpf__destroy(evsel->follower_skel); | |
539 | evsel->follower_skel = NULL; | |
540 | goto out; | |
541 | } | |
542 | ||
543 | for (i = 0; i < filter_entry_cnt; i++) { | |
544 | int filter_map_fd; | |
545 | __u32 key; | |
546 | ||
547 | if (filter_type == BPERF_FILTER_PID || | |
548 | filter_type == BPERF_FILTER_TGID) | |
fd3f518f | 549 | key = perf_thread_map__pid(evsel->core.threads, i); |
7fac83aa | 550 | else if (filter_type == BPERF_FILTER_CPU) |
39b5e434 | 551 | key = perf_cpu_map__cpu(evsel->core.cpus, i).cpu; |
7fac83aa SL |
552 | else |
553 | break; | |
554 | ||
555 | filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter); | |
556 | bpf_map_update_elem(filter_map_fd, &key, &i, BPF_ANY); | |
557 | } | |
558 | ||
559 | evsel->follower_skel->bss->type = filter_type; | |
560 | ||
561 | err = bperf_follower_bpf__attach(evsel->follower_skel); | |
562 | ||
563 | out: | |
564 | if (err && evsel->bperf_leader_link_fd >= 0) | |
565 | close(evsel->bperf_leader_link_fd); | |
566 | if (err && evsel->bperf_leader_prog_fd >= 0) | |
567 | close(evsel->bperf_leader_prog_fd); | |
568 | if (diff_map_fd >= 0) | |
569 | close(diff_map_fd); | |
570 | ||
571 | flock(attr_map_fd, LOCK_UN); | |
572 | close(attr_map_fd); | |
573 | ||
574 | return err; | |
575 | } | |
576 | ||
7263f349 | 577 | static int bperf__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) |
7fac83aa SL |
578 | { |
579 | struct bperf_leader_bpf *skel = evsel->leader_skel; | |
580 | ||
581 | return bpf_map_update_elem(bpf_map__fd(skel->maps.events), | |
7263f349 | 582 | &cpu_map_idx, &fd, BPF_ANY); |
7fac83aa SL |
583 | } |
584 | ||
585 | /* | |
586 | * trigger the leader prog on each cpu, so the accum_reading map could get | |
587 | * the latest readings. | |
588 | */ | |
589 | static int bperf_sync_counters(struct evsel *evsel) | |
590 | { | |
591 | int num_cpu, i, cpu; | |
592 | ||
39b5e434 | 593 | num_cpu = perf_cpu_map__nr(all_cpu_map); |
7fac83aa | 594 | for (i = 0; i < num_cpu; i++) { |
39b5e434 | 595 | cpu = perf_cpu_map__cpu(all_cpu_map, i).cpu; |
7fac83aa SL |
596 | bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu); |
597 | } | |
598 | return 0; | |
599 | } | |
600 | ||
601 | static int bperf__enable(struct evsel *evsel) | |
602 | { | |
603 | evsel->follower_skel->bss->enabled = 1; | |
604 | return 0; | |
605 | } | |
606 | ||
5508c9da SL |
607 | static int bperf__disable(struct evsel *evsel) |
608 | { | |
609 | evsel->follower_skel->bss->enabled = 0; | |
610 | return 0; | |
611 | } | |
612 | ||
7fac83aa SL |
613 | static int bperf__read(struct evsel *evsel) |
614 | { | |
615 | struct bperf_follower_bpf *skel = evsel->follower_skel; | |
6d18804b | 616 | __u32 num_cpu_bpf = cpu__max_cpu().cpu; |
7fac83aa | 617 | struct bpf_perf_event_value values[num_cpu_bpf]; |
54668a4e | 618 | struct perf_counts_values *counts; |
7fac83aa | 619 | int reading_map_fd, err = 0; |
7263f349 IR |
620 | __u32 i; |
621 | int j; | |
7fac83aa SL |
622 | |
623 | bperf_sync_counters(evsel); | |
624 | reading_map_fd = bpf_map__fd(skel->maps.accum_readings); | |
625 | ||
626 | for (i = 0; i < bpf_map__max_entries(skel->maps.accum_readings); i++) { | |
6d18804b | 627 | struct perf_cpu entry; |
7fac83aa SL |
628 | __u32 cpu; |
629 | ||
630 | err = bpf_map_lookup_elem(reading_map_fd, &i, values); | |
631 | if (err) | |
632 | goto out; | |
633 | switch (evsel->follower_skel->bss->type) { | |
634 | case BPERF_FILTER_GLOBAL: | |
635 | assert(i == 0); | |
636 | ||
54668a4e IR |
637 | perf_cpu_map__for_each_cpu(entry, j, evsel__cpus(evsel)) { |
638 | counts = perf_counts(evsel->counts, j, 0); | |
639 | counts->val = values[entry.cpu].counter; | |
640 | counts->ena = values[entry.cpu].enabled; | |
641 | counts->run = values[entry.cpu].running; | |
7fac83aa SL |
642 | } |
643 | break; | |
644 | case BPERF_FILTER_CPU: | |
54668a4e IR |
645 | cpu = perf_cpu_map__cpu(evsel__cpus(evsel), i).cpu; |
646 | assert(cpu >= 0); | |
647 | counts = perf_counts(evsel->counts, i, 0); | |
648 | counts->val = values[cpu].counter; | |
649 | counts->ena = values[cpu].enabled; | |
650 | counts->run = values[cpu].running; | |
7fac83aa SL |
651 | break; |
652 | case BPERF_FILTER_PID: | |
653 | case BPERF_FILTER_TGID: | |
54668a4e IR |
654 | counts = perf_counts(evsel->counts, 0, i); |
655 | counts->val = 0; | |
656 | counts->ena = 0; | |
657 | counts->run = 0; | |
7fac83aa SL |
658 | |
659 | for (cpu = 0; cpu < num_cpu_bpf; cpu++) { | |
54668a4e IR |
660 | counts->val += values[cpu].counter; |
661 | counts->ena += values[cpu].enabled; | |
662 | counts->run += values[cpu].running; | |
7fac83aa SL |
663 | } |
664 | break; | |
665 | default: | |
666 | break; | |
667 | } | |
668 | } | |
669 | out: | |
670 | return err; | |
671 | } | |
672 | ||
673 | static int bperf__destroy(struct evsel *evsel) | |
674 | { | |
675 | bperf_follower_bpf__destroy(evsel->follower_skel); | |
676 | close(evsel->bperf_leader_prog_fd); | |
677 | close(evsel->bperf_leader_link_fd); | |
678 | return 0; | |
679 | } | |
680 | ||
681 | /* | |
682 | * bperf: share hardware PMCs with BPF | |
683 | * | |
684 | * perf uses performance monitoring counters (PMC) to monitor system | |
685 | * performance. The PMCs are limited hardware resources. For example, | |
686 | * Intel CPUs have 3x fixed PMCs and 4x programmable PMCs per cpu. | |
687 | * | |
688 | * Modern data center systems use these PMCs in many different ways: | |
689 | * system level monitoring, (maybe nested) container level monitoring, per | |
690 | * process monitoring, profiling (in sample mode), etc. In some cases, | |
691 | * there are more active perf_events than available hardware PMCs. To allow | |
692 | * all perf_events to have a chance to run, it is necessary to do expensive | |
693 | * time multiplexing of events. | |
694 | * | |
695 | * On the other hand, many monitoring tools count the common metrics | |
696 | * (cycles, instructions). It is a waste to have multiple tools create | |
697 | * multiple perf_events of "cycles" and occupy multiple PMCs. | |
698 | * | |
699 | * bperf tries to reduce such wastes by allowing multiple perf_events of | |
700 | * "cycles" or "instructions" (at different scopes) to share PMUs. Instead | |
701 | * of having each perf-stat session to read its own perf_events, bperf uses | |
702 | * BPF programs to read the perf_events and aggregate readings to BPF maps. | |
703 | * Then, the perf-stat session(s) reads the values from these BPF maps. | |
704 | * | |
705 | * || | |
706 | * shared progs and maps <- || -> per session progs and maps | |
707 | * || | |
708 | * --------------- || | |
709 | * | perf_events | || | |
710 | * --------------- fexit || ----------------- | |
711 | * | --------||----> | follower prog | | |
712 | * --------------- / || --- ----------------- | |
713 | * cs -> | leader prog |/ ||/ | | | |
714 | * --> --------------- /|| -------------- ------------------ | |
715 | * / | | / || | filter map | | accum_readings | | |
716 | * / ------------ ------------ || -------------- ------------------ | |
717 | * | | prev map | | diff map | || | | |
718 | * | ------------ ------------ || | | |
719 | * \ || | | |
720 | * = \ ==================================================== | ============ | |
721 | * \ / user space | |
722 | * \ / | |
723 | * \ / | |
724 | * BPF_PROG_TEST_RUN BPF_MAP_LOOKUP_ELEM | |
725 | * \ / | |
726 | * \ / | |
727 | * \------ perf-stat ----------------------/ | |
728 | * | |
729 | * The figure above shows the architecture of bperf. Note that the figure | |
730 | * is divided into 3 regions: shared progs and maps (top left), per session | |
731 | * progs and maps (top right), and user space (bottom). | |
732 | * | |
733 | * The leader prog is triggered on each context switch (cs). The leader | |
734 | * prog reads perf_events and stores the difference (current_reading - | |
735 | * previous_reading) to the diff map. For the same metric, e.g. "cycles", | |
736 | * multiple perf-stat sessions share the same leader prog. | |
737 | * | |
738 | * Each perf-stat session creates a follower prog as fexit program to the | |
739 | * leader prog. It is possible to attach up to BPF_MAX_TRAMP_PROGS (38) | |
740 | * follower progs to the same leader prog. The follower prog checks current | |
741 | * task and processor ID to decide whether to add the value from the diff | |
742 | * map to its accumulated reading map (accum_readings). | |
743 | * | |
744 | * Finally, perf-stat user space reads the value from accum_reading map. | |
745 | * | |
746 | * Besides context switch, it is also necessary to trigger the leader prog | |
747 | * before perf-stat reads the value. Otherwise, the accum_reading map may | |
748 | * not have the latest reading from the perf_events. This is achieved by | |
749 | * triggering the event via sys_bpf(BPF_PROG_TEST_RUN) to each CPU. | |
750 | * | |
751 | * Comment before the definition of struct perf_event_attr_map_entry | |
752 | * describes how different sessions of perf-stat share information about | |
753 | * the leader prog. | |
754 | */ | |
755 | ||
756 | struct bpf_counter_ops bperf_ops = { | |
757 | .load = bperf__load, | |
758 | .enable = bperf__enable, | |
5508c9da | 759 | .disable = bperf__disable, |
7fac83aa SL |
760 | .read = bperf__read, |
761 | .install_pe = bperf__install_pe, | |
762 | .destroy = bperf__destroy, | |
763 | }; | |
764 | ||
944138f0 NK |
765 | extern struct bpf_counter_ops bperf_cgrp_ops; |
766 | ||
7fac83aa SL |
767 | static inline bool bpf_counter_skip(struct evsel *evsel) |
768 | { | |
d180aa56 | 769 | return evsel->bpf_counter_ops == NULL; |
7fac83aa SL |
770 | } |
771 | ||
7263f349 | 772 | int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) |
fa853c4b | 773 | { |
7fac83aa | 774 | if (bpf_counter_skip(evsel)) |
fa853c4b | 775 | return 0; |
7263f349 | 776 | return evsel->bpf_counter_ops->install_pe(evsel, cpu_map_idx, fd); |
fa853c4b SL |
777 | } |
778 | ||
779 | int bpf_counter__load(struct evsel *evsel, struct target *target) | |
780 | { | |
7fac83aa | 781 | if (target->bpf_str) |
fa853c4b | 782 | evsel->bpf_counter_ops = &bpf_program_profiler_ops; |
944138f0 NK |
783 | else if (cgrp_event_expanded && target->use_bpf) |
784 | evsel->bpf_counter_ops = &bperf_cgrp_ops; | |
01bd8efc | 785 | else if (target->use_bpf || evsel->bpf_counter || |
112cb561 | 786 | evsel__match_bpf_counter_events(evsel->name)) |
7fac83aa | 787 | evsel->bpf_counter_ops = &bperf_ops; |
fa853c4b SL |
788 | |
789 | if (evsel->bpf_counter_ops) | |
790 | return evsel->bpf_counter_ops->load(evsel, target); | |
791 | return 0; | |
792 | } | |
793 | ||
794 | int bpf_counter__enable(struct evsel *evsel) | |
795 | { | |
7fac83aa | 796 | if (bpf_counter_skip(evsel)) |
fa853c4b SL |
797 | return 0; |
798 | return evsel->bpf_counter_ops->enable(evsel); | |
799 | } | |
800 | ||
5508c9da SL |
801 | int bpf_counter__disable(struct evsel *evsel) |
802 | { | |
803 | if (bpf_counter_skip(evsel)) | |
804 | return 0; | |
805 | return evsel->bpf_counter_ops->disable(evsel); | |
806 | } | |
807 | ||
fa853c4b SL |
808 | int bpf_counter__read(struct evsel *evsel) |
809 | { | |
7fac83aa | 810 | if (bpf_counter_skip(evsel)) |
fa853c4b SL |
811 | return -EAGAIN; |
812 | return evsel->bpf_counter_ops->read(evsel); | |
813 | } | |
814 | ||
815 | void bpf_counter__destroy(struct evsel *evsel) | |
816 | { | |
7fac83aa | 817 | if (bpf_counter_skip(evsel)) |
fa853c4b SL |
818 | return; |
819 | evsel->bpf_counter_ops->destroy(evsel); | |
820 | evsel->bpf_counter_ops = NULL; | |
e0137336 | 821 | evsel->bpf_skel = NULL; |
fa853c4b | 822 | } |