projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tools/Makefile: remove cgroup target
[linux-2.6-block.git] / tools / perf / builtin-inject.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
454c407e
TZ		 2/*
3 * builtin-inject.c
4 *
5 * Builtin inject command: Examine the live mode (stdin) event stream
6 * and repipe it to stdout while optionally injecting additional
7 * events into it.
8 */
9#include "builtin.h"
10
26a031e1 11#include "util/color.h"
4a3cec84 12#include "util/dso.h"
27c9c342 13#include "util/vdso.h"
26a031e1
AV		 14#include "util/evlist.h"
15#include "util/evsel.h"
1101f69a 16#include "util/map.h"
454c407e 17#include "util/session.h"
45694aa7 18#include "util/tool.h"
454c407e 19#include "util/debug.h"
54a3cf59 20#include "util/build-id.h"
f5fc1412 21#include "util/data.h"
0f0aa5e0 22#include "util/auxtrace.h"
9b07e27f 23#include "util/jit.h"
8012243e 24#include "util/string2.h"
daecf9e0 25#include "util/symbol.h"
ea49e01c 26#include "util/synthetic-events.h"
e7ff8920 27#include "util/thread.h"
336c95b2 28#include "util/namespaces.h"
75d48c56 29#include "util/util.h"
97406a7e 30#include "util/tsc.h"
454c407e 31
180b3d06
AH		 32#include <internal/lib.h>
33
e7b60c5a 34#include <linux/err.h>
4b6ab94e 35#include <subcmd/parse-options.h>
e7b60c5a 36#include <uapi/linux/mman.h> /* To get things like MAP_HUGETLB even on older libc headers */
454c407e 37
26a031e1 38#include <linux/list.h>
83d7f5f1 39#include <linux/string.h>
97406a7e
AH		 40#include <linux/zalloc.h>
41#include <linux/hash.h>
8012243e 42#include <ctype.h>
a43783ae 43#include <errno.h>
9607ad3a 44#include <signal.h>
97406a7e
AH		 45#include <inttypes.h>
46
47struct guest_event {
48 struct perf_sample sample;
49 union perf_event *event;
892d00fb 50 char *event_buf;
97406a7e
AH		 51};
52
53struct guest_id {
54 /* hlist_node must be first, see free_hlist() */
55 struct hlist_node node;
56 u64 id;
57 u64 host_id;
58 u32 vcpu;
59};
60
61struct guest_tid {
62 /* hlist_node must be first, see free_hlist() */
63 struct hlist_node node;
64 /* Thread ID of QEMU thread */
65 u32 tid;
66 u32 vcpu;
67};
68
69struct guest_vcpu {
70 /* Current host CPU */
71 u32 cpu;
72 /* Thread ID of QEMU thread */
73 u32 tid;
74};
75
76struct guest_session {
77 char *perf_data_file;
78 u32 machine_pid;
79 u64 time_offset;
80 double time_scale;
81 struct perf_tool tool;
82 struct perf_data data;
83 struct perf_session *session;
84 char *tmp_file_name;
85 int tmp_fd;
86 struct perf_tsc_conversion host_tc;
87 struct perf_tsc_conversion guest_tc;
88 bool copy_kcore_dir;
89 bool have_tc;
90 bool fetched;
91 bool ready;
92 u16 dflt_id_hdr_size;
93 u64 dflt_id;
94 u64 highest_id;
95 /* Array of guest_vcpu */
96 struct guest_vcpu *vcpu;
97 size_t vcpu_cnt;
98 /* Hash table for guest_id */
99 struct hlist_head heads[PERF_EVLIST__HLIST_SIZE];
100 /* Hash table for guest_tid */
101 struct hlist_head tids[PERF_EVLIST__HLIST_SIZE];
102 /* Place to stash next guest event */
103 struct guest_event ev;
104};
26a031e1 105
5ded57ac 106struct perf_inject {
3406912c 107 struct perf_tool tool;
1cb8bdcc 108 struct perf_session *session;
3406912c 109 bool build_ids;
27c9c342 110 bool build_id_all;
3406912c 111 bool sched_stat;
cd10b289 112 bool have_auxtrace;
f56fb986 113 bool strip;
9b07e27f 114 bool jit_mode;
2a525f6a
AH		 115 bool in_place_update;
116 bool in_place_update_dry_run;
fea20d66 117 bool is_pipe;
d8fc0855 118 bool copy_kcore_dir;
3406912c 119 const char *input_name;
8ceb41d7 120 struct perf_data output;
3406912c 121 u64 bytes_written;
73117308 122 u64 aux_id;
3406912c 123 struct list_head samples;
0f0aa5e0 124 struct itrace_synth_opts itrace_synth_opts;
d3944f0e 125 char *event_copy;
180b3d06 126 struct perf_file_section secs[HEADER_FEAT_BITS];
97406a7e 127 struct guest_session guest_session;
8012243e 128 struct strlist *known_build_ids;
26a031e1
AV		 129};
130
131struct event_entry {
132 struct list_head node;
133 u32 tid;
6549a8c0 134 union perf_event event[];
5ded57ac 135};
454c407e 136
27c9c342
NK		 137static int dso__inject_build_id(struct dso *dso, struct perf_tool *tool,
138 struct machine *machine, u8 cpumode, u32 flags);
139
cd17a9b5 140static int output_bytes(struct perf_inject *inject, void *buf, size_t sz)
454c407e 141{
3406912c 142 ssize_t size;
454c407e 143
8ceb41d7 144 size = perf_data__write(&inject->output, buf, sz);
3406912c
JO		 145 if (size < 0)
146 return -errno;
454c407e 147
3406912c 148 inject->bytes_written += size;
454c407e
TZ		 149 return 0;
150}
151
cd17a9b5
AH		 152static int perf_event__repipe_synth(struct perf_tool *tool,
153 union perf_event *event)
154{
155 struct perf_inject *inject = container_of(tool, struct perf_inject,
156 tool);
157
158 return output_bytes(inject, event, event->header.size);
159}
160
d704ebda
ACM		 161static int perf_event__repipe_oe_synth(struct perf_tool *tool,
162 union perf_event *event,
163 struct ordered_events *oe __maybe_unused)
164{
165 return perf_event__repipe_synth(tool, event);
166}
167
e12b202f 168#ifdef HAVE_JITDUMP
9b07e27f
SE		 169static int perf_event__drop_oe(struct perf_tool *tool __maybe_unused,
170 union perf_event *event __maybe_unused,
171 struct ordered_events *oe __maybe_unused)
172{
173 return 0;
174}
175#endif
176
89f1688a
JO		 177static int perf_event__repipe_op2_synth(struct perf_session *session,
178 union perf_event *event)
743eb868 179{
89f1688a 180 return perf_event__repipe_synth(session->tool, event);
743eb868
ACM		 181}
182
2946eced
NK		 183static int perf_event__repipe_op4_synth(struct perf_session *session,
184 union perf_event *event,
2292083f
AB		 185 u64 data __maybe_unused,
186 const char *str __maybe_unused)
2946eced
NK		 187{
188 return perf_event__repipe_synth(session->tool, event);
189}
190
47c3d109
AH		 191static int perf_event__repipe_attr(struct perf_tool *tool,
192 union perf_event *event,
63503dba 193 struct evlist **pevlist)
10d0f086 194{
89c97d93
AH		 195 struct perf_inject *inject = container_of(tool, struct perf_inject,
196 tool);
1a1ed1ba 197 int ret;
47c3d109
AH		 198
199 ret = perf_event__process_attr(tool, event, pevlist);
1a1ed1ba
SE		 200 if (ret)
201 return ret;
202
fea20d66 203 if (!inject->is_pipe)
89c97d93
AH		 204 return 0;
205
47c3d109 206 return perf_event__repipe_synth(tool, event);
10d0f086
ACM		 207}
208
2946eced
NK		 209static int perf_event__repipe_event_update(struct perf_tool *tool,
210 union perf_event *event,
211 struct evlist **pevlist __maybe_unused)
212{
213 return perf_event__repipe_synth(tool, event);
214}
215
e31f0d01
AH		 216#ifdef HAVE_AUXTRACE_SUPPORT
217
1746212d 218static int copy_bytes(struct perf_inject *inject, struct perf_data *data, off_t size)
e31f0d01
AH		 219{
220 char buf[4096];
221 ssize_t ssz;
222 int ret;
223
224 while (size > 0) {
1746212d 225 ssz = perf_data__read(data, buf, min(size, (off_t)sizeof(buf)));
e31f0d01
AH		 226 if (ssz < 0)
227 return -errno;
228 ret = output_bytes(inject, buf, ssz);
229 if (ret)
230 return ret;
231 size -= ssz;
232 }
233
234 return 0;
235}
236
7336555a
JO		 237static s64 perf_event__repipe_auxtrace(struct perf_session *session,
238 union perf_event *event)
cd17a9b5 239{
7336555a 240 struct perf_tool *tool = session->tool;
cd17a9b5
AH		 241 struct perf_inject *inject = container_of(tool, struct perf_inject,
242 tool);
243 int ret;
244
cd10b289
AH		 245 inject->have_auxtrace = true;
246
99fa2984
AH		 247 if (!inject->output.is_pipe) {
248 off_t offset;
249
eae8ad80 250 offset = lseek(inject->output.file.fd, 0, SEEK_CUR);
99fa2984
AH		 251 if (offset == -1)
252 return -errno;
253 ret = auxtrace_index__auxtrace_event(&session->auxtrace_index,
254 event, offset);
255 if (ret < 0)
256 return ret;
257 }
258
8ceb41d7 259 if (perf_data__is_pipe(session->data) || !session->one_mmap) {
cd17a9b5
AH		 260 ret = output_bytes(inject, event, event->header.size);
261 if (ret < 0)
262 return ret;
1746212d 263 ret = copy_bytes(inject, session->data,
cd17a9b5
AH		 264 event->auxtrace.size);
265 } else {
266 ret = output_bytes(inject, event,
267 event->header.size + event->auxtrace.size);
268 }
269 if (ret < 0)
270 return ret;
271
272 return event->auxtrace.size;
273}
274
e31f0d01
AH		 275#else
276
277static s64
7336555a
JO		 278perf_event__repipe_auxtrace(struct perf_session *session __maybe_unused,
279 union perf_event *event __maybe_unused)
e31f0d01
AH		 280{
281 pr_err("AUX area tracing not supported\n");
282 return -EINVAL;
283}
284
285#endif
286
45694aa7 287static int perf_event__repipe(struct perf_tool *tool,
d20deb64 288 union perf_event *event,
1d037ca1 289 struct perf_sample *sample __maybe_unused,
63c2c9f8 290 struct machine *machine __maybe_unused)
640c03ce 291{
63c2c9f8 292 return perf_event__repipe_synth(tool, event);
640c03ce
ACM		 293}
294
f56fb986
AH		 295static int perf_event__drop(struct perf_tool *tool __maybe_unused,
296 union perf_event *event __maybe_unused,
297 struct perf_sample *sample __maybe_unused,
298 struct machine *machine __maybe_unused)
299{
300 return 0;
301}
302
73117308
AH		 303static int perf_event__drop_aux(struct perf_tool *tool,
304 union perf_event *event __maybe_unused,
305 struct perf_sample *sample,
306 struct machine *machine __maybe_unused)
307{
308 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
309
310 if (!inject->aux_id)
311 inject->aux_id = sample->id;
312
313 return 0;
314}
315
ba2675bf
AH		 316static union perf_event *
317perf_inject__cut_auxtrace_sample(struct perf_inject *inject,
318 union perf_event *event,
319 struct perf_sample *sample)
320{
321 size_t sz1 = sample->aux_sample.data - (void *)event;
322 size_t sz2 = event->header.size - sample->aux_sample.size - sz1;
d3944f0e 323 union perf_event *ev;
ba2675bf 324
d3944f0e
IR		 325 if (inject->event_copy == NULL) {
326 inject->event_copy = malloc(PERF_SAMPLE_MAX_SIZE);
327 if (!inject->event_copy)
328 return ERR_PTR(-ENOMEM);
329 }
330 ev = (union perf_event *)inject->event_copy;
ba2675bf
AH		 331 if (sz1 > event->header.size || sz2 > event->header.size ||
332 sz1 + sz2 > event->header.size ||
333 sz1 < sizeof(struct perf_event_header) + sizeof(u64))
334 return event;
335
336 memcpy(ev, event, sz1);
337 memcpy((void *)ev + sz1, (void *)event + event->header.size - sz2, sz2);
338 ev->header.size = sz1 + sz2;
339 ((u64 *)((void *)ev + sz1))[-1] = 0;
340
341 return ev;
342}
343
26a031e1
AV		 344typedef int (*inject_handler)(struct perf_tool *tool,
345 union perf_event *event,
346 struct perf_sample *sample,
32dcd021 347 struct evsel *evsel,
26a031e1
AV		 348 struct machine *machine);
349
45694aa7 350static int perf_event__repipe_sample(struct perf_tool *tool,
d20deb64 351 union perf_event *event,
26a031e1 352 struct perf_sample *sample,
32dcd021 353 struct evsel *evsel,
26a031e1 354 struct machine *machine)
9e69c210 355{
ba2675bf
AH		 356 struct perf_inject *inject = container_of(tool, struct perf_inject,
357 tool);
358
40978e9b 359 if (evsel && evsel->handler) {
744a9719 360 inject_handler f = evsel->handler;
26a031e1
AV		 361 return f(tool, event, sample, evsel, machine);
362 }
363
54a3cf59
AV		 364 build_id__mark_dso_hit(tool, event, sample, evsel, machine);
365
d3944f0e 366 if (inject->itrace_synth_opts.set && sample->aux_sample.size) {
ba2675bf 367 event = perf_inject__cut_auxtrace_sample(inject, event, sample);
d3944f0e
IR		 368 if (IS_ERR(event))
369 return PTR_ERR(event);
370 }
ba2675bf 371
63c2c9f8 372 return perf_event__repipe_synth(tool, event);
9e69c210
ACM		 373}
374
45694aa7 375static int perf_event__repipe_mmap(struct perf_tool *tool,
d20deb64 376 union perf_event *event,
8115d60c 377 struct perf_sample *sample,
743eb868 378 struct machine *machine)
454c407e
TZ		 379{
380 int err;
381
45694aa7
ACM		 382 err = perf_event__process_mmap(tool, event, sample, machine);
383 perf_event__repipe(tool, event, sample, machine);
454c407e
TZ		 384
385 return err;
386}
387
e12b202f 388#ifdef HAVE_JITDUMP
9b07e27f
SE		 389static int perf_event__jit_repipe_mmap(struct perf_tool *tool,
390 union perf_event *event,
391 struct perf_sample *sample,
392 struct machine *machine)
393{
394 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
395 u64 n = 0;
570735b3 396 int ret;
9b07e27f
SE		 397
398 /*
399 * if jit marker, then inject jit mmaps and generate ELF images
400 */
570735b3 401 ret = jit_process(inject->session, &inject->output, machine,
67dec926 402 event->mmap.filename, event->mmap.pid, event->mmap.tid, &n);
570735b3
AH		 403 if (ret < 0)
404 return ret;
405 if (ret) {
9b07e27f
SE		 406 inject->bytes_written += n;
407 return 0;
408 }
409 return perf_event__repipe_mmap(tool, event, sample, machine);
410}
411#endif
412
27c9c342
NK		 413static struct dso *findnew_dso(int pid, int tid, const char *filename,
414 struct dso_id *id, struct machine *machine)
415{
416 struct thread *thread;
417 struct nsinfo *nsi = NULL;
418 struct nsinfo *nnsi;
419 struct dso *dso;
420 bool vdso;
421
422 thread = machine__findnew_thread(machine, pid, tid);
423 if (thread == NULL) {
424 pr_err("cannot find or create a task %d/%d.\n", tid, pid);
425 return NULL;
426 }
427
428 vdso = is_vdso_map(filename);
ee84a303 429 nsi = nsinfo__get(thread__nsinfo(thread));
27c9c342
NK		 430
431 if (vdso) {
432 /* The vdso maps are always on the host and not the
433 * container. Ensure that we don't use setns to look
434 * them up.
435 */
436 nnsi = nsinfo__copy(nsi);
437 if (nnsi) {
438 nsinfo__put(nsi);
bcaf0a97 439 nsinfo__clear_need_setns(nnsi);
27c9c342
NK		 440 nsi = nnsi;
441 }
442 dso = machine__findnew_vdso(machine, thread);
443 } else {
444 dso = machine__findnew_dso_id(machine, filename, id);
445 }
446
0967ebff 447 if (dso) {
e54dea69 448 mutex_lock(&dso->lock);
0967ebff 449 nsinfo__put(dso->nsinfo);
27c9c342 450 dso->nsinfo = nsi;
e54dea69 451 mutex_unlock(&dso->lock);
0967ebff 452 } else
27c9c342
NK		 453 nsinfo__put(nsi);
454
455 thread__put(thread);
456 return dso;
457}
458
459static int perf_event__repipe_buildid_mmap(struct perf_tool *tool,
460 union perf_event *event,
461 struct perf_sample *sample,
462 struct machine *machine)
463{
464 struct dso *dso;
465
466 dso = findnew_dso(event->mmap.pid, event->mmap.tid,
467 event->mmap.filename, NULL, machine);
468
469 if (dso && !dso->hit) {
470 dso->hit = 1;
471 dso__inject_build_id(dso, tool, machine, sample->cpumode, 0);
27c9c342 472 }
0c3f7b38 473 dso__put(dso);
27c9c342
NK		 474
475 return perf_event__repipe(tool, event, sample, machine);
476}
477
5c5e854b
SE		 478static int perf_event__repipe_mmap2(struct perf_tool *tool,
479 union perf_event *event,
480 struct perf_sample *sample,
481 struct machine *machine)
482{
483 int err;
484
485 err = perf_event__process_mmap2(tool, event, sample, machine);
486 perf_event__repipe(tool, event, sample, machine);
487
f7fc0d1c
NK		 488 if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
489 struct dso *dso;
490
491 dso = findnew_dso(event->mmap2.pid, event->mmap2.tid,
492 event->mmap2.filename, NULL, machine);
493 if (dso) {
494 /* mark it not to inject build-id */
495 dso->hit = 1;
496 }
497 dso__put(dso);
498 }
499
5c5e854b
SE		 500 return err;
501}
502
e12b202f 503#ifdef HAVE_JITDUMP
9b07e27f
SE		 504static int perf_event__jit_repipe_mmap2(struct perf_tool *tool,
505 union perf_event *event,
506 struct perf_sample *sample,
507 struct machine *machine)
508{
509 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
510 u64 n = 0;
570735b3 511 int ret;
9b07e27f
SE		 512
513 /*
514 * if jit marker, then inject jit mmaps and generate ELF images
515 */
570735b3 516 ret = jit_process(inject->session, &inject->output, machine,
67dec926 517 event->mmap2.filename, event->mmap2.pid, event->mmap2.tid, &n);
570735b3
AH		 518 if (ret < 0)
519 return ret;
520 if (ret) {
9b07e27f
SE		 521 inject->bytes_written += n;
522 return 0;
523 }
524 return perf_event__repipe_mmap2(tool, event, sample, machine);
525}
526#endif
527
27c9c342
NK		 528static int perf_event__repipe_buildid_mmap2(struct perf_tool *tool,
529 union perf_event *event,
530 struct perf_sample *sample,
531 struct machine *machine)
532{
533 struct dso_id dso_id = {
534 .maj = event->mmap2.maj,
535 .min = event->mmap2.min,
536 .ino = event->mmap2.ino,
537 .ino_generation = event->mmap2.ino_generation,
538 };
539 struct dso *dso;
540
f7fc0d1c
NK		 541 if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) {
542 /* cannot use dso_id since it'd have invalid info */
543 dso = findnew_dso(event->mmap2.pid, event->mmap2.tid,
544 event->mmap2.filename, NULL, machine);
545 if (dso) {
546 /* mark it not to inject build-id */
547 dso->hit = 1;
548 }
549 dso__put(dso);
ce9f1c05 550 perf_event__repipe(tool, event, sample, machine);
f7fc0d1c
NK		 551 return 0;
552 }
553
27c9c342
NK		 554 dso = findnew_dso(event->mmap2.pid, event->mmap2.tid,
555 event->mmap2.filename, &dso_id, machine);
556
557 if (dso && !dso->hit) {
558 dso->hit = 1;
559 dso__inject_build_id(dso, tool, machine, sample->cpumode,
560 event->mmap2.flags);
27c9c342 561 }
0c3f7b38 562 dso__put(dso);
27c9c342
NK		 563
564 perf_event__repipe(tool, event, sample, machine);
565
566 return 0;
567}
568
f62d3f0f 569static int perf_event__repipe_fork(struct perf_tool *tool,
d20deb64 570 union perf_event *event,
8115d60c 571 struct perf_sample *sample,
743eb868 572 struct machine *machine)
454c407e
TZ		 573{
574 int err;
575
f62d3f0f 576 err = perf_event__process_fork(tool, event, sample, machine);
45694aa7 577 perf_event__repipe(tool, event, sample, machine);
454c407e
TZ		 578
579 return err;
580}
581
0f0aa5e0
AH		 582static int perf_event__repipe_comm(struct perf_tool *tool,
583 union perf_event *event,
584 struct perf_sample *sample,
585 struct machine *machine)
586{
587 int err;
588
589 err = perf_event__process_comm(tool, event, sample, machine);
590 perf_event__repipe(tool, event, sample, machine);
591
592 return err;
593}
594
f3b3614a
HB		 595static int perf_event__repipe_namespaces(struct perf_tool *tool,
596 union perf_event *event,
597 struct perf_sample *sample,
598 struct machine *machine)
599{
600 int err = perf_event__process_namespaces(tool, event, sample, machine);
601
602 perf_event__repipe(tool, event, sample, machine);
603
604 return err;
605}
606
0f0aa5e0
AH		 607static int perf_event__repipe_exit(struct perf_tool *tool,
608 union perf_event *event,
609 struct perf_sample *sample,
610 struct machine *machine)
611{
612 int err;
613
614 err = perf_event__process_exit(tool, event, sample, machine);
615 perf_event__repipe(tool, event, sample, machine);
616
617 return err;
618}
619
378ef0f5 620#ifdef HAVE_LIBTRACEEVENT
89f1688a
JO		 621static int perf_event__repipe_tracing_data(struct perf_session *session,
622 union perf_event *event)
454c407e 623{
89f1688a 624 perf_event__repipe_synth(session->tool, event);
454c407e 625
f0ac5b85 626 return perf_event__process_tracing_data(session, event);
454c407e 627}
378ef0f5 628#endif
454c407e 629
c824c433 630static int dso__read_build_id(struct dso *dso)
454c407e 631{
336c95b2
NK		 632 struct nscookie nsc;
633
c824c433 634 if (dso->has_build_id)
090f7204 635 return 0;
454c407e 636
e54dea69 637 mutex_lock(&dso->lock);
336c95b2 638 nsinfo__mountns_enter(dso->nsinfo, &nsc);
f766819c 639 if (filename__read_build_id(dso->long_name, &dso->bid) > 0)
c824c433 640 dso->has_build_id = true;
75d48c56 641 else if (dso->nsinfo) {
7031edac 642 char *new_name = dso__filename_with_chroot(dso, dso->long_name);
75d48c56 643
75d48c56
NK		 644 if (new_name && filename__read_build_id(new_name, &dso->bid) > 0)
645 dso->has_build_id = true;
646 free(new_name);
647 }
336c95b2 648 nsinfo__mountns_exit(&nsc);
e54dea69 649 mutex_unlock(&dso->lock);
454c407e 650
336c95b2 651 return dso->has_build_id ? 0 : -1;
090f7204 652}
454c407e 653
8012243e
RS		 654static struct strlist *perf_inject__parse_known_build_ids(
655 const char *known_build_ids_string)
656{
657 struct str_node *pos, *tmp;
658 struct strlist *known_build_ids;
659 int bid_len;
660
661 known_build_ids = strlist__new(known_build_ids_string, NULL);
662 if (known_build_ids == NULL)
663 return NULL;
664 strlist__for_each_entry_safe(pos, tmp, known_build_ids) {
665 const char *build_id, *dso_name;
666
667 build_id = skip_spaces(pos->s);
668 dso_name = strchr(build_id, ' ');
669 if (dso_name == NULL) {
670 strlist__remove(known_build_ids, pos);
671 continue;
672 }
673 bid_len = dso_name - pos->s;
674 dso_name = skip_spaces(dso_name);
675 if (bid_len % 2 != 0 || bid_len >= SBUILD_ID_SIZE) {
676 strlist__remove(known_build_ids, pos);
677 continue;
678 }
679 for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) {
680 if (!isxdigit(build_id[2 * ix]) ||
681 !isxdigit(build_id[2 * ix + 1])) {
682 strlist__remove(known_build_ids, pos);
683 break;
684 }
685 }
686 }
687 return known_build_ids;
688}
689
690static bool perf_inject__lookup_known_build_id(struct perf_inject *inject,
691 struct dso *dso)
692{
693 struct str_node *pos;
694 int bid_len;
695
696 strlist__for_each_entry(pos, inject->known_build_ids) {
697 const char *build_id, *dso_name;
698
699 build_id = skip_spaces(pos->s);
700 dso_name = strchr(build_id, ' ');
701 bid_len = dso_name - pos->s;
702 dso_name = skip_spaces(dso_name);
703 if (strcmp(dso->long_name, dso_name))
704 continue;
705 for (int ix = 0; 2 * ix + 1 < bid_len; ++ix) {
706 dso->bid.data[ix] = (hex(build_id[2 * ix]) << 4 |
707 hex(build_id[2 * ix + 1]));
708 }
709 dso->bid.size = bid_len / 2;
710 dso->has_build_id = 1;
711 return true;
712 }
713 return false;
714}
715
c824c433 716static int dso__inject_build_id(struct dso *dso, struct perf_tool *tool,
e7b60c5a 717 struct machine *machine, u8 cpumode, u32 flags)
090f7204 718{
8012243e
RS		 719 struct perf_inject *inject = container_of(tool, struct perf_inject,
720 tool);
090f7204 721 int err;
454c407e 722
e7b60c5a
NK		 723 if (is_anon_memory(dso->long_name) || flags & MAP_HUGETLB)
724 return 0;
725 if (is_no_dso_memory(dso->long_name))
726 return 0;
727
8012243e
RS		 728 if (inject->known_build_ids != NULL &&
729 perf_inject__lookup_known_build_id(inject, dso))
730 return 1;
731
c824c433
ACM		 732 if (dso__read_build_id(dso) < 0) {
733 pr_debug("no build_id found for %s\n", dso->long_name);
090f7204
ACM		 734 return -1;
735 }
454c407e 736
e7b60c5a
NK		 737 err = perf_event__synthesize_build_id(tool, dso, cpumode,
738 perf_event__repipe, machine);
090f7204 739 if (err) {
c824c433 740 pr_err("Can't synthesize build_id event for %s\n", dso->long_name);
454c407e
TZ		 741 return -1;
742 }
743
744 return 0;
745}
746
0bf02a0d
NK		 747int perf_event__inject_buildid(struct perf_tool *tool, union perf_event *event,
748 struct perf_sample *sample,
749 struct evsel *evsel __maybe_unused,
750 struct machine *machine)
454c407e
TZ		 751{
752 struct addr_location al;
753 struct thread *thread;
454c407e 754
0dd5041c 755 addr_location__init(&al);
13ce34df 756 thread = machine__findnew_thread(machine, sample->pid, sample->tid);
454c407e
TZ		 757 if (thread == NULL) {
758 pr_err("problem processing %d event, skipping it.\n",
759 event->header.type);
454c407e
TZ		 760 goto repipe;
761 }
762
71a84b5a 763 if (thread__find_map(thread, sample->cpumode, sample->ip, &al)) {
63df0e4b
IR		 764 struct dso *dso = map__dso(al.map);
765
766 if (!dso->hit) {
767 dso->hit = 1;
768 dso__inject_build_id(dso, tool, machine,
ddee3f2b 769 sample->cpumode, map__flags(al.map));
454c407e
TZ		 770 }
771 }
772
b91fc39f 773 thread__put(thread);
454c407e 774repipe:
45694aa7 775 perf_event__repipe(tool, event, sample, machine);
0dd5041c 776 addr_location__exit(&al);
090f7204 777 return 0;
454c407e
TZ		 778}
779
26a031e1
AV		 780static int perf_inject__sched_process_exit(struct perf_tool *tool,
781 union perf_event *event __maybe_unused,
782 struct perf_sample *sample,
32dcd021 783 struct evsel *evsel __maybe_unused,
26a031e1
AV		 784 struct machine *machine __maybe_unused)
785{
786 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
787 struct event_entry *ent;
788
789 list_for_each_entry(ent, &inject->samples, node) {
790 if (sample->tid == ent->tid) {
791 list_del_init(&ent->node);
792 free(ent);
793 break;
794 }
795 }
796
797 return 0;
798}
799
800static int perf_inject__sched_switch(struct perf_tool *tool,
801 union perf_event *event,
802 struct perf_sample *sample,
32dcd021 803 struct evsel *evsel,
26a031e1
AV		 804 struct machine *machine)
805{
806 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
807 struct event_entry *ent;
808
809 perf_inject__sched_process_exit(tool, event, sample, evsel, machine);
810
811 ent = malloc(event->header.size + sizeof(struct event_entry));
812 if (ent == NULL) {
813 color_fprintf(stderr, PERF_COLOR_RED,
814 "Not enough memory to process sched switch event!");
815 return -1;
816 }
817
818 ent->tid = sample->tid;
819 memcpy(&ent->event, event, event->header.size);
820 list_add(&ent->node, &inject->samples);
821 return 0;
822}
823
378ef0f5 824#ifdef HAVE_LIBTRACEEVENT
26a031e1
AV		 825static int perf_inject__sched_stat(struct perf_tool *tool,
826 union perf_event *event __maybe_unused,
827 struct perf_sample *sample,
32dcd021 828 struct evsel *evsel,
26a031e1
AV		 829 struct machine *machine)
830{
831 struct event_entry *ent;
832 union perf_event *event_sw;
833 struct perf_sample sample_sw;
834 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
efc0cdc9 835 u32 pid = evsel__intval(evsel, sample, "pid");
26a031e1
AV		 836
837 list_for_each_entry(ent, &inject->samples, node) {
838 if (pid == ent->tid)
839 goto found;
840 }
841
842 return 0;
843found:
844 event_sw = &ent->event[0];
6b6017a2 845 evsel__parse_sample(evsel, event_sw, &sample_sw);
26a031e1
AV		 846
847 sample_sw.period = sample->period;
848 sample_sw.time = sample->time;
1fc632ce
JO		 849 perf_event__synthesize_sample(event_sw, evsel->core.attr.sample_type,
850 evsel->core.attr.read_format, &sample_sw);
54a3cf59 851 build_id__mark_dso_hit(tool, event_sw, &sample_sw, evsel, machine);
26a031e1
AV		 852 return perf_event__repipe(tool, event_sw, &sample_sw, machine);
853}
378ef0f5 854#endif
26a031e1 855
97406a7e
AH		 856static struct guest_vcpu *guest_session__vcpu(struct guest_session *gs, u32 vcpu)
857{
858 if (realloc_array_as_needed(gs->vcpu, gs->vcpu_cnt, vcpu, NULL))
859 return NULL;
860 return &gs->vcpu[vcpu];
861}
862
863static int guest_session__output_bytes(struct guest_session *gs, void *buf, size_t sz)
864{
865 ssize_t ret = writen(gs->tmp_fd, buf, sz);
866
867 return ret < 0 ? ret : 0;
868}
869
870static int guest_session__repipe(struct perf_tool *tool,
871 union perf_event *event,
872 struct perf_sample *sample __maybe_unused,
873 struct machine *machine __maybe_unused)
874{
875 struct guest_session *gs = container_of(tool, struct guest_session, tool);
876
877 return guest_session__output_bytes(gs, event, event->header.size);
878}
879
880static int guest_session__map_tid(struct guest_session *gs, u32 tid, u32 vcpu)
881{
882 struct guest_tid *guest_tid = zalloc(sizeof(*guest_tid));
883 int hash;
884
885 if (!guest_tid)
886 return -ENOMEM;
887
888 guest_tid->tid = tid;
889 guest_tid->vcpu = vcpu;
890 hash = hash_32(guest_tid->tid, PERF_EVLIST__HLIST_BITS);
891 hlist_add_head(&guest_tid->node, &gs->tids[hash]);
892
893 return 0;
894}
895
896static int host_peek_vm_comms_cb(struct perf_session *session __maybe_unused,
897 union perf_event *event,
898 u64 offset __maybe_unused, void *data)
899{
900 struct guest_session *gs = data;
901 unsigned int vcpu;
902 struct guest_vcpu *guest_vcpu;
903 int ret;
904
905 if (event->header.type != PERF_RECORD_COMM ||
906 event->comm.pid != gs->machine_pid)
907 return 0;
908
909 /*
910 * QEMU option -name debug-threads=on, causes thread names formatted as
911 * below, although it is not an ABI. Also libvirt seems to use this by
912 * default. Here we rely on it to tell us which thread is which VCPU.
913 */
914 ret = sscanf(event->comm.comm, "CPU %u/KVM", &vcpu);
915 if (ret <= 0)
916 return ret;
917 pr_debug("Found VCPU: tid %u comm %s vcpu %u\n",
918 event->comm.tid, event->comm.comm, vcpu);
919 if (vcpu > INT_MAX) {
920 pr_err("Invalid VCPU %u\n", vcpu);
921 return -EINVAL;
922 }
923 guest_vcpu = guest_session__vcpu(gs, vcpu);
924 if (!guest_vcpu)
925 return -ENOMEM;
926 if (guest_vcpu->tid && guest_vcpu->tid != event->comm.tid) {
927 pr_err("Fatal error: Two threads found with the same VCPU\n");
928 return -EINVAL;
929 }
930 guest_vcpu->tid = event->comm.tid;
931
932 return guest_session__map_tid(gs, event->comm.tid, vcpu);
933}
934
935static int host_peek_vm_comms(struct perf_session *session, struct guest_session *gs)
936{
937 return perf_session__peek_events(session, session->header.data_offset,
938 session->header.data_size,
939 host_peek_vm_comms_cb, gs);
940}
941
942static bool evlist__is_id_used(struct evlist *evlist, u64 id)
943{
944 return evlist__id2sid(evlist, id);
945}
946
947static u64 guest_session__allocate_new_id(struct guest_session *gs, struct evlist *host_evlist)
948{
949 do {
950 gs->highest_id += 1;
951 } while (!gs->highest_id || evlist__is_id_used(host_evlist, gs->highest_id));
952
953 return gs->highest_id;
954}
955
956static int guest_session__map_id(struct guest_session *gs, u64 id, u64 host_id, u32 vcpu)
957{
958 struct guest_id *guest_id = zalloc(sizeof(*guest_id));
959 int hash;
960
961 if (!guest_id)
962 return -ENOMEM;
963
964 guest_id->id = id;
965 guest_id->host_id = host_id;
966 guest_id->vcpu = vcpu;
967 hash = hash_64(guest_id->id, PERF_EVLIST__HLIST_BITS);
968 hlist_add_head(&guest_id->node, &gs->heads[hash]);
969
970 return 0;
971}
972
973static u64 evlist__find_highest_id(struct evlist *evlist)
974{
975 struct evsel *evsel;
976 u64 highest_id = 1;
977
978 evlist__for_each_entry(evlist, evsel) {
979 u32 j;
980
981 for (j = 0; j < evsel->core.ids; j++) {
982 u64 id = evsel->core.id[j];
983
984 if (id > highest_id)
985 highest_id = id;
986 }
987 }
988
989 return highest_id;
990}
991
992static int guest_session__map_ids(struct guest_session *gs, struct evlist *host_evlist)
993{
994 struct evlist *evlist = gs->session->evlist;
995 struct evsel *evsel;
996 int ret;
997
998 evlist__for_each_entry(evlist, evsel) {
999 u32 j;
1000
1001 for (j = 0; j < evsel->core.ids; j++) {
1002 struct perf_sample_id *sid;
1003 u64 host_id;
1004 u64 id;
1005
1006 id = evsel->core.id[j];
1007 sid = evlist__id2sid(evlist, id);
1008 if (!sid || sid->cpu.cpu == -1)
1009 continue;
1010 host_id = guest_session__allocate_new_id(gs, host_evlist);
1011 ret = guest_session__map_id(gs, id, host_id, sid->cpu.cpu);
1012 if (ret)
1013 return ret;
1014 }
1015 }
1016
1017 return 0;
1018}
1019
1020static struct guest_id *guest_session__lookup_id(struct guest_session *gs, u64 id)
1021{
1022 struct hlist_head *head;
1023 struct guest_id *guest_id;
1024 int hash;
1025
1026 hash = hash_64(id, PERF_EVLIST__HLIST_BITS);
1027 head = &gs->heads[hash];
1028
1029 hlist_for_each_entry(guest_id, head, node)
1030 if (guest_id->id == id)
1031 return guest_id;
1032
1033 return NULL;
1034}
1035
1036static int process_attr(struct perf_tool *tool, union perf_event *event,
1037 struct perf_sample *sample __maybe_unused,
1038 struct machine *machine __maybe_unused)
1039{
1040 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1041
1042 return perf_event__process_attr(tool, event, &inject->session->evlist);
1043}
1044
1045static int guest_session__add_attr(struct guest_session *gs, struct evsel *evsel)
1046{
1047 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1048 struct perf_event_attr attr = evsel->core.attr;
1049 u64 *id_array;
1050 u32 *vcpu_array;
1051 int ret = -ENOMEM;
1052 u32 i;
1053
1054 id_array = calloc(evsel->core.ids, sizeof(*id_array));
1055 if (!id_array)
1056 return -ENOMEM;
1057
1058 vcpu_array = calloc(evsel->core.ids, sizeof(*vcpu_array));
1059 if (!vcpu_array)
1060 goto out;
1061
1062 for (i = 0; i < evsel->core.ids; i++) {
1063 u64 id = evsel->core.id[i];
1064 struct guest_id *guest_id = guest_session__lookup_id(gs, id);
1065
1066 if (!guest_id) {
1067 pr_err("Failed to find guest id %"PRIu64"\n", id);
1068 ret = -EINVAL;
1069 goto out;
1070 }
1071 id_array[i] = guest_id->host_id;
1072 vcpu_array[i] = guest_id->vcpu;
1073 }
1074
1075 attr.sample_type |= PERF_SAMPLE_IDENTIFIER;
1076 attr.exclude_host = 1;
1077 attr.exclude_guest = 0;
1078
1079 ret = perf_event__synthesize_attr(&inject->tool, &attr, evsel->core.ids,
1080 id_array, process_attr);
1081 if (ret)
1082 pr_err("Failed to add guest attr.\n");
1083
1084 for (i = 0; i < evsel->core.ids; i++) {
1085 struct perf_sample_id *sid;
1086 u32 vcpu = vcpu_array[i];
1087
1088 sid = evlist__id2sid(inject->session->evlist, id_array[i]);
1089 /* Guest event is per-thread from the host point of view */
1090 sid->cpu.cpu = -1;
1091 sid->tid = gs->vcpu[vcpu].tid;
1092 sid->machine_pid = gs->machine_pid;
1093 sid->vcpu.cpu = vcpu;
1094 }
1095out:
1096 free(vcpu_array);
1097 free(id_array);
1098 return ret;
1099}
1100
1101static int guest_session__add_attrs(struct guest_session *gs)
1102{
1103 struct evlist *evlist = gs->session->evlist;
1104 struct evsel *evsel;
1105 int ret;
1106
1107 evlist__for_each_entry(evlist, evsel) {
1108 ret = guest_session__add_attr(gs, evsel);
1109 if (ret)
1110 return ret;
1111 }
1112
1113 return 0;
1114}
1115
1116static int synthesize_id_index(struct perf_inject *inject, size_t new_cnt)
1117{
1118 struct perf_session *session = inject->session;
1119 struct evlist *evlist = session->evlist;
1120 struct machine *machine = &session->machines.host;
1121 size_t from = evlist->core.nr_entries - new_cnt;
1122
1123 return __perf_event__synthesize_id_index(&inject->tool, perf_event__repipe,
1124 evlist, machine, from);
1125}
1126
1127static struct guest_tid *guest_session__lookup_tid(struct guest_session *gs, u32 tid)
1128{
1129 struct hlist_head *head;
1130 struct guest_tid *guest_tid;
1131 int hash;
1132
1133 hash = hash_32(tid, PERF_EVLIST__HLIST_BITS);
1134 head = &gs->tids[hash];
1135
1136 hlist_for_each_entry(guest_tid, head, node)
1137 if (guest_tid->tid == tid)
1138 return guest_tid;
1139
1140 return NULL;
1141}
1142
1143static bool dso__is_in_kernel_space(struct dso *dso)
1144{
1145 if (dso__is_vdso(dso))
1146 return false;
1147
1148 return dso__is_kcore(dso) ||
1149 dso->kernel ||
1150 is_kernel_module(dso->long_name, PERF_RECORD_MISC_CPUMODE_UNKNOWN);
1151}
1152
1153static u64 evlist__first_id(struct evlist *evlist)
1154{
1155 struct evsel *evsel;
1156
1157 evlist__for_each_entry(evlist, evsel) {
1158 if (evsel->core.ids)
1159 return evsel->core.id[0];
1160 }
1161 return 0;
1162}
1163
1164static int process_build_id(struct perf_tool *tool,
1165 union perf_event *event,
1166 struct perf_sample *sample __maybe_unused,
1167 struct machine *machine __maybe_unused)
1168{
1169 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1170
1171 return perf_event__process_build_id(inject->session, event);
1172}
1173
1174static int synthesize_build_id(struct perf_inject *inject, struct dso *dso, pid_t machine_pid)
1175{
1176 struct machine *machine = perf_session__findnew_machine(inject->session, machine_pid);
1177 u8 cpumode = dso__is_in_kernel_space(dso) ?
1178 PERF_RECORD_MISC_GUEST_KERNEL :
1179 PERF_RECORD_MISC_GUEST_USER;
1180
1181 if (!machine)
1182 return -ENOMEM;
1183
1184 dso->hit = 1;
1185
1186 return perf_event__synthesize_build_id(&inject->tool, dso, cpumode,
1187 process_build_id, machine);
1188}
1189
1190static int guest_session__add_build_ids(struct guest_session *gs)
1191{
1192 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1193 struct machine *machine = &gs->session->machines.host;
1194 struct dso *dso;
1195 int ret;
1196
1197 /* Build IDs will be put in the Build ID feature section */
1198 perf_header__set_feat(&inject->session->header, HEADER_BUILD_ID);
1199
1200 dsos__for_each_with_build_id(dso, &machine->dsos.head) {
1201 ret = synthesize_build_id(inject, dso, gs->machine_pid);
1202 if (ret)
1203 return ret;
1204 }
1205
1206 return 0;
1207}
1208
1209static int guest_session__ksymbol_event(struct perf_tool *tool,
1210 union perf_event *event,
1211 struct perf_sample *sample __maybe_unused,
1212 struct machine *machine __maybe_unused)
1213{
1214 struct guest_session *gs = container_of(tool, struct guest_session, tool);
1215
1216 /* Only support out-of-line i.e. no BPF support */
1217 if (event->ksymbol.ksym_type != PERF_RECORD_KSYMBOL_TYPE_OOL)
1218 return 0;
1219
1220 return guest_session__output_bytes(gs, event, event->header.size);
1221}
1222
1223static int guest_session__start(struct guest_session *gs, const char *name, bool force)
1224{
1225 char tmp_file_name[] = "/tmp/perf-inject-guest_session-XXXXXX";
1226 struct perf_session *session;
1227 int ret;
1228
1229 /* Only these events will be injected */
1230 gs->tool.mmap = guest_session__repipe;
1231 gs->tool.mmap2 = guest_session__repipe;
1232 gs->tool.comm = guest_session__repipe;
1233 gs->tool.fork = guest_session__repipe;
1234 gs->tool.exit = guest_session__repipe;
1235 gs->tool.lost = guest_session__repipe;
1236 gs->tool.context_switch = guest_session__repipe;
1237 gs->tool.ksymbol = guest_session__ksymbol_event;
1238 gs->tool.text_poke = guest_session__repipe;
1239 /*
1240 * Processing a build ID creates a struct dso with that build ID. Later,
1241 * all guest dsos are iterated and the build IDs processed into the host
1242 * session where they will be output to the Build ID feature section
1243 * when the perf.data file header is written.
1244 */
1245 gs->tool.build_id = perf_event__process_build_id;
1246 /* Process the id index to know what VCPU an ID belongs to */
1247 gs->tool.id_index = perf_event__process_id_index;
1248
1249 gs->tool.ordered_events = true;
1250 gs->tool.ordering_requires_timestamps = true;
1251
1252 gs->data.path = name;
1253 gs->data.force = force;
1254 gs->data.mode = PERF_DATA_MODE_READ;
1255
1256 session = perf_session__new(&gs->data, &gs->tool);
1257 if (IS_ERR(session))
1258 return PTR_ERR(session);
1259 gs->session = session;
1260
1261 /*
1262 * Initial events have zero'd ID samples. Get default ID sample size
1263 * used for removing them.
1264 */
1265 gs->dflt_id_hdr_size = session->machines.host.id_hdr_size;
1266 /* And default ID for adding back a host-compatible ID sample */
1267 gs->dflt_id = evlist__first_id(session->evlist);
1268 if (!gs->dflt_id) {
1269 pr_err("Guest data has no sample IDs");
1270 return -EINVAL;
1271 }
1272
1273 /* Temporary file for guest events */
1274 gs->tmp_file_name = strdup(tmp_file_name);
1275 if (!gs->tmp_file_name)
1276 return -ENOMEM;
1277 gs->tmp_fd = mkstemp(gs->tmp_file_name);
1278 if (gs->tmp_fd < 0)
1279 return -errno;
1280
1281 if (zstd_init(&gs->session->zstd_data, 0) < 0)
1282 pr_warning("Guest session decompression initialization failed.\n");
1283
1284 /*
1285 * perf does not support processing 2 sessions simultaneously, so output
1286 * guest events to a temporary file.
1287 */
1288 ret = perf_session__process_events(gs->session);
1289 if (ret)
1290 return ret;
1291
1292 if (lseek(gs->tmp_fd, 0, SEEK_SET))
1293 return -errno;
1294
1295 return 0;
1296}
1297
1298/* Free hlist nodes assuming hlist_node is the first member of hlist entries */
1299static void free_hlist(struct hlist_head *heads, size_t hlist_sz)
1300{
1301 struct hlist_node *pos, *n;
1302 size_t i;
1303
1304 for (i = 0; i < hlist_sz; ++i) {
1305 hlist_for_each_safe(pos, n, &heads[i]) {
1306 hlist_del(pos);
1307 free(pos);
1308 }
1309 }
1310}
1311
1312static void guest_session__exit(struct guest_session *gs)
1313{
1314 if (gs->session) {
1315 perf_session__delete(gs->session);
1316 free_hlist(gs->heads, PERF_EVLIST__HLIST_SIZE);
1317 free_hlist(gs->tids, PERF_EVLIST__HLIST_SIZE);
1318 }
1319 if (gs->tmp_file_name) {
1320 if (gs->tmp_fd >= 0)
1321 close(gs->tmp_fd);
1322 unlink(gs->tmp_file_name);
79b40a1b 1323 zfree(&gs->tmp_file_name);
97406a7e 1324 }
79b40a1b
ACM		 1325 zfree(&gs->vcpu);
1326 zfree(&gs->perf_data_file);
97406a7e
AH		 1327}
1328
1329static void get_tsc_conv(struct perf_tsc_conversion *tc, struct perf_record_time_conv *time_conv)
1330{
1331 tc->time_shift = time_conv->time_shift;
1332 tc->time_mult = time_conv->time_mult;
1333 tc->time_zero = time_conv->time_zero;
1334 tc->time_cycles = time_conv->time_cycles;
1335 tc->time_mask = time_conv->time_mask;
1336 tc->cap_user_time_zero = time_conv->cap_user_time_zero;
1337 tc->cap_user_time_short = time_conv->cap_user_time_short;
1338}
1339
1340static void guest_session__get_tc(struct guest_session *gs)
1341{
1342 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1343
1344 get_tsc_conv(&gs->host_tc, &inject->session->time_conv);
1345 get_tsc_conv(&gs->guest_tc, &gs->session->time_conv);
1346}
1347
1348static void guest_session__convert_time(struct guest_session *gs, u64 guest_time, u64 *host_time)
1349{
1350 u64 tsc;
1351
1352 if (!guest_time) {
1353 *host_time = 0;
1354 return;
1355 }
1356
1357 if (gs->guest_tc.cap_user_time_zero)
1358 tsc = perf_time_to_tsc(guest_time, &gs->guest_tc);
1359 else
1360 tsc = guest_time;
1361
1362 /*
1363 * This is the correct order of operations for x86 if the TSC Offset and
1364 * Multiplier values are used.
1365 */
1366 tsc -= gs->time_offset;
1367 tsc /= gs->time_scale;
1368
1369 if (gs->host_tc.cap_user_time_zero)
1370 *host_time = tsc_to_perf_time(tsc, &gs->host_tc);
1371 else
1372 *host_time = tsc;
1373}
1374
1375static int guest_session__fetch(struct guest_session *gs)
1376{
892d00fb
IR		 1377 void *buf;
1378 struct perf_event_header *hdr;
97406a7e
AH		 1379 size_t hdr_sz = sizeof(*hdr);
1380 ssize_t ret;
1381
892d00fb
IR		 1382 buf = gs->ev.event_buf;
1383 if (!buf) {
1384 buf = malloc(PERF_SAMPLE_MAX_SIZE);
1385 if (!buf)
1386 return -ENOMEM;
1387 gs->ev.event_buf = buf;
1388 }
1389 hdr = buf;
97406a7e
AH		 1390 ret = readn(gs->tmp_fd, buf, hdr_sz);
1391 if (ret < 0)
1392 return ret;
1393
1394 if (!ret) {
1395 /* Zero size means EOF */
1396 hdr->size = 0;
1397 return 0;
1398 }
1399
1400 buf += hdr_sz;
1401
1402 ret = readn(gs->tmp_fd, buf, hdr->size - hdr_sz);
1403 if (ret < 0)
1404 return ret;
1405
1406 gs->ev.event = (union perf_event *)gs->ev.event_buf;
1407 gs->ev.sample.time = 0;
1408
1409 if (hdr->type >= PERF_RECORD_USER_TYPE_START) {
1410 pr_err("Unexpected type fetching guest event");
1411 return 0;
1412 }
1413
1414 ret = evlist__parse_sample(gs->session->evlist, gs->ev.event, &gs->ev.sample);
1415 if (ret) {
1416 pr_err("Parse failed fetching guest event");
1417 return ret;
1418 }
1419
1420 if (!gs->have_tc) {
1421 guest_session__get_tc(gs);
1422 gs->have_tc = true;
1423 }
1424
1425 guest_session__convert_time(gs, gs->ev.sample.time, &gs->ev.sample.time);
1426
1427 return 0;
1428}
1429
1430static int evlist__append_id_sample(struct evlist *evlist, union perf_event *ev,
1431 const struct perf_sample *sample)
1432{
1433 struct evsel *evsel;
1434 void *array;
1435 int ret;
1436
1437 evsel = evlist__id2evsel(evlist, sample->id);
1438 array = ev;
1439
1440 if (!evsel) {
1441 pr_err("No evsel for id %"PRIu64"\n", sample->id);
1442 return -EINVAL;
1443 }
1444
1445 array += ev->header.size;
1446 ret = perf_event__synthesize_id_sample(array, evsel->core.attr.sample_type, sample);
1447 if (ret < 0)
1448 return ret;
1449
1450 if (ret & 7) {
1451 pr_err("Bad id sample size %d\n", ret);
1452 return -EINVAL;
1453 }
1454
1455 ev->header.size += ret;
1456
1457 return 0;
1458}
1459
1460static int guest_session__inject_events(struct guest_session *gs, u64 timestamp)
1461{
1462 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1463 int ret;
1464
1465 if (!gs->ready)
1466 return 0;
1467
1468 while (1) {
1469 struct perf_sample *sample;
1470 struct guest_id *guest_id;
1471 union perf_event *ev;
1472 u16 id_hdr_size;
1473 u8 cpumode;
1474 u64 id;
1475
1476 if (!gs->fetched) {
1477 ret = guest_session__fetch(gs);
1478 if (ret)
1479 return ret;
1480 gs->fetched = true;
1481 }
1482
1483 ev = gs->ev.event;
1484 sample = &gs->ev.sample;
1485
1486 if (!ev->header.size)
1487 return 0; /* EOF */
1488
1489 if (sample->time > timestamp)
1490 return 0;
1491
1492 /* Change cpumode to guest */
1493 cpumode = ev->header.misc & PERF_RECORD_MISC_CPUMODE_MASK;
1494 if (cpumode & PERF_RECORD_MISC_USER)
1495 cpumode = PERF_RECORD_MISC_GUEST_USER;
1496 else
1497 cpumode = PERF_RECORD_MISC_GUEST_KERNEL;
1498 ev->header.misc &= ~PERF_RECORD_MISC_CPUMODE_MASK;
1499 ev->header.misc |= cpumode;
1500
1501 id = sample->id;
1502 if (!id) {
1503 id = gs->dflt_id;
1504 id_hdr_size = gs->dflt_id_hdr_size;
1505 } else {
1506 struct evsel *evsel = evlist__id2evsel(gs->session->evlist, id);
1507
1508 id_hdr_size = evsel__id_hdr_size(evsel);
1509 }
1510
1511 if (id_hdr_size & 7) {
1512 pr_err("Bad id_hdr_size %u\n", id_hdr_size);
1513 return -EINVAL;
1514 }
1515
1516 if (ev->header.size & 7) {
1517 pr_err("Bad event size %u\n", ev->header.size);
1518 return -EINVAL;
1519 }
1520
1521 /* Remove guest id sample */
1522 ev->header.size -= id_hdr_size;
1523
1524 if (ev->header.size & 7) {
1525 pr_err("Bad raw event size %u\n", ev->header.size);
1526 return -EINVAL;
1527 }
1528
1529 guest_id = guest_session__lookup_id(gs, id);
1530 if (!guest_id) {
1531 pr_err("Guest event with unknown id %llu\n",
1532 (unsigned long long)id);
1533 return -EINVAL;
1534 }
1535
1536 /* Change to host ID to avoid conflicting ID values */
1537 sample->id = guest_id->host_id;
1538 sample->stream_id = guest_id->host_id;
1539
1540 if (sample->cpu != (u32)-1) {
1541 if (sample->cpu >= gs->vcpu_cnt) {
1542 pr_err("Guest event with unknown VCPU %u\n",
1543 sample->cpu);
1544 return -EINVAL;
1545 }
1546 /* Change to host CPU instead of guest VCPU */
1547 sample->cpu = gs->vcpu[sample->cpu].cpu;
1548 }
1549
1550 /* New id sample with new ID and CPU */
1551 ret = evlist__append_id_sample(inject->session->evlist, ev, sample);
1552 if (ret)
1553 return ret;
1554
1555 if (ev->header.size & 7) {
1556 pr_err("Bad new event size %u\n", ev->header.size);
1557 return -EINVAL;
1558 }
1559
1560 gs->fetched = false;
1561
1562 ret = output_bytes(inject, ev, ev->header.size);
1563 if (ret)
1564 return ret;
1565 }
1566}
1567
1568static int guest_session__flush_events(struct guest_session *gs)
1569{
1570 return guest_session__inject_events(gs, -1);
1571}
1572
1573static int host__repipe(struct perf_tool *tool,
1574 union perf_event *event,
1575 struct perf_sample *sample,
1576 struct machine *machine)
1577{
1578 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1579 int ret;
1580
1581 ret = guest_session__inject_events(&inject->guest_session, sample->time);
1582 if (ret)
1583 return ret;
1584
1585 return perf_event__repipe(tool, event, sample, machine);
1586}
1587
1588static int host__finished_init(struct perf_session *session, union perf_event *event)
1589{
1590 struct perf_inject *inject = container_of(session->tool, struct perf_inject, tool);
1591 struct guest_session *gs = &inject->guest_session;
1592 int ret;
1593
1594 /*
1595 * Peek through host COMM events to find QEMU threads and the VCPU they
1596 * are running.
1597 */
1598 ret = host_peek_vm_comms(session, gs);
1599 if (ret)
1600 return ret;
1601
1602 if (!gs->vcpu_cnt) {
8147f79e 1603 pr_err("No VCPU threads found for pid %u\n", gs->machine_pid);
97406a7e
AH		 1604 return -EINVAL;
1605 }
1606
1607 /*
1608 * Allocate new (unused) host sample IDs and map them to the guest IDs.
1609 */
1610 gs->highest_id = evlist__find_highest_id(session->evlist);
1611 ret = guest_session__map_ids(gs, session->evlist);
1612 if (ret)
1613 return ret;
1614
1615 ret = guest_session__add_attrs(gs);
1616 if (ret)
1617 return ret;
1618
1619 ret = synthesize_id_index(inject, gs->session->evlist->core.nr_entries);
1620 if (ret) {
1621 pr_err("Failed to synthesize id_index\n");
1622 return ret;
1623 }
1624
1625 ret = guest_session__add_build_ids(gs);
1626 if (ret) {
1627 pr_err("Failed to add guest build IDs\n");
1628 return ret;
1629 }
1630
1631 gs->ready = true;
1632
1633 ret = guest_session__inject_events(gs, 0);
1634 if (ret)
1635 return ret;
1636
1637 return perf_event__repipe_op2_synth(session, event);
1638}
1639
1640/*
1641 * Obey finished-round ordering. The FINISHED_ROUND event is first processed
1642 * which flushes host events to file up until the last flush time. Then inject
1643 * guest events up to the same time. Finally write out the FINISHED_ROUND event
1644 * itself.
1645 */
1646static int host__finished_round(struct perf_tool *tool,
1647 union perf_event *event,
1648 struct ordered_events *oe)
1649{
1650 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1651 int ret = perf_event__process_finished_round(tool, event, oe);
1652 u64 timestamp = ordered_events__last_flush_time(oe);
1653
1654 if (ret)
1655 return ret;
1656
1657 ret = guest_session__inject_events(&inject->guest_session, timestamp);
1658 if (ret)
1659 return ret;
1660
1661 return perf_event__repipe_oe_synth(tool, event, oe);
1662}
1663
1664static int host__context_switch(struct perf_tool *tool,
1665 union perf_event *event,
1666 struct perf_sample *sample,
1667 struct machine *machine)
1668{
1669 struct perf_inject *inject = container_of(tool, struct perf_inject, tool);
1670 bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1671 struct guest_session *gs = &inject->guest_session;
1672 u32 pid = event->context_switch.next_prev_pid;
1673 u32 tid = event->context_switch.next_prev_tid;
1674 struct guest_tid *guest_tid;
1675 u32 vcpu;
1676
1677 if (out || pid != gs->machine_pid)
1678 goto out;
1679
1680 guest_tid = guest_session__lookup_tid(gs, tid);
1681 if (!guest_tid)
1682 goto out;
1683
1684 if (sample->cpu == (u32)-1) {
1685 pr_err("Switch event does not have CPU\n");
1686 return -EINVAL;
1687 }
1688
1689 vcpu = guest_tid->vcpu;
1690 if (vcpu >= gs->vcpu_cnt)
1691 return -EINVAL;
1692
1693 /* Guest is switching in, record which CPU the VCPU is now running on */
1694 gs->vcpu[vcpu].cpu = sample->cpu;
1695out:
1696 return host__repipe(tool, event, sample, machine);
1697}
1698
1d037ca1 1699static void sig_handler(int sig __maybe_unused)
454c407e
TZ		 1700{
1701 session_done = 1;
1702}
1703
b14b36d0 1704static int evsel__check_stype(struct evsel *evsel, u64 sample_type, const char *sample_msg)
26a031e1 1705{
1fc632ce 1706 struct perf_event_attr *attr = &evsel->core.attr;
8ab2e96d 1707 const char *name = evsel__name(evsel);
26a031e1
AV		 1708
1709 if (!(attr->sample_type & sample_type)) {
1710 pr_err("Samples for %s event do not have %s attribute set.",
1711 name, sample_msg);
1712 return -EINVAL;
1713 }
1714
1715 return 0;
1716}
1717
f56fb986
AH		 1718static int drop_sample(struct perf_tool *tool __maybe_unused,
1719 union perf_event *event __maybe_unused,
1720 struct perf_sample *sample __maybe_unused,
32dcd021 1721 struct evsel *evsel __maybe_unused,
f56fb986
AH		 1722 struct machine *machine __maybe_unused)
1723{
1724 return 0;
1725}
1726
1727static void strip_init(struct perf_inject *inject)
1728{
63503dba 1729 struct evlist *evlist = inject->session->evlist;
32dcd021 1730 struct evsel *evsel;
f56fb986
AH		 1731
1732 inject->tool.context_switch = perf_event__drop;
1733
e5cadb93 1734 evlist__for_each_entry(evlist, evsel)
f56fb986
AH		 1735 evsel->handler = drop_sample;
1736}
1737
83d7f5f1
AH		 1738static int parse_vm_time_correlation(const struct option *opt, const char *str, int unset)
1739{
1740 struct perf_inject *inject = opt->value;
1741 const char *args;
1742 char *dry_run;
1743
1744 if (unset)
1745 return 0;
1746
1747 inject->itrace_synth_opts.set = true;
1748 inject->itrace_synth_opts.vm_time_correlation = true;
1749 inject->in_place_update = true;
1750
1751 if (!str)
1752 return 0;
1753
1754 dry_run = skip_spaces(str);
1755 if (!strncmp(dry_run, "dry-run", strlen("dry-run"))) {
1756 inject->itrace_synth_opts.vm_tm_corr_dry_run = true;
1757 inject->in_place_update_dry_run = true;
1758 args = dry_run + strlen("dry-run");
1759 } else {
1760 args = str;
1761 }
1762
1763 inject->itrace_synth_opts.vm_tm_corr_args = strdup(args);
1764
1765 return inject->itrace_synth_opts.vm_tm_corr_args ? 0 : -ENOMEM;
1766}
1767
97406a7e
AH		 1768static int parse_guest_data(const struct option *opt, const char *str, int unset)
1769{
1770 struct perf_inject *inject = opt->value;
1771 struct guest_session *gs = &inject->guest_session;
1772 char *tok;
1773 char *s;
1774
1775 if (unset)
1776 return 0;
1777
1778 if (!str)
1779 goto bad_args;
1780
1781 s = strdup(str);
1782 if (!s)
1783 return -ENOMEM;
1784
1785 gs->perf_data_file = strsep(&s, ",");
1786 if (!gs->perf_data_file)
1787 goto bad_args;
1788
1789 gs->copy_kcore_dir = has_kcore_dir(gs->perf_data_file);
1790 if (gs->copy_kcore_dir)
1791 inject->output.is_dir = true;
1792
1793 tok = strsep(&s, ",");
1794 if (!tok)
1795 goto bad_args;
1796 gs->machine_pid = strtoul(tok, NULL, 0);
1797 if (!inject->guest_session.machine_pid)
1798 goto bad_args;
1799
1800 gs->time_scale = 1;
1801
1802 tok = strsep(&s, ",");
1803 if (!tok)
1804 goto out;
1805 gs->time_offset = strtoull(tok, NULL, 0);
1806
1807 tok = strsep(&s, ",");
1808 if (!tok)
1809 goto out;
1810 gs->time_scale = strtod(tok, NULL);
1811 if (!gs->time_scale)
1812 goto bad_args;
1813out:
1814 return 0;
1815
1816bad_args:
1817 pr_err("--guest-data option requires guest perf.data file name, "
1818 "guest machine PID, and optionally guest timestamp offset, "
1819 "and guest timestamp scale factor, separated by commas.\n");
1820 return -1;
1821}
1822
180b3d06
AH		 1823static int save_section_info_cb(struct perf_file_section *section,
1824 struct perf_header *ph __maybe_unused,
1825 int feat, int fd __maybe_unused, void *data)
1826{
1827 struct perf_inject *inject = data;
1828
1829 inject->secs[feat] = *section;
1830 return 0;
1831}
1832
1833static int save_section_info(struct perf_inject *inject)
1834{
1835 struct perf_header *header = &inject->session->header;
1836 int fd = perf_data__fd(inject->session->data);
1837
1838 return perf_header__process_sections(header, fd, inject, save_section_info_cb);
1839}
1840
1841static bool keep_feat(int feat)
1842{
1843 switch (feat) {
1844 /* Keep original information that describes the machine or software */
1845 case HEADER_TRACING_DATA:
1846 case HEADER_HOSTNAME:
1847 case HEADER_OSRELEASE:
1848 case HEADER_VERSION:
1849 case HEADER_ARCH:
1850 case HEADER_NRCPUS:
1851 case HEADER_CPUDESC:
1852 case HEADER_CPUID:
1853 case HEADER_TOTAL_MEM:
1854 case HEADER_CPU_TOPOLOGY:
1855 case HEADER_NUMA_TOPOLOGY:
1856 case HEADER_PMU_MAPPINGS:
1857 case HEADER_CACHE:
1858 case HEADER_MEM_TOPOLOGY:
1859 case HEADER_CLOCKID:
1860 case HEADER_BPF_PROG_INFO:
1861 case HEADER_BPF_BTF:
1862 case HEADER_CPU_PMU_CAPS:
1863 case HEADER_CLOCK_DATA:
1864 case HEADER_HYBRID_TOPOLOGY:
2139f742 1865 case HEADER_PMU_CAPS:
180b3d06
AH		 1866 return true;
1867 /* Information that can be updated */
1868 case HEADER_BUILD_ID:
1869 case HEADER_CMDLINE:
1870 case HEADER_EVENT_DESC:
1871 case HEADER_BRANCH_STACK:
1872 case HEADER_GROUP_DESC:
1873 case HEADER_AUXTRACE:
1874 case HEADER_STAT:
1875 case HEADER_SAMPLE_TIME:
1876 case HEADER_DIR_FORMAT:
1877 case HEADER_COMPRESSED:
1878 default:
1879 return false;
1880 };
1881}
1882
1883static int read_file(int fd, u64 offs, void *buf, size_t sz)
1884{
1885 ssize_t ret = preadn(fd, buf, sz, offs);
1886
1887 if (ret < 0)
1888 return -errno;
1889 if ((size_t)ret != sz)
1890 return -EINVAL;
1891 return 0;
1892}
1893
1894static int feat_copy(struct perf_inject *inject, int feat, struct feat_writer *fw)
1895{
1896 int fd = perf_data__fd(inject->session->data);
1897 u64 offs = inject->secs[feat].offset;
1898 size_t sz = inject->secs[feat].size;
1899 void *buf = malloc(sz);
1900 int ret;
1901
1902 if (!buf)
1903 return -ENOMEM;
1904
1905 ret = read_file(fd, offs, buf, sz);
1906 if (ret)
1907 goto out_free;
1908
1909 ret = fw->write(fw, buf, sz);
1910out_free:
1911 free(buf);
1912 return ret;
1913}
1914
1915struct inject_fc {
1916 struct feat_copier fc;
1917 struct perf_inject *inject;
1918};
1919
1920static int feat_copy_cb(struct feat_copier *fc, int feat, struct feat_writer *fw)
1921{
1922 struct inject_fc *inj_fc = container_of(fc, struct inject_fc, fc);
1923 struct perf_inject *inject = inj_fc->inject;
1924 int ret;
1925
1926 if (!inject->secs[feat].offset ||
1927 !keep_feat(feat))
1928 return 0;
1929
1930 ret = feat_copy(inject, feat, fw);
1931 if (ret < 0)
1932 return ret;
1933
1934 return 1; /* Feature section copied */
1935}
1936
d8fc0855
AH		 1937static int copy_kcore_dir(struct perf_inject *inject)
1938{
1939 char *cmd;
1940 int ret;
1941
1942 ret = asprintf(&cmd, "cp -r -n %s/kcore_dir* %s >/dev/null 2>&1",
1943 inject->input_name, inject->output.path);
1944 if (ret < 0)
1945 return ret;
1946 pr_debug("%s\n", cmd);
342cb0d8
AH		 1947 ret = system(cmd);
1948 free(cmd);
1949 return ret;
d8fc0855
AH		 1950}
1951
97406a7e
AH		 1952static int guest_session__copy_kcore_dir(struct guest_session *gs)
1953{
1954 struct perf_inject *inject = container_of(gs, struct perf_inject, guest_session);
1955 char *cmd;
1956 int ret;
1957
1958 ret = asprintf(&cmd, "cp -r -n %s/kcore_dir %s/kcore_dir__%u >/dev/null 2>&1",
1959 gs->perf_data_file, inject->output.path, gs->machine_pid);
1960 if (ret < 0)
1961 return ret;
1962 pr_debug("%s\n", cmd);
1963 ret = system(cmd);
1964 free(cmd);
1965 return ret;
1966}
1967
c271a55b
AH		 1968static int output_fd(struct perf_inject *inject)
1969{
1970 return inject->in_place_update ? -1 : perf_data__fd(&inject->output);
1971}
1972
5ded57ac 1973static int __cmd_inject(struct perf_inject *inject)
454c407e 1974{
454c407e 1975 int ret = -EINVAL;
97406a7e 1976 struct guest_session *gs = &inject->guest_session;
1cb8bdcc 1977 struct perf_session *session = inject->session;
c271a55b 1978 int fd = output_fd(inject);
0f0aa5e0 1979 u64 output_data_offset;
454c407e
TZ		 1980
1981 signal(SIGINT, sig_handler);
1982
0f0aa5e0 1983 if (inject->build_ids || inject->sched_stat ||
27c9c342 1984 inject->itrace_synth_opts.set || inject->build_id_all) {
5ded57ac 1985 inject->tool.mmap = perf_event__repipe_mmap;
5c5e854b 1986 inject->tool.mmap2 = perf_event__repipe_mmap2;
f62d3f0f 1987 inject->tool.fork = perf_event__repipe_fork;
378ef0f5 1988#ifdef HAVE_LIBTRACEEVENT
5ded57ac 1989 inject->tool.tracing_data = perf_event__repipe_tracing_data;
378ef0f5 1990#endif
454c407e
TZ		 1991 }
1992
37ed2cdd 1993 output_data_offset = perf_session__data_offset(session->evlist);
0f0aa5e0 1994
27c9c342
NK		 1995 if (inject->build_id_all) {
1996 inject->tool.mmap = perf_event__repipe_buildid_mmap;
1997 inject->tool.mmap2 = perf_event__repipe_buildid_mmap2;
1998 } else if (inject->build_ids) {
54a3cf59
AV		 1999 inject->tool.sample = perf_event__inject_buildid;
2000 } else if (inject->sched_stat) {
32dcd021 2001 struct evsel *evsel;
26a031e1 2002
e5cadb93 2003 evlist__for_each_entry(session->evlist, evsel) {
8ab2e96d 2004 const char *name = evsel__name(evsel);
26a031e1
AV		 2005
2006 if (!strcmp(name, "sched:sched_switch")) {
b14b36d0 2007 if (evsel__check_stype(evsel, PERF_SAMPLE_TID, "TID"))
26a031e1
AV		 2008 return -EINVAL;
2009
744a9719 2010 evsel->handler = perf_inject__sched_switch;
26a031e1 2011 } else if (!strcmp(name, "sched:sched_process_exit"))
744a9719 2012 evsel->handler = perf_inject__sched_process_exit;
378ef0f5 2013#ifdef HAVE_LIBTRACEEVENT
26a031e1 2014 else if (!strncmp(name, "sched:sched_stat_", 17))
744a9719 2015 evsel->handler = perf_inject__sched_stat;
378ef0f5 2016#endif
26a031e1 2017 }
83d7f5f1
AH		 2018 } else if (inject->itrace_synth_opts.vm_time_correlation) {
2019 session->itrace_synth_opts = &inject->itrace_synth_opts;
2020 memset(&inject->tool, 0, sizeof(inject->tool));
2021 inject->tool.id_index = perf_event__process_id_index;
2022 inject->tool.auxtrace_info = perf_event__process_auxtrace_info;
2023 inject->tool.auxtrace = perf_event__process_auxtrace;
2024 inject->tool.auxtrace_error = perf_event__process_auxtrace_error;
2025 inject->tool.ordered_events = true;
2026 inject->tool.ordering_requires_timestamps = true;
0f0aa5e0
AH		 2027 } else if (inject->itrace_synth_opts.set) {
2028 session->itrace_synth_opts = &inject->itrace_synth_opts;
2029 inject->itrace_synth_opts.inject = true;
2030 inject->tool.comm = perf_event__repipe_comm;
f3b3614a 2031 inject->tool.namespaces = perf_event__repipe_namespaces;
0f0aa5e0 2032 inject->tool.exit = perf_event__repipe_exit;
29f6eeca 2033 inject->tool.id_index = perf_event__process_id_index;
0f0aa5e0
AH		 2034 inject->tool.auxtrace_info = perf_event__process_auxtrace_info;
2035 inject->tool.auxtrace = perf_event__process_auxtrace;
73117308 2036 inject->tool.aux = perf_event__drop_aux;
61750473
AH		 2037 inject->tool.itrace_start = perf_event__drop_aux;
2038 inject->tool.aux_output_hw_id = perf_event__drop_aux;
0f0aa5e0
AH		 2039 inject->tool.ordered_events = true;
2040 inject->tool.ordering_requires_timestamps = true;
2041 /* Allow space in the header for new attributes */
c29d9792 2042 output_data_offset = roundup(8192 + session->header.data_offset, 4096);
f56fb986
AH		 2043 if (inject->strip)
2044 strip_init(inject);
97406a7e
AH		 2045 } else if (gs->perf_data_file) {
2046 char *name = gs->perf_data_file;
2047
2048 /*
2049 * Not strictly necessary, but keep these events in order wrt
2050 * guest events.
2051 */
2052 inject->tool.mmap = host__repipe;
2053 inject->tool.mmap2 = host__repipe;
2054 inject->tool.comm = host__repipe;
2055 inject->tool.fork = host__repipe;
2056 inject->tool.exit = host__repipe;
2057 inject->tool.lost = host__repipe;
2058 inject->tool.context_switch = host__repipe;
2059 inject->tool.ksymbol = host__repipe;
2060 inject->tool.text_poke = host__repipe;
2061 /*
2062 * Once the host session has initialized, set up sample ID
2063 * mapping and feed in guest attrs, build IDs and initial
2064 * events.
2065 */
2066 inject->tool.finished_init = host__finished_init;
2067 /* Obey finished round ordering */
2068 inject->tool.finished_round = host__finished_round,
2069 /* Keep track of which CPU a VCPU is runnng on */
2070 inject->tool.context_switch = host__context_switch;
2071 /*
2072 * Must order events to be able to obey finished round
2073 * ordering.
2074 */
2075 inject->tool.ordered_events = true;
2076 inject->tool.ordering_requires_timestamps = true;
2077 /* Set up a separate session to process guest perf.data file */
2078 ret = guest_session__start(gs, name, session->data->force);
2079 if (ret) {
2080 pr_err("Failed to process %s, error %d\n", name, ret);
2081 return ret;
2082 }
2083 /* Allow space in the header for guest attributes */
2084 output_data_offset += gs->session->header.data_offset;
2085 output_data_offset = roundup(output_data_offset, 4096);
26a031e1
AV		 2086 }
2087
99fa2984
AH		 2088 if (!inject->itrace_synth_opts.set)
2089 auxtrace_index__free(&session->auxtrace_index);
2090
fea20d66 2091 if (!inject->is_pipe && !inject->in_place_update)
0f0aa5e0 2092 lseek(fd, output_data_offset, SEEK_SET);
e558a5bd 2093
b7b61cbe 2094 ret = perf_session__process_events(session);
bb8d521f
DCC		 2095 if (ret)
2096 return ret;
454c407e 2097
97406a7e
AH		 2098 if (gs->session) {
2099 /*
2100 * Remaining guest events have later timestamps. Flush them
2101 * out to file.
2102 */
2103 ret = guest_session__flush_events(gs);
2104 if (ret) {
2105 pr_err("Failed to flush guest events\n");
2106 return ret;
2107 }
2108 }
2109
fea20d66 2110 if (!inject->is_pipe && !inject->in_place_update) {
180b3d06
AH		 2111 struct inject_fc inj_fc = {
2112 .fc.copy = feat_copy_cb,
2113 .inject = inject,
2114 };
2115
640dad47 2116 if (inject->build_ids)
e38b43c3
AH		 2117 perf_header__set_feat(&session->header,
2118 HEADER_BUILD_ID);
640dad47
AH		 2119 /*
2120 * Keep all buildids when there is unprocessed AUX data because
2121 * it is not known which ones the AUX trace hits.
2122 */
2123 if (perf_header__has_feat(&session->header, HEADER_BUILD_ID) &&
2124 inject->have_auxtrace && !inject->itrace_synth_opts.set)
2125 dsos__hit_all(session);
0f0aa5e0
AH		 2126 /*
2127 * The AUX areas have been removed and replaced with
1c756cd4 2128 * synthesized hardware events, so clear the feature flag.
0f0aa5e0 2129 */
051a01b9 2130 if (inject->itrace_synth_opts.set) {
0f0aa5e0
AH		 2131 perf_header__clear_feat(&session->header,
2132 HEADER_AUXTRACE);
ec90e42c
AH		 2133 if (inject->itrace_synth_opts.last_branch ||
2134 inject->itrace_synth_opts.add_last_branch)
051a01b9
AH		 2135 perf_header__set_feat(&session->header,
2136 HEADER_BRANCH_STACK);
2137 }
0f0aa5e0 2138 session->header.data_offset = output_data_offset;
e558a5bd 2139 session->header.data_size = inject->bytes_written;
180b3d06 2140 perf_session__inject_header(session, session->evlist, fd, &inj_fc.fc);
d8fc0855
AH		 2141
2142 if (inject->copy_kcore_dir) {
2143 ret = copy_kcore_dir(inject);
97406a7e
AH		 2144 if (ret) {
2145 pr_err("Failed to copy kcore\n");
d8fc0855 2146 return ret;
97406a7e
AH		 2147 }
2148 }
2149 if (gs->copy_kcore_dir) {
2150 ret = guest_session__copy_kcore_dir(gs);
2151 if (ret) {
2152 pr_err("Failed to copy guest kcore\n");
2153 return ret;
2154 }
d8fc0855 2155 }
e558a5bd
AV		 2156 }
2157
454c407e
TZ		 2158 return ret;
2159}
2160
b0ad8ea6 2161int cmd_inject(int argc, const char **argv)
454c407e 2162{
5ded57ac
ACM		 2163 struct perf_inject inject = {
2164 .tool = {
2165 .sample = perf_event__repipe_sample,
2946eced 2166 .read = perf_event__repipe_sample,
5ded57ac 2167 .mmap = perf_event__repipe,
5c5e854b 2168 .mmap2 = perf_event__repipe,
5ded57ac 2169 .comm = perf_event__repipe,
2946eced
NK		 2170 .namespaces = perf_event__repipe,
2171 .cgroup = perf_event__repipe,
5ded57ac
ACM		 2172 .fork = perf_event__repipe,
2173 .exit = perf_event__repipe,
2174 .lost = perf_event__repipe,
d8145b3e 2175 .lost_samples = perf_event__repipe,
4a96f7a0 2176 .aux = perf_event__repipe,
0ad21f68 2177 .itrace_start = perf_event__repipe,
61750473 2178 .aux_output_hw_id = perf_event__repipe,
0286039f 2179 .context_switch = perf_event__repipe,
5ded57ac
ACM		 2180 .throttle = perf_event__repipe,
2181 .unthrottle = perf_event__repipe,
2946eced
NK		 2182 .ksymbol = perf_event__repipe,
2183 .bpf = perf_event__repipe,
2184 .text_poke = perf_event__repipe,
5ded57ac 2185 .attr = perf_event__repipe_attr,
2946eced 2186 .event_update = perf_event__repipe_event_update,
47c3d109 2187 .tracing_data = perf_event__repipe_op2_synth,
d704ebda 2188 .finished_round = perf_event__repipe_oe_synth,
5ded57ac 2189 .build_id = perf_event__repipe_op2_synth,
3c659eed 2190 .id_index = perf_event__repipe_op2_synth,
2946eced
NK		 2191 .auxtrace_info = perf_event__repipe_op2_synth,
2192 .auxtrace_error = perf_event__repipe_op2_synth,
2193 .time_conv = perf_event__repipe_op2_synth,
2194 .thread_map = perf_event__repipe_op2_synth,
2195 .cpu_map = perf_event__repipe_op2_synth,
2196 .stat_config = perf_event__repipe_op2_synth,
2197 .stat = perf_event__repipe_op2_synth,
2198 .stat_round = perf_event__repipe_op2_synth,
e9def1b2 2199 .feature = perf_event__repipe_op2_synth,
3812d298 2200 .finished_init = perf_event__repipe_op2_synth,
2946eced
NK		 2201 .compressed = perf_event__repipe_op4_synth,
2202 .auxtrace = perf_event__repipe_auxtrace,
5ded57ac 2203 },
e558a5bd 2204 .input_name = "-",
26a031e1 2205 .samples = LIST_HEAD_INIT(inject.samples),
3406912c 2206 .output = {
2d4f2799
JO		 2207 .path = "-",
2208 .mode = PERF_DATA_MODE_WRITE,
60136667 2209 .use_stdio = true,
3406912c 2210 },
5ded57ac 2211 };
8ceb41d7 2212 struct perf_data data = {
1cb8bdcc 2213 .mode = PERF_DATA_MODE_READ,
60136667 2214 .use_stdio = true,
1cb8bdcc
NK		 2215 };
2216 int ret;
c3a057dc 2217 bool repipe = true;
8012243e 2218 const char *known_build_ids = NULL;
1cb8bdcc 2219
9b07e27f 2220 struct option options[] = {
5ded57ac
ACM		 2221 OPT_BOOLEAN('b', "build-ids", &inject.build_ids,
2222 "Inject build-ids into the output stream"),
27c9c342
NK		 2223 OPT_BOOLEAN(0, "buildid-all", &inject.build_id_all,
2224 "Inject build-ids of all DSOs into the output stream"),
8012243e
RS		 2225 OPT_STRING(0, "known-build-ids", &known_build_ids,
2226 "buildid path [,buildid path...]",
2227 "build-ids to use for given paths"),
e558a5bd
AV		 2228 OPT_STRING('i', "input", &inject.input_name, "file",
2229 "input file name"),
2d4f2799 2230 OPT_STRING('o', "output", &inject.output.path, "file",
e558a5bd 2231 "output file name"),
26a031e1
AV		 2232 OPT_BOOLEAN('s', "sched-stat", &inject.sched_stat,
2233 "Merge sched-stat and sched-switch for getting events "
2234 "where and how long tasks slept"),
e12b202f 2235#ifdef HAVE_JITDUMP
9b07e27f 2236 OPT_BOOLEAN('j', "jit", &inject.jit_mode, "merge jitdump files into perf.data file"),
e12b202f 2237#endif
5ded57ac
ACM		 2238 OPT_INCR('v', "verbose", &verbose,
2239 "be more verbose (show build ids, etc)"),
b3a018fc
JC		 2240 OPT_STRING('k', "vmlinux", &symbol_conf.vmlinux_name,
2241 "file", "vmlinux pathname"),
2242 OPT_BOOLEAN(0, "ignore-vmlinux", &symbol_conf.ignore_vmlinux,
2243 "don't load vmlinux even if found"),
a7a2b8b4
AH		 2244 OPT_STRING(0, "kallsyms", &symbol_conf.kallsyms_name, "file",
2245 "kallsyms pathname"),
8ceb41d7 2246 OPT_BOOLEAN('f', "force", &data.force, "don't complain, do it"),
0f0aa5e0 2247 OPT_CALLBACK_OPTARG(0, "itrace", &inject.itrace_synth_opts,
c12e039d
AK		 2248 NULL, "opts", "Instruction Tracing options\n"
2249 ITRACE_HELP,
0f0aa5e0 2250 itrace_parse_synth_opts),
f56fb986
AH		 2251 OPT_BOOLEAN(0, "strip", &inject.strip,
2252 "strip non-synthesized events (use with --itrace)"),
83d7f5f1
AH		 2253 OPT_CALLBACK_OPTARG(0, "vm-time-correlation", &inject, NULL, "opts",
2254 "correlate time between VM guests and the host",
2255 parse_vm_time_correlation),
97406a7e
AH		 2256 OPT_CALLBACK_OPTARG(0, "guest-data", &inject, NULL, "opts",
2257 "inject events from a guest perf.data file",
2258 parse_guest_data),
2259 OPT_STRING(0, "guestmount", &symbol_conf.guestmount, "directory",
2260 "guest mount directory under which every guest os"
2261 " instance has a subdir"),
5ded57ac
ACM		 2262 OPT_END()
2263 };
002439e8
ACM		 2264 const char * const inject_usage[] = {
2265 "perf inject [<options>]",
2266 NULL
2267 };
1a27fc01
IR		 2268
2269 if (!inject.itrace_synth_opts.set) {
2270 /* Disable eager loading of kernel symbols that adds overhead to perf inject. */
2271 symbol_conf.lazy_load_kernel_maps = true;
2272 }
2273
e12b202f 2274#ifndef HAVE_JITDUMP
9b07e27f
SE		 2275 set_option_nobuild(options, 'j', "jit", "NO_LIBELF=1", true);
2276#endif
002439e8 2277 argc = parse_options(argc, argv, options, inject_usage, 0);
454c407e
TZ		 2278
2279 /*
2280 * Any (unrecognized) arguments left?
2281 */
2282 if (argc)
002439e8 2283 usage_with_options(inject_usage, options);
454c407e 2284
f56fb986
AH		 2285 if (inject.strip && !inject.itrace_synth_opts.set) {
2286 pr_err("--strip option requires --itrace option\n");
2287 return -1;
2288 }
2289
b3a018fc
JC		 2290 if (symbol__validate_sym_arguments())
2291 return -1;
2292
2a525f6a
AH		 2293 if (inject.in_place_update) {
2294 if (!strcmp(inject.input_name, "-")) {
2295 pr_err("Input file name required for in-place updating\n");
2296 return -1;
2297 }
2298 if (strcmp(inject.output.path, "-")) {
2299 pr_err("Output file name must not be specified for in-place updating\n");
2300 return -1;
2301 }
2302 if (!data.force && !inject.in_place_update_dry_run) {
2303 pr_err("The input file would be updated in place, "
2304 "the --force option is required.\n");
2305 return -1;
2306 }
2307 if (!inject.in_place_update_dry_run)
2308 data.in_place_update = true;
d8fc0855
AH		 2309 } else {
2310 if (strcmp(inject.output.path, "-") && !inject.strip &&
2311 has_kcore_dir(inject.input_name)) {
2312 inject.output.is_dir = true;
2313 inject.copy_kcore_dir = true;
2314 }
2315 if (perf_data__open(&inject.output)) {
2316 perror("failed to create output file");
2317 return -1;
2318 }
e558a5bd
AV		 2319 }
2320
2d4f2799 2321 data.path = inject.input_name;
c3a057dc 2322 if (!strcmp(inject.input_name, "-") || inject.output.is_pipe) {
fea20d66 2323 inject.is_pipe = true;
c3a057dc
NK		 2324 /*
2325 * Do not repipe header when input is a regular file
2326 * since either it can rewrite the header at the end
2327 * or write a new pipe header.
2328 */
2329 if (strcmp(inject.input_name, "-"))
2330 repipe = false;
2331 }
fea20d66 2332
c3a057dc 2333 inject.session = __perf_session__new(&data, repipe,
c271a55b 2334 output_fd(&inject),
fea20d66 2335 &inject.tool);
02e6246f
RM		 2336 if (IS_ERR(inject.session)) {
2337 ret = PTR_ERR(inject.session);
2338 goto out_close_output;
2339 }
1cb8bdcc 2340
371a3378
AB		 2341 if (zstd_init(&(inject.session->zstd_data), 0) < 0)
2342 pr_warning("Decompression initialization failed.\n");
2343
180b3d06
AH		 2344 /* Save original section info before feature bits change */
2345 ret = save_section_info(&inject);
2346 if (ret)
2347 goto out_delete;
2348
c3a057dc
NK		 2349 if (!data.is_pipe && inject.output.is_pipe) {
2350 ret = perf_header__write_pipe(perf_data__fd(&inject.output));
2351 if (ret < 0) {
2352 pr_err("Couldn't write a new pipe header.\n");
2353 goto out_delete;
2354 }
2355
2356 ret = perf_event__synthesize_for_pipe(&inject.tool,
2357 inject.session,
2358 &inject.output,
2359 perf_event__repipe);
2360 if (ret < 0)
2361 goto out_delete;
2362 }
2363
27c9c342 2364 if (inject.build_ids && !inject.build_id_all) {
921f3fad
ACM		 2365 /*
2366 * to make sure the mmap records are ordered correctly
2367 * and so that the correct especially due to jitted code
2368 * mmaps. We cannot generate the buildid hit list and
2369 * inject the jit mmaps at the same time for now.
2370 */
2371 inject.tool.ordered_events = true;
2372 inject.tool.ordering_requires_timestamps = true;
8012243e
RS		 2373 if (known_build_ids != NULL) {
2374 inject.known_build_ids =
2375 perf_inject__parse_known_build_ids(known_build_ids);
2376
2377 if (inject.known_build_ids == NULL) {
2378 pr_err("Couldn't parse known build ids.\n");
2379 goto out_delete;
2380 }
2381 }
921f3fad 2382 }
27c9c342
NK		 2383
2384 if (inject.sched_stat) {
2385 inject.tool.ordered_events = true;
2386 }
2387
e12b202f 2388#ifdef HAVE_JITDUMP
9b07e27f 2389 if (inject.jit_mode) {
9b07e27f
SE		 2390 inject.tool.mmap2 = perf_event__jit_repipe_mmap2;
2391 inject.tool.mmap = perf_event__jit_repipe_mmap;
2392 inject.tool.ordered_events = true;
2393 inject.tool.ordering_requires_timestamps = true;
2394 /*
2395 * JIT MMAP injection injects all MMAP events in one go, so it
2396 * does not obey finished_round semantics.
2397 */
2398 inject.tool.finished_round = perf_event__drop_oe;
2399 }
2400#endif
9fedfb0c
TS		 2401 ret = symbol__init(&inject.session->header.env);
2402 if (ret < 0)
2403 goto out_delete;
454c407e 2404
1cb8bdcc
NK		 2405 ret = __cmd_inject(&inject);
2406
97406a7e
AH		 2407 guest_session__exit(&inject.guest_session);
2408
9fedfb0c 2409out_delete:
8012243e 2410 strlist__delete(inject.known_build_ids);
371a3378 2411 zstd_fini(&(inject.session->zstd_data));
1cb8bdcc 2412 perf_session__delete(inject.session);
02e6246f 2413out_close_output:
0c8e32fe
AH		 2414 if (!inject.in_place_update)
2415 perf_data__close(&inject.output);
83d7f5f1 2416 free(inject.itrace_synth_opts.vm_tm_corr_args);
d3944f0e 2417 free(inject.event_copy);
892d00fb 2418 free(inject.guest_session.ev.event_buf);
1cb8bdcc 2419 return ret;
454c407e 2420}
Linux 6.x block layer and io_uring tree(s)