projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
perf record: Introduce --threads command line option
[linux-2.6-block.git] / tools / perf / builtin-record.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
abaff32a 2/*
bf9e1876
IM		 3 * builtin-record.c
4 *
5 * Builtin record command: Record the profile of a workload
6 * (or a CPU, or a PID) into the perf.data output file - for
7 * later analysis via perf report.
abaff32a 8 */
16f762a2 9#include "builtin.h"
bf9e1876 10
6122e4e4 11#include "util/build-id.h"
4b6ab94e 12#include <subcmd/parse-options.h>
8ad8db37 13#include "util/parse-events.h"
41840d21 14#include "util/config.h"
6eda5838 15
8f651eae 16#include "util/callchain.h"
f14d5707 17#include "util/cgroup.h"
7c6a1c65 18#include "util/header.h"
66e274f3 19#include "util/event.h"
361c99a6 20#include "util/evlist.h"
69aad6f1 21#include "util/evsel.h"
8f28827a 22#include "util/debug.h"
e0fcfb08 23#include "util/mmap.h"
aeb00b1a 24#include "util/target.h"
94c744b6 25#include "util/session.h"
45694aa7 26#include "util/tool.h"
8d06367f 27#include "util/symbol.h"
aeb00b1a 28#include "util/record.h"
a12b51c4 29#include "util/cpumap.h"
fd78260b 30#include "util/thread_map.h"
f5fc1412 31#include "util/data.h"
bcc84ec6 32#include "util/perf_regs.h"
ef149c25 33#include "util/auxtrace.h"
46bc29b9 34#include "util/tsc.h"
f00898f4 35#include "util/parse-branch-options.h"
bcc84ec6 36#include "util/parse-regs-options.h"
40c7d246 37#include "util/perf_api_probe.h"
71dc2326 38#include "util/llvm-utils.h"
8690a2a7 39#include "util/bpf-loader.h"
5f9cf599 40#include "util/trigger.h"
a074865e 41#include "util/perf-hooks.h"
f13de660 42#include "util/cpu-set-sched.h"
ea49e01c 43#include "util/synthetic-events.h"
c5e4027e 44#include "util/time-utils.h"
58db1d6e 45#include "util/units.h"
7b612e29 46#include "util/bpf-event.h"
d99c22ea 47#include "util/util.h"
70943490 48#include "util/pfm.h"
6953beb4 49#include "util/clockid.h"
b53a0755
JY		 50#include "util/pmu-hybrid.h"
51#include "util/evlist-hybrid.h"
d8871ea7 52#include "asm/bug.h"
c1a604df 53#include "perf.h"
7c6a1c65 54
a43783ae 55#include <errno.h>
fd20e811 56#include <inttypes.h>
67230479 57#include <locale.h>
4208735d 58#include <poll.h>
d99c22ea 59#include <pthread.h>
97124d5e 60#include <unistd.h>
415ccb58
AB		 61#ifndef HAVE_GETTID
62#include <syscall.h>
63#endif
de9ac07b 64#include <sched.h>
9607ad3a 65#include <signal.h>
da231338
AM		 66#ifdef HAVE_EVENTFD_SUPPORT
67#include <sys/eventfd.h>
68#endif
a41794cd 69#include <sys/mman.h>
4208735d 70#include <sys/wait.h>
eeb399b5
AH		 71#include <sys/types.h>
72#include <sys/stat.h>
73#include <fcntl.h>
6ef81c55 74#include <linux/err.h>
8520a98d 75#include <linux/string.h>
0693e680 76#include <linux/time64.h>
d8f9da24 77#include <linux/zalloc.h>
8384a260 78#include <linux/bitmap.h>
d1e325cf 79#include <sys/time.h>
78da39fa 80
1b43b704 81struct switch_output {
dc0c6127 82 bool enabled;
1b43b704 83 bool signal;
dc0c6127 84 unsigned long size;
bfacbe3b 85 unsigned long time;
cb4e1ebb
JO		 86 const char *str;
87 bool set;
03724b2e
AK		 88 char **filenames;
89 int num_files;
90 int cur_file;
1b43b704
JO		 91};
92
7954f716
AB		 93struct thread_mask {
94 struct mmap_cpu_mask maps;
95 struct mmap_cpu_mask affinity;
96};
97
415ccb58
AB		 98struct record_thread {
99 pid_t tid;
100 struct thread_mask *mask;
101 struct {
102 int msg[2];
103 int ack[2];
104 } pipes;
105 struct fdarray pollfd;
106 int ctlfd_pos;
107 int nr_mmaps;
108 struct mmap **maps;
109 struct mmap **overwrite_maps;
110 struct record *rec;
396b626b
AB		 111 unsigned long long samples;
112 unsigned long waking;
ae9c7242 113 u64 bytes_written;
610fbc01
AB		 114 u64 bytes_transferred;
115 u64 bytes_compressed;
415ccb58
AB		 116};
117
396b626b
AB		 118static __thread struct record_thread *thread;
119
1e5de7d9
AB		 120enum thread_msg {
121 THREAD_MSG__UNDEFINED = 0,
122 THREAD_MSG__READY,
123 THREAD_MSG__MAX,
124};
125
126static const char *thread_msg_tags[THREAD_MSG__MAX] = {
127 "UNDEFINED", "READY"
128};
129
06380a84
AB		 130enum thread_spec {
131 THREAD_SPEC__UNDEFINED = 0,
132 THREAD_SPEC__CPU,
133};
134
8c6f45a7 135struct record {
45694aa7 136 struct perf_tool tool;
b4006796 137 struct record_opts opts;
d20deb64 138 u64 bytes_written;
8ceb41d7 139 struct perf_data data;
ef149c25 140 struct auxtrace_record *itr;
63503dba 141 struct evlist *evlist;
d20deb64 142 struct perf_session *session;
bc477d79 143 struct evlist *sb_evlist;
899e5ffb 144 pthread_t thread_id;
d20deb64 145 int realtime_prio;
899e5ffb 146 bool switch_output_event_set;
d20deb64 147 bool no_buildid;
d2db9a98 148 bool no_buildid_set;
d20deb64 149 bool no_buildid_cache;
d2db9a98 150 bool no_buildid_cache_set;
6156681b 151 bool buildid_all;
e29386c8 152 bool buildid_mmap;
ecfd7a9c 153 bool timestamp_filename;
68588baf 154 bool timestamp_boundary;
1b43b704 155 struct switch_output switch_output;
9f065194 156 unsigned long long samples;
6d575816 157 unsigned long output_max_size; /* = 0: unlimited */
9bce13ea 158 struct perf_debuginfod debuginfod;
7954f716
AB		 159 int nr_threads;
160 struct thread_mask *thread_masks;
415ccb58 161 struct record_thread *thread_data;
0f82ebc4 162};
a21ca2ca 163
6d575816
JS		 164static volatile int done;
165
dc0c6127
JO		 166static volatile int auxtrace_record__snapshot_started;
167static DEFINE_TRIGGER(auxtrace_snapshot_trigger);
168static DEFINE_TRIGGER(switch_output_trigger);
169
9d2ed645
AB		 170static const char *affinity_tags[PERF_AFFINITY_MAX] = {
171 "SYS", "NODE", "CPU"
172};
173
415ccb58
AB		 174#ifndef HAVE_GETTID
175static inline pid_t gettid(void)
176{
177 return (pid_t)syscall(__NR_gettid);
178}
179#endif
180
3217e9fe
AB		 181static int record__threads_enabled(struct record *rec)
182{
183 return rec->opts.threads_spec;
184}
185
dc0c6127
JO		 186static bool switch_output_signal(struct record *rec)
187{
188 return rec->switch_output.signal &&
189 trigger_is_ready(&switch_output_trigger);
190}
191
192static bool switch_output_size(struct record *rec)
193{
194 return rec->switch_output.size &&
195 trigger_is_ready(&switch_output_trigger) &&
196 (rec->bytes_written >= rec->switch_output.size);
197}
198
bfacbe3b
JO		 199static bool switch_output_time(struct record *rec)
200{
201 return rec->switch_output.time &&
202 trigger_is_ready(&switch_output_trigger);
203}
204
ae9c7242
AB		 205static u64 record__bytes_written(struct record *rec)
206{
207 int t;
208 u64 bytes_written = rec->bytes_written;
209 struct record_thread *thread_data = rec->thread_data;
210
211 for (t = 0; t < rec->nr_threads; t++)
212 bytes_written += thread_data[t].bytes_written;
213
214 return bytes_written;
215}
216
6d575816
JS		 217static bool record__output_max_size_exceeded(struct record *rec)
218{
219 return rec->output_max_size &&
ae9c7242 220 (record__bytes_written(rec) >= rec->output_max_size);
6d575816
JS		 221}
222
a5830532 223static int record__write(struct record *rec, struct mmap *map __maybe_unused,
ded2b8fe 224 void *bf, size_t size)
f5970550 225{
ded2b8fe
JO		 226 struct perf_data_file *file = &rec->session->data->file;
227
56f735ff
AB		 228 if (map && map->file)
229 file = map->file;
230
ded2b8fe 231 if (perf_data_file__write(file, bf, size) < 0) {
50a9b868
JO		 232 pr_err("failed to write perf data, error: %m\n");
233 return -1;
f5970550 234 }
8d3eca20 235
ae9c7242
AB		 236 if (map && map->file)
237 thread->bytes_written += size;
238 else
56f735ff 239 rec->bytes_written += size;
dc0c6127 240
6d575816
JS		 241 if (record__output_max_size_exceeded(rec) && !done) {
242 fprintf(stderr, "[ perf record: perf size limit reached (%" PRIu64 " KB),"
243 " stopping session ]\n",
ae9c7242 244 record__bytes_written(rec) >> 10);
6d575816
JS		 245 done = 1;
246 }
247
dc0c6127
JO		 248 if (switch_output_size(rec))
249 trigger_hit(&switch_output_trigger);
250
8d3eca20 251 return 0;
f5970550
PZ		 252}
253
ef781128
AB		 254static int record__aio_enabled(struct record *rec);
255static int record__comp_enabled(struct record *rec);
75f5f1fc
AB		 256static size_t zstd_compress(struct perf_session *session, struct mmap *map,
257 void *dst, size_t dst_size, void *src, size_t src_size);
5d7f4116 258
d3d1af6f
AB		 259#ifdef HAVE_AIO_SUPPORT
260static int record__aio_write(struct aiocb *cblock, int trace_fd,
261 void *buf, size_t size, off_t off)
262{
263 int rc;
264
265 cblock->aio_fildes = trace_fd;
266 cblock->aio_buf = buf;
267 cblock->aio_nbytes = size;
268 cblock->aio_offset = off;
269 cblock->aio_sigevent.sigev_notify = SIGEV_NONE;
270
271 do {
272 rc = aio_write(cblock);
273 if (rc == 0) {
274 break;
275 } else if (errno != EAGAIN) {
276 cblock->aio_fildes = -1;
277 pr_err("failed to queue perf data, error: %m\n");
278 break;
279 }
280 } while (1);
281
282 return rc;
283}
284
a5830532 285static int record__aio_complete(struct mmap *md, struct aiocb *cblock)
d3d1af6f
AB		 286{
287 void *rem_buf;
288 off_t rem_off;
289 size_t rem_size;
290 int rc, aio_errno;
291 ssize_t aio_ret, written;
292
293 aio_errno = aio_error(cblock);
294 if (aio_errno == EINPROGRESS)
295 return 0;
296
297 written = aio_ret = aio_return(cblock);
298 if (aio_ret < 0) {
299 if (aio_errno != EINTR)
300 pr_err("failed to write perf data, error: %m\n");
301 written = 0;
302 }
303
304 rem_size = cblock->aio_nbytes - written;
305
306 if (rem_size == 0) {
307 cblock->aio_fildes = -1;
308 /*
ef781128
AB		 309 * md->refcount is incremented in record__aio_pushfn() for
310 * every aio write request started in record__aio_push() so
311 * decrement it because the request is now complete.
d3d1af6f 312 */
80e53d11 313 perf_mmap__put(&md->core);
d3d1af6f
AB		 314 rc = 1;
315 } else {
316 /*
317 * aio write request may require restart with the
318 * reminder if the kernel didn't write whole
319 * chunk at once.
320 */
321 rem_off = cblock->aio_offset + written;
322 rem_buf = (void *)(cblock->aio_buf + written);
323 record__aio_write(cblock, cblock->aio_fildes,
324 rem_buf, rem_size, rem_off);
325 rc = 0;
326 }
327
328 return rc;
329}
330
a5830532 331static int record__aio_sync(struct mmap *md, bool sync_all)
d3d1af6f 332{
93f20c0f
AB		 333 struct aiocb **aiocb = md->aio.aiocb;
334 struct aiocb *cblocks = md->aio.cblocks;
d3d1af6f 335 struct timespec timeout = { 0, 1000 * 1000 * 1 }; /* 1ms */
93f20c0f 336 int i, do_suspend;
d3d1af6f
AB		 337
338 do {
93f20c0f
AB		 339 do_suspend = 0;
340 for (i = 0; i < md->aio.nr_cblocks; ++i) {
341 if (cblocks[i].aio_fildes == -1 || record__aio_complete(md, &cblocks[i])) {
342 if (sync_all)
343 aiocb[i] = NULL;
344 else
345 return i;
346 } else {
347 /*
348 * Started aio write is not complete yet
349 * so it has to be waited before the
350 * next allocation.
351 */
352 aiocb[i] = &cblocks[i];
353 do_suspend = 1;
354 }
355 }
356 if (!do_suspend)
357 return -1;
d3d1af6f 358
93f20c0f 359 while (aio_suspend((const struct aiocb **)aiocb, md->aio.nr_cblocks, &timeout)) {
d3d1af6f
AB		 360 if (!(errno == EAGAIN || errno == EINTR))
361 pr_err("failed to sync perf data, error: %m\n");
362 }
363 } while (1);
364}
365
ef781128
AB		 366struct record_aio {
367 struct record *rec;
368 void *data;
369 size_t size;
370};
371
a5830532 372static int record__aio_pushfn(struct mmap *map, void *to, void *buf, size_t size)
d3d1af6f 373{
ef781128 374 struct record_aio *aio = to;
d3d1af6f 375
ef781128 376 /*
547740f7 377 * map->core.base data pointed by buf is copied into free map->aio.data[] buffer
ef781128
AB		 378 * to release space in the kernel buffer as fast as possible, calling
379 * perf_mmap__consume() from perf_mmap__push() function.
380 *
381 * That lets the kernel to proceed with storing more profiling data into
382 * the kernel buffer earlier than other per-cpu kernel buffers are handled.
383 *
384 * Coping can be done in two steps in case the chunk of profiling data
385 * crosses the upper bound of the kernel buffer. In this case we first move
386 * part of data from map->start till the upper bound and then the reminder
387 * from the beginning of the kernel buffer till the end of the data chunk.
388 */
389
390 if (record__comp_enabled(aio->rec)) {
75f5f1fc 391 size = zstd_compress(aio->rec->session, NULL, aio->data + aio->size,
bf59b305 392 mmap__mmap_len(map) - aio->size,
ef781128
AB		 393 buf, size);
394 } else {
395 memcpy(aio->data + aio->size, buf, size);
396 }
397
398 if (!aio->size) {
399 /*
400 * Increment map->refcount to guard map->aio.data[] buffer
401 * from premature deallocation because map object can be
402 * released earlier than aio write request started on
403 * map->aio.data[] buffer is complete.
404 *
405 * perf_mmap__put() is done at record__aio_complete()
406 * after started aio request completion or at record__aio_push()
407 * if the request failed to start.
408 */
e75710f0 409 perf_mmap__get(&map->core);
ef781128
AB		 410 }
411
412 aio->size += size;
413
414 return size;
415}
416
a5830532 417static int record__aio_push(struct record *rec, struct mmap *map, off_t *off)
ef781128
AB		 418{
419 int ret, idx;
420 int trace_fd = rec->session->data->file.fd;
421 struct record_aio aio = { .rec = rec, .size = 0 };
d3d1af6f 422
ef781128
AB		 423 /*
424 * Call record__aio_sync() to wait till map->aio.data[] buffer
425 * becomes available after previous aio write operation.
426 */
427
428 idx = record__aio_sync(map, false);
429 aio.data = map->aio.data[idx];
430 ret = perf_mmap__push(map, &aio, record__aio_pushfn);
431 if (ret != 0) /* ret > 0 - no data, ret < 0 - error */
432 return ret;
433
434 rec->samples++;
435 ret = record__aio_write(&(map->aio.cblocks[idx]), trace_fd, aio.data, aio.size, *off);
d3d1af6f 436 if (!ret) {
ef781128
AB		 437 *off += aio.size;
438 rec->bytes_written += aio.size;
d3d1af6f
AB		 439 if (switch_output_size(rec))
440 trigger_hit(&switch_output_trigger);
ef781128
AB		 441 } else {
442 /*
443 * Decrement map->refcount incremented in record__aio_pushfn()
444 * back if record__aio_write() operation failed to start, otherwise
445 * map->refcount is decremented in record__aio_complete() after
446 * aio write operation finishes successfully.
447 */
80e53d11 448 perf_mmap__put(&map->core);
d3d1af6f
AB		 449 }
450
451 return ret;
452}
453
454static off_t record__aio_get_pos(int trace_fd)
455{
456 return lseek(trace_fd, 0, SEEK_CUR);
457}
458
459static void record__aio_set_pos(int trace_fd, off_t pos)
460{
461 lseek(trace_fd, pos, SEEK_SET);
462}
463
464static void record__aio_mmap_read_sync(struct record *rec)
465{
466 int i;
63503dba 467 struct evlist *evlist = rec->evlist;
a5830532 468 struct mmap *maps = evlist->mmap;
d3d1af6f 469
ef781128 470 if (!record__aio_enabled(rec))
d3d1af6f
AB		 471 return;
472
c976ee11 473 for (i = 0; i < evlist->core.nr_mmaps; i++) {
a5830532 474 struct mmap *map = &maps[i];
d3d1af6f 475
547740f7 476 if (map->core.base)
93f20c0f 477 record__aio_sync(map, true);
d3d1af6f
AB		 478 }
479}
480
481static int nr_cblocks_default = 1;
93f20c0f 482static int nr_cblocks_max = 4;
d3d1af6f
AB		 483
484static int record__aio_parse(const struct option *opt,
93f20c0f 485 const char *str,
d3d1af6f
AB		 486 int unset)
487{
488 struct record_opts *opts = (struct record_opts *)opt->value;
489
93f20c0f 490 if (unset) {
d3d1af6f 491 opts->nr_cblocks = 0;
93f20c0f
AB		 492 } else {
493 if (str)
494 opts->nr_cblocks = strtol(str, NULL, 0);
495 if (!opts->nr_cblocks)
496 opts->nr_cblocks = nr_cblocks_default;
497 }
d3d1af6f
AB		 498
499 return 0;
500}
501#else /* HAVE_AIO_SUPPORT */
93f20c0f
AB		 502static int nr_cblocks_max = 0;
503
a5830532 504static int record__aio_push(struct record *rec __maybe_unused, struct mmap *map __maybe_unused,
ef781128 505 off_t *off __maybe_unused)
d3d1af6f
AB		 506{
507 return -1;
508}
509
510static off_t record__aio_get_pos(int trace_fd __maybe_unused)
511{
512 return -1;
513}
514
515static void record__aio_set_pos(int trace_fd __maybe_unused, off_t pos __maybe_unused)
516{
517}
518
519static void record__aio_mmap_read_sync(struct record *rec __maybe_unused)
520{
521}
522#endif
523
524static int record__aio_enabled(struct record *rec)
525{
526 return rec->opts.nr_cblocks > 0;
527}
528
470530bb
AB		 529#define MMAP_FLUSH_DEFAULT 1
530static int record__mmap_flush_parse(const struct option *opt,
531 const char *str,
532 int unset)
533{
534 int flush_max;
535 struct record_opts *opts = (struct record_opts *)opt->value;
536 static struct parse_tag tags[] = {
537 { .tag = 'B', .mult = 1 },
538 { .tag = 'K', .mult = 1 << 10 },
539 { .tag = 'M', .mult = 1 << 20 },
540 { .tag = 'G', .mult = 1 << 30 },
541 { .tag = 0 },
542 };
543
544 if (unset)
545 return 0;
546
547 if (str) {
548 opts->mmap_flush = parse_tag_value(str, tags);
549 if (opts->mmap_flush == (int)-1)
550 opts->mmap_flush = strtol(str, NULL, 0);
551 }
552
553 if (!opts->mmap_flush)
554 opts->mmap_flush = MMAP_FLUSH_DEFAULT;
555
9521b5f2 556 flush_max = evlist__mmap_size(opts->mmap_pages);
470530bb
AB		 557 flush_max /= 4;
558 if (opts->mmap_flush > flush_max)
559 opts->mmap_flush = flush_max;
560
561 return 0;
562}
563
504c1ad1
AB		 564#ifdef HAVE_ZSTD_SUPPORT
565static unsigned int comp_level_default = 1;
566
567static int record__parse_comp_level(const struct option *opt, const char *str, int unset)
568{
569 struct record_opts *opts = opt->value;
570
571 if (unset) {
572 opts->comp_level = 0;
573 } else {
574 if (str)
575 opts->comp_level = strtol(str, NULL, 0);
576 if (!opts->comp_level)
577 opts->comp_level = comp_level_default;
578 }
579
580 return 0;
581}
582#endif
51255a8a
AB		 583static unsigned int comp_level_max = 22;
584
42e1fd80
AB		 585static int record__comp_enabled(struct record *rec)
586{
587 return rec->opts.comp_level > 0;
588}
589
45694aa7 590static int process_synthesized_event(struct perf_tool *tool,
d20deb64 591 union perf_event *event,
1d037ca1
IT		 592 struct perf_sample *sample __maybe_unused,
593 struct machine *machine __maybe_unused)
234fbbf5 594{
8c6f45a7 595 struct record *rec = container_of(tool, struct record, tool);
ded2b8fe 596 return record__write(rec, NULL, event, event->header.size);
234fbbf5
ACM		 597}
598
d99c22ea
SE		 599static int process_locked_synthesized_event(struct perf_tool *tool,
600 union perf_event *event,
601 struct perf_sample *sample __maybe_unused,
602 struct machine *machine __maybe_unused)
603{
604 static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
605 int ret;
606
607 pthread_mutex_lock(&synth_lock);
608 ret = process_synthesized_event(tool, event, sample, machine);
609 pthread_mutex_unlock(&synth_lock);
610 return ret;
611}
612
a5830532 613static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
d37f1586
ACM		 614{
615 struct record *rec = to;
616
5d7f4116 617 if (record__comp_enabled(rec)) {
75f5f1fc 618 size = zstd_compress(rec->session, map, map->data, mmap__mmap_len(map), bf, size);
5d7f4116
AB		 619 bf = map->data;
620 }
621
396b626b 622 thread->samples++;
ded2b8fe 623 return record__write(rec, map, bf, size);
d37f1586
ACM		 624}
625
2dd6d8a1
AH		 626static volatile int signr = -1;
627static volatile int child_finished;
da231338
AM		 628#ifdef HAVE_EVENTFD_SUPPORT
629static int done_fd = -1;
630#endif
c0bdc1c4 631
2dd6d8a1
AH		 632static void sig_handler(int sig)
633{
634 if (sig == SIGCHLD)
635 child_finished = 1;
636 else
637 signr = sig;
638
639 done = 1;
da231338
AM		 640#ifdef HAVE_EVENTFD_SUPPORT
641{
642 u64 tmp = 1;
643 /*
644 * It is possible for this signal handler to run after done is checked
645 * in the main loop, but before the perf counter fds are polled. If this
646 * happens, the poll() will continue to wait even though done is set,
647 * and will only break out if either another signal is received, or the
648 * counters are ready for read. To ensure the poll() doesn't sleep when
649 * done is set, use an eventfd (done_fd) to wake up the poll().
650 */
651 if (write(done_fd, &tmp, sizeof(tmp)) < 0)
652 pr_err("failed to signal wakeup fd, error: %m\n");
653}
654#endif // HAVE_EVENTFD_SUPPORT
2dd6d8a1
AH		 655}
656
a074865e
WN		 657static void sigsegv_handler(int sig)
658{
659 perf_hooks__recover();
660 sighandler_dump_stack(sig);
661}
662
2dd6d8a1
AH		 663static void record__sig_exit(void)
664{
665 if (signr == -1)
666 return;
667
668 signal(signr, SIG_DFL);
669 raise(signr);
670}
671
e31f0d01
AH		 672#ifdef HAVE_AUXTRACE_SUPPORT
673
ef149c25 674static int record__process_auxtrace(struct perf_tool *tool,
a5830532 675 struct mmap *map,
ef149c25
AH		 676 union perf_event *event, void *data1,
677 size_t len1, void *data2, size_t len2)
678{
679 struct record *rec = container_of(tool, struct record, tool);
8ceb41d7 680 struct perf_data *data = &rec->data;
ef149c25
AH		 681 size_t padding;
682 u8 pad[8] = {0};
683
46e201ef 684 if (!perf_data__is_pipe(data) && perf_data__is_single_file(data)) {
99fa2984 685 off_t file_offset;
8ceb41d7 686 int fd = perf_data__fd(data);
99fa2984
AH		 687 int err;
688
689 file_offset = lseek(fd, 0, SEEK_CUR);
690 if (file_offset == -1)
691 return -1;
692 err = auxtrace_index__auxtrace_event(&rec->session->auxtrace_index,
693 event, file_offset);
694 if (err)
695 return err;
696 }
697
ef149c25
AH		 698 /* event.auxtrace.size includes padding, see __auxtrace_mmap__read() */
699 padding = (len1 + len2) & 7;
700 if (padding)
701 padding = 8 - padding;
702
ded2b8fe
JO		 703 record__write(rec, map, event, event->header.size);
704 record__write(rec, map, data1, len1);
ef149c25 705 if (len2)
ded2b8fe
JO		 706 record__write(rec, map, data2, len2);
707 record__write(rec, map, &pad, padding);
ef149c25
AH		 708
709 return 0;
710}
711
712static int record__auxtrace_mmap_read(struct record *rec,
a5830532 713 struct mmap *map)
ef149c25
AH		 714{
715 int ret;
716
e035f4ca 717 ret = auxtrace_mmap__read(map, rec->itr, &rec->tool,
ef149c25
AH		 718 record__process_auxtrace);
719 if (ret < 0)
720 return ret;
721
722 if (ret)
723 rec->samples++;
724
725 return 0;
726}
727
2dd6d8a1 728static int record__auxtrace_mmap_read_snapshot(struct record *rec,
a5830532 729 struct mmap *map)
2dd6d8a1
AH		 730{
731 int ret;
732
e035f4ca 733 ret = auxtrace_mmap__read_snapshot(map, rec->itr, &rec->tool,
2dd6d8a1
AH		 734 record__process_auxtrace,
735 rec->opts.auxtrace_snapshot_size);
736 if (ret < 0)
737 return ret;
738
739 if (ret)
740 rec->samples++;
741
742 return 0;
743}
744
745static int record__auxtrace_read_snapshot_all(struct record *rec)
746{
747 int i;
748 int rc = 0;
749
c976ee11 750 for (i = 0; i < rec->evlist->core.nr_mmaps; i++) {
a5830532 751 struct mmap *map = &rec->evlist->mmap[i];
2dd6d8a1 752
e035f4ca 753 if (!map->auxtrace_mmap.base)
2dd6d8a1
AH		 754 continue;
755
e035f4ca 756 if (record__auxtrace_mmap_read_snapshot(rec, map) != 0) {
2dd6d8a1
AH		 757 rc = -1;
758 goto out;
759 }
760 }
761out:
762 return rc;
763}
764
ce7b0e42 765static void record__read_auxtrace_snapshot(struct record *rec, bool on_exit)
2dd6d8a1
AH		 766{
767 pr_debug("Recording AUX area tracing snapshot\n");
768 if (record__auxtrace_read_snapshot_all(rec) < 0) {
5f9cf599 769 trigger_error(&auxtrace_snapshot_trigger);
2dd6d8a1 770 } else {
ce7b0e42 771 if (auxtrace_record__snapshot_finish(rec->itr, on_exit))
5f9cf599
WN		 772 trigger_error(&auxtrace_snapshot_trigger);
773 else
774 trigger_ready(&auxtrace_snapshot_trigger);
2dd6d8a1
AH		 775 }
776}
777
ce7b0e42
AS		 778static int record__auxtrace_snapshot_exit(struct record *rec)
779{
780 if (trigger_is_error(&auxtrace_snapshot_trigger))
781 return 0;
782
783 if (!auxtrace_record__snapshot_started &&
784 auxtrace_record__snapshot_start(rec->itr))
785 return -1;
786
787 record__read_auxtrace_snapshot(rec, true);
788 if (trigger_is_error(&auxtrace_snapshot_trigger))
789 return -1;
790
791 return 0;
792}
793
4b5ea3bd
AH		 794static int record__auxtrace_init(struct record *rec)
795{
796 int err;
797
798 if (!rec->itr) {
799 rec->itr = auxtrace_record__init(rec->evlist, &err);
800 if (err)
801 return err;
802 }
803
804 err = auxtrace_parse_snapshot_options(rec->itr, &rec->opts,
805 rec->opts.auxtrace_snapshot_opts);
806 if (err)
807 return err;
808
c0a6de06
AH		 809 err = auxtrace_parse_sample_options(rec->itr, rec->evlist, &rec->opts,
810 rec->opts.auxtrace_sample_opts);
811 if (err)
812 return err;
813
d58b3f7e
AH		 814 auxtrace_regroup_aux_output(rec->evlist);
815
4b5ea3bd
AH		 816 return auxtrace_parse_filters(rec->evlist);
817}
818
e31f0d01
AH		 819#else
820
821static inline
822int record__auxtrace_mmap_read(struct record *rec __maybe_unused,
a5830532 823 struct mmap *map __maybe_unused)
e31f0d01
AH		 824{
825 return 0;
826}
827
2dd6d8a1 828static inline
ce7b0e42
AS		 829void record__read_auxtrace_snapshot(struct record *rec __maybe_unused,
830 bool on_exit __maybe_unused)
de9ac07b 831{
f7b7c26e
PZ		 832}
833
2dd6d8a1
AH		 834static inline
835int auxtrace_record__snapshot_start(struct auxtrace_record *itr __maybe_unused)
f7b7c26e 836{
2dd6d8a1 837 return 0;
de9ac07b
PZ		 838}
839
ce7b0e42
AS		 840static inline
841int record__auxtrace_snapshot_exit(struct record *rec __maybe_unused)
842{
843 return 0;
844}
845
4b5ea3bd
AH		 846static int record__auxtrace_init(struct record *rec __maybe_unused)
847{
848 return 0;
849}
850
2dd6d8a1
AH		 851#endif
852
246eba8e
AH		 853static int record__config_text_poke(struct evlist *evlist)
854{
855 struct evsel *evsel;
856 int err;
857
858 /* Nothing to do if text poke is already configured */
859 evlist__for_each_entry(evlist, evsel) {
860 if (evsel->core.attr.text_poke)
861 return 0;
862 }
863
864 err = parse_events(evlist, "dummy:u", NULL);
865 if (err)
866 return err;
867
868 evsel = evlist__last(evlist);
869
870 evsel->core.attr.freq = 0;
871 evsel->core.attr.sample_period = 1;
872 evsel->core.attr.text_poke = 1;
873 evsel->core.attr.ksymbol = 1;
874
875 evsel->core.system_wide = true;
876 evsel->no_aux_samples = true;
877 evsel->immediate = true;
878
879 /* Text poke must be collected on all CPUs */
880 perf_cpu_map__put(evsel->core.own_cpus);
881 evsel->core.own_cpus = perf_cpu_map__new(NULL);
882 perf_cpu_map__put(evsel->core.cpus);
883 evsel->core.cpus = perf_cpu_map__get(evsel->core.own_cpus);
884
885 evsel__set_sample_bit(evsel, TIME);
886
887 return 0;
888}
889
eeb399b5
AH		 890static bool record__kcore_readable(struct machine *machine)
891{
892 char kcore[PATH_MAX];
893 int fd;
894
895 scnprintf(kcore, sizeof(kcore), "%s/proc/kcore", machine->root_dir);
896
897 fd = open(kcore, O_RDONLY);
898 if (fd < 0)
899 return false;
900
901 close(fd);
902
903 return true;
904}
905
906static int record__kcore_copy(struct machine *machine, struct perf_data *data)
907{
908 char from_dir[PATH_MAX];
909 char kcore_dir[PATH_MAX];
910 int ret;
911
912 snprintf(from_dir, sizeof(from_dir), "%s/proc", machine->root_dir);
913
914 ret = perf_data__make_kcore_dir(data, kcore_dir, sizeof(kcore_dir));
915 if (ret)
916 return ret;
917
918 return kcore_copy(from_dir, kcore_dir);
919}
920
415ccb58
AB		 921static void record__thread_data_init_pipes(struct record_thread *thread_data)
922{
923 thread_data->pipes.msg[0] = -1;
924 thread_data->pipes.msg[1] = -1;
925 thread_data->pipes.ack[0] = -1;
926 thread_data->pipes.ack[1] = -1;
927}
928
929static int record__thread_data_open_pipes(struct record_thread *thread_data)
930{
931 if (pipe(thread_data->pipes.msg))
932 return -EINVAL;
933
934 if (pipe(thread_data->pipes.ack)) {
935 close(thread_data->pipes.msg[0]);
936 thread_data->pipes.msg[0] = -1;
937 close(thread_data->pipes.msg[1]);
938 thread_data->pipes.msg[1] = -1;
939 return -EINVAL;
940 }
941
942 pr_debug2("thread_data[%p]: msg=[%d,%d], ack=[%d,%d]\n", thread_data,
943 thread_data->pipes.msg[0], thread_data->pipes.msg[1],
944 thread_data->pipes.ack[0], thread_data->pipes.ack[1]);
945
946 return 0;
947}
948
949static void record__thread_data_close_pipes(struct record_thread *thread_data)
950{
951 if (thread_data->pipes.msg[0] != -1) {
952 close(thread_data->pipes.msg[0]);
953 thread_data->pipes.msg[0] = -1;
954 }
955 if (thread_data->pipes.msg[1] != -1) {
956 close(thread_data->pipes.msg[1]);
957 thread_data->pipes.msg[1] = -1;
958 }
959 if (thread_data->pipes.ack[0] != -1) {
960 close(thread_data->pipes.ack[0]);
961 thread_data->pipes.ack[0] = -1;
962 }
963 if (thread_data->pipes.ack[1] != -1) {
964 close(thread_data->pipes.ack[1]);
965 thread_data->pipes.ack[1] = -1;
966 }
967}
968
969static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist)
970{
971 int m, tm, nr_mmaps = evlist->core.nr_mmaps;
972 struct mmap *mmap = evlist->mmap;
973 struct mmap *overwrite_mmap = evlist->overwrite_mmap;
974 struct perf_cpu_map *cpus = evlist->core.cpus;
975
976 thread_data->nr_mmaps = bitmap_weight(thread_data->mask->maps.bits,
977 thread_data->mask->maps.nbits);
978 if (mmap) {
979 thread_data->maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
980 if (!thread_data->maps)
981 return -ENOMEM;
982 }
983 if (overwrite_mmap) {
984 thread_data->overwrite_maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
985 if (!thread_data->overwrite_maps) {
986 zfree(&thread_data->maps);
987 return -ENOMEM;
988 }
989 }
990 pr_debug2("thread_data[%p]: nr_mmaps=%d, maps=%p, ow_maps=%p\n", thread_data,
991 thread_data->nr_mmaps, thread_data->maps, thread_data->overwrite_maps);
992
993 for (m = 0, tm = 0; m < nr_mmaps && tm < thread_data->nr_mmaps; m++) {
994 if (test_bit(cpus->map[m].cpu, thread_data->mask->maps.bits)) {
995 if (thread_data->maps) {
996 thread_data->maps[tm] = &mmap[m];
997 pr_debug2("thread_data[%p]: cpu%d: maps[%d] -> mmap[%d]\n",
998 thread_data, cpus->map[m].cpu, tm, m);
999 }
1000 if (thread_data->overwrite_maps) {
1001 thread_data->overwrite_maps[tm] = &overwrite_mmap[m];
1002 pr_debug2("thread_data[%p]: cpu%d: ow_maps[%d] -> ow_mmap[%d]\n",
1003 thread_data, cpus->map[m].cpu, tm, m);
1004 }
1005 tm++;
1006 }
1007 }
1008
1009 return 0;
1010}
1011
1012static int record__thread_data_init_pollfd(struct record_thread *thread_data, struct evlist *evlist)
1013{
1014 int f, tm, pos;
1015 struct mmap *map, *overwrite_map;
1016
1017 fdarray__init(&thread_data->pollfd, 64);
1018
1019 for (tm = 0; tm < thread_data->nr_mmaps; tm++) {
1020 map = thread_data->maps ? thread_data->maps[tm] : NULL;
1021 overwrite_map = thread_data->overwrite_maps ?
1022 thread_data->overwrite_maps[tm] : NULL;
1023
1024 for (f = 0; f < evlist->core.pollfd.nr; f++) {
1025 void *ptr = evlist->core.pollfd.priv[f].ptr;
1026
1027 if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) {
1028 pos = fdarray__dup_entry_from(&thread_data->pollfd, f,
1029 &evlist->core.pollfd);
1030 if (pos < 0)
1031 return pos;
1032 pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n",
1033 thread_data, pos, evlist->core.pollfd.entries[f].fd);
1034 }
1035 }
1036 }
1037
1038 return 0;
1039}
1040
1041static void record__free_thread_data(struct record *rec)
1042{
1043 int t;
1044 struct record_thread *thread_data = rec->thread_data;
1045
1046 if (thread_data == NULL)
1047 return;
1048
1049 for (t = 0; t < rec->nr_threads; t++) {
1050 record__thread_data_close_pipes(&thread_data[t]);
1051 zfree(&thread_data[t].maps);
1052 zfree(&thread_data[t].overwrite_maps);
1053 fdarray__exit(&thread_data[t].pollfd);
1054 }
1055
1056 zfree(&rec->thread_data);
1057}
1058
1059static int record__alloc_thread_data(struct record *rec, struct evlist *evlist)
1060{
1061 int t, ret;
1062 struct record_thread *thread_data;
1063
1064 rec->thread_data = zalloc(rec->nr_threads * sizeof(*(rec->thread_data)));
1065 if (!rec->thread_data) {
1066 pr_err("Failed to allocate thread data\n");
1067 return -ENOMEM;
1068 }
1069 thread_data = rec->thread_data;
1070
1071 for (t = 0; t < rec->nr_threads; t++)
1072 record__thread_data_init_pipes(&thread_data[t]);
1073
1074 for (t = 0; t < rec->nr_threads; t++) {
1075 thread_data[t].rec = rec;
1076 thread_data[t].mask = &rec->thread_masks[t];
1077 ret = record__thread_data_init_maps(&thread_data[t], evlist);
1078 if (ret) {
1079 pr_err("Failed to initialize thread[%d] maps\n", t);
1080 goto out_free;
1081 }
1082 ret = record__thread_data_init_pollfd(&thread_data[t], evlist);
1083 if (ret) {
1084 pr_err("Failed to initialize thread[%d] pollfd\n", t);
1085 goto out_free;
1086 }
1087 if (t) {
1088 thread_data[t].tid = -1;
1089 ret = record__thread_data_open_pipes(&thread_data[t]);
1090 if (ret) {
1091 pr_err("Failed to open thread[%d] communication pipes\n", t);
1092 goto out_free;
1093 }
1094 ret = fdarray__add(&thread_data[t].pollfd, thread_data[t].pipes.msg[0],
1095 POLLIN | POLLERR | POLLHUP, fdarray_flag__nonfilterable);
1096 if (ret < 0) {
1097 pr_err("Failed to add descriptor to thread[%d] pollfd\n", t);
1098 goto out_free;
1099 }
1100 thread_data[t].ctlfd_pos = ret;
1101 pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
1102 thread_data, thread_data[t].ctlfd_pos,
1103 thread_data[t].pipes.msg[0]);
1104 } else {
1105 thread_data[t].tid = gettid();
1106 if (evlist->ctl_fd.pos == -1)
1107 continue;
1108 ret = fdarray__dup_entry_from(&thread_data[t].pollfd, evlist->ctl_fd.pos,
1109 &evlist->core.pollfd);
1110 if (ret < 0) {
1111 pr_err("Failed to duplicate descriptor in main thread pollfd\n");
1112 goto out_free;
1113 }
1114 thread_data[t].ctlfd_pos = ret;
1115 pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
1116 thread_data, thread_data[t].ctlfd_pos,
1117 evlist->core.pollfd.entries[evlist->ctl_fd.pos].fd);
1118 }
1119 }
1120
1121 return 0;
1122
1123out_free:
1124 record__free_thread_data(rec);
1125
1126 return ret;
1127}
1128
cda57a8c 1129static int record__mmap_evlist(struct record *rec,
63503dba 1130 struct evlist *evlist)
cda57a8c 1131{
56f735ff 1132 int i, ret;
cda57a8c 1133 struct record_opts *opts = &rec->opts;
c0a6de06
AH		 1134 bool auxtrace_overwrite = opts->auxtrace_snapshot_mode ||
1135 opts->auxtrace_sample_mode;
cda57a8c
WN		 1136 char msg[512];
1137
f13de660
AB		 1138 if (opts->affinity != PERF_AFFINITY_SYS)
1139 cpu__setup_cpunode_map();
1140
9521b5f2 1141 if (evlist__mmap_ex(evlist, opts->mmap_pages,
cda57a8c 1142 opts->auxtrace_mmap_pages,
c0a6de06 1143 auxtrace_overwrite,
470530bb 1144 opts->nr_cblocks, opts->affinity,
51255a8a 1145 opts->mmap_flush, opts->comp_level) < 0) {
cda57a8c
WN		 1146 if (errno == EPERM) {
1147 pr_err("Permission error mapping pages.\n"
1148 "Consider increasing "
1149 "/proc/sys/kernel/perf_event_mlock_kb,\n"
1150 "or try again with a smaller value of -m/--mmap_pages.\n"
1151 "(current value: %u,%u)\n",
1152 opts->mmap_pages, opts->auxtrace_mmap_pages);
1153 return -errno;
1154 } else {
1155 pr_err("failed to mmap with %d (%s)\n", errno,
c8b5f2c9 1156 str_error_r(errno, msg, sizeof(msg)));
cda57a8c
WN		 1157 if (errno)
1158 return -errno;
1159 else
1160 return -EINVAL;
1161 }
1162 }
415ccb58
AB		 1163
1164 if (evlist__initialize_ctlfd(evlist, opts->ctl_fd, opts->ctl_fd_ack))
1165 return -1;
1166
1167 ret = record__alloc_thread_data(rec, evlist);
1168 if (ret)
1169 return ret;
1170
56f735ff
AB		 1171 if (record__threads_enabled(rec)) {
1172 ret = perf_data__create_dir(&rec->data, evlist->core.nr_mmaps);
1173 if (ret)
1174 return ret;
1175 for (i = 0; i < evlist->core.nr_mmaps; i++) {
1176 if (evlist->mmap)
1177 evlist->mmap[i].file = &rec->data.dir.files[i];
1178 if (evlist->overwrite_mmap)
1179 evlist->overwrite_mmap[i].file = &rec->data.dir.files[i];
1180 }
1181 }
1182
cda57a8c
WN		 1183 return 0;
1184}
1185
1186static int record__mmap(struct record *rec)
1187{
1188 return record__mmap_evlist(rec, rec->evlist);
1189}
1190
8c6f45a7 1191static int record__open(struct record *rec)
dd7927f4 1192{
d6195a6a 1193 char msg[BUFSIZ];
32dcd021 1194 struct evsel *pos;
63503dba 1195 struct evlist *evlist = rec->evlist;
d20deb64 1196 struct perf_session *session = rec->session;
b4006796 1197 struct record_opts *opts = &rec->opts;
8d3eca20 1198 int rc = 0;
dd7927f4 1199
d3dbf43c 1200 /*
b91e5492
KL		 1201 * For initial_delay, system wide or a hybrid system, we need to add a
1202 * dummy event so that we can track PERF_RECORD_MMAP to cover the delay
1203 * of waiting or event synthesis.
d3dbf43c 1204 */
b91e5492
KL		 1205 if (opts->initial_delay || target__has_cpu(&opts->target) ||
1206 perf_pmu__has_hybrid()) {
e80db255 1207 pos = evlist__get_tracking_event(evlist);
442ad225
AH		 1208 if (!evsel__is_dummy_event(pos)) {
1209 /* Set up dummy event. */
facbf0b9 1210 if (evlist__add_dummy(evlist))
442ad225
AH		 1211 return -ENOMEM;
1212 pos = evlist__last(evlist);
e80db255 1213 evlist__set_tracking_event(evlist, pos);
442ad225
AH		 1214 }
1215
0a892c1c
IR		 1216 /*
1217 * Enable the dummy event when the process is forked for
1218 * initial_delay, immediately for system wide.
1219 */
bb07d62e
NK		 1220 if (opts->initial_delay && !pos->immediate &&
1221 !target__has_cpu(&opts->target))
0a892c1c
IR		 1222 pos->core.attr.enable_on_exec = 1;
1223 else
1224 pos->immediate = 1;
d3dbf43c
ACM		 1225 }
1226
78e1bc25 1227 evlist__config(evlist, opts, &callchain_param);
cac21425 1228
e5cadb93 1229 evlist__for_each_entry(evlist, pos) {
dd7927f4 1230try_again:
af663bd0 1231 if (evsel__open(pos, pos->core.cpus, pos->core.threads) < 0) {
ae430892 1232 if (evsel__fallback(pos, errno, msg, sizeof(msg))) {
bb963e16 1233 if (verbose > 0)
c0a54341 1234 ui__warning("%s\n", msg);
d6d901c2
ZY		 1235 goto try_again;
1236 }
cf99ad14 1237 if ((errno == EINVAL || errno == EBADF) &&
fba7c866 1238 pos->core.leader != &pos->core &&
cf99ad14 1239 pos->weak_group) {
64b4778b 1240 pos = evlist__reset_weak_group(evlist, pos, true);
cf99ad14
AK		 1241 goto try_again;
1242 }
56e52e85 1243 rc = -errno;
2bb72dbb 1244 evsel__open_strerror(pos, &opts->target, errno, msg, sizeof(msg));
56e52e85 1245 ui__error("%s\n", msg);
8d3eca20 1246 goto out;
c171b552 1247 }
bfd8f72c
AK		 1248
1249 pos->supported = true;
c171b552 1250 }
a43d3f08 1251
78e1bc25 1252 if (symbol_conf.kptr_restrict && !evlist__exclude_kernel(evlist)) {
c8b567c8
ACM		 1253 pr_warning(
1254"WARNING: Kernel address maps (/proc/{kallsyms,modules}) are restricted,\n"
1255"check /proc/sys/kernel/kptr_restrict and /proc/sys/kernel/perf_event_paranoid.\n\n"
1256"Samples in kernel functions may not be resolved if a suitable vmlinux\n"
1257"file is not found in the buildid cache or in the vmlinux path.\n\n"
1258"Samples in kernel modules won't be resolved at all.\n\n"
1259"If some relocation was applied (e.g. kexec) symbols may be misresolved\n"
1260"even with a suitable vmlinux or kallsyms file.\n\n");
1261 }
1262
24bf91a7 1263 if (evlist__apply_filters(evlist, &pos)) {
62d94b00 1264 pr_err("failed to set filter \"%s\" on event %s with %d (%s)\n",
8ab2e96d 1265 pos->filter, evsel__name(pos), errno,
c8b5f2c9 1266 str_error_r(errno, msg, sizeof(msg)));
8d3eca20 1267 rc = -1;
5d8bb1ec
MP		 1268 goto out;
1269 }
1270
cda57a8c
WN		 1271 rc = record__mmap(rec);
1272 if (rc)
8d3eca20 1273 goto out;
0a27d7f9 1274
563aecb2 1275 session->evlist = evlist;
7b56cce2 1276 perf_session__set_id_hdr_size(session);
8d3eca20
DA		 1277out:
1278 return rc;
16c8a109
PZ		 1279}
1280
66286ed3
AH		 1281static void set_timestamp_boundary(struct record *rec, u64 sample_time)
1282{
1283 if (rec->evlist->first_sample_time == 0)
1284 rec->evlist->first_sample_time = sample_time;
1285
1286 if (sample_time)
1287 rec->evlist->last_sample_time = sample_time;
1288}
1289
e3d59112
NK		 1290static int process_sample_event(struct perf_tool *tool,
1291 union perf_event *event,
1292 struct perf_sample *sample,
32dcd021 1293 struct evsel *evsel,
e3d59112
NK		 1294 struct machine *machine)
1295{
1296 struct record *rec = container_of(tool, struct record, tool);
1297
66286ed3 1298 set_timestamp_boundary(rec, sample->time);
e3d59112 1299
68588baf
JY		 1300 if (rec->buildid_all)
1301 return 0;
1302
1303 rec->samples++;
e3d59112
NK		 1304 return build_id__mark_dso_hit(tool, event, sample, evsel, machine);
1305}
1306
8c6f45a7 1307static int process_buildids(struct record *rec)
6122e4e4 1308{
f5fc1412 1309 struct perf_session *session = rec->session;
6122e4e4 1310
45112e89 1311 if (perf_data__size(&rec->data) == 0)
9f591fd7
ACM		 1312 return 0;
1313
00dc8657
NK		 1314 /*
1315 * During this process, it'll load kernel map and replace the
1316 * dso->long_name to a real pathname it found. In this case
1317 * we prefer the vmlinux path like
1318 * /lib/modules/3.16.4/build/vmlinux
1319 *
1320 * rather than build-id path (in debug directory).
1321 * $HOME/.debug/.build-id/f0/6e17aa50adf4d00b88925e03775de107611551
1322 */
1323 symbol_conf.ignore_vmlinux_buildid = true;
1324
6156681b
NK		 1325 /*
1326 * If --buildid-all is given, it marks all DSO regardless of hits,
68588baf
JY		 1327 * so no need to process samples. But if timestamp_boundary is enabled,
1328 * it still needs to walk on all samples to get the timestamps of
1329 * first/last samples.
6156681b 1330 */
68588baf 1331 if (rec->buildid_all && !rec->timestamp_boundary)
6156681b
NK		 1332 rec->tool.sample = NULL;
1333
b7b61cbe 1334 return perf_session__process_events(session);
6122e4e4
ACM		 1335}
1336
8115d60c 1337static void perf_event__synthesize_guest_os(struct machine *machine, void *data)
a1645ce1
ZY		 1338{
1339 int err;
45694aa7 1340 struct perf_tool *tool = data;
a1645ce1
ZY		 1341 /*
1342 *As for guest kernel when processing subcommand record&report,
1343 *we arrange module mmap prior to guest kernel mmap and trigger
1344 *a preload dso because default guest module symbols are loaded
1345 *from guest kallsyms instead of /lib/modules/XXX/XXX. This
1346 *method is used to avoid symbol missing when the first addr is
1347 *in module instead of in guest kernel.
1348 */
45694aa7 1349 err = perf_event__synthesize_modules(tool, process_synthesized_event,
743eb868 1350 machine);
a1645ce1
ZY		 1351 if (err < 0)
1352 pr_err("Couldn't record guest kernel [%d]'s reference"
23346f21 1353 " relocation symbol.\n", machine->pid);
a1645ce1 1354
a1645ce1
ZY		 1355 /*
1356 * We use _stext for guest kernel because guest kernel's /proc/kallsyms
1357 * have no _text sometimes.
1358 */
45694aa7 1359 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
0ae617be 1360 machine);
a1645ce1
ZY		 1361 if (err < 0)
1362 pr_err("Couldn't record guest kernel [%d]'s reference"
23346f21 1363 " relocation symbol.\n", machine->pid);
a1645ce1
ZY		 1364}
1365
98402807
FW		 1366static struct perf_event_header finished_round_event = {
1367 .size = sizeof(struct perf_event_header),
1368 .type = PERF_RECORD_FINISHED_ROUND,
1369};
1370
a5830532 1371static void record__adjust_affinity(struct record *rec, struct mmap *map)
f13de660
AB		 1372{
1373 if (rec->opts.affinity != PERF_AFFINITY_SYS &&
396b626b
AB		 1374 !bitmap_equal(thread->mask->affinity.bits, map->affinity_mask.bits,
1375 thread->mask->affinity.nbits)) {
1376 bitmap_zero(thread->mask->affinity.bits, thread->mask->affinity.nbits);
1377 bitmap_or(thread->mask->affinity.bits, thread->mask->affinity.bits,
1378 map->affinity_mask.bits, thread->mask->affinity.nbits);
1379 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
1380 (cpu_set_t *)thread->mask->affinity.bits);
1381 if (verbose == 2) {
1382 pr_debug("threads[%d]: running on cpu%d: ", thread->tid, sched_getcpu());
1383 mmap_cpu_mask__scnprintf(&thread->mask->affinity, "affinity");
1384 }
f13de660
AB		 1385 }
1386}
1387
5d7f4116
AB		 1388static size_t process_comp_header(void *record, size_t increment)
1389{
72932371 1390 struct perf_record_compressed *event = record;
5d7f4116
AB		 1391 size_t size = sizeof(*event);
1392
1393 if (increment) {
1394 event->header.size += increment;
1395 return increment;
1396 }
1397
1398 event->header.type = PERF_RECORD_COMPRESSED;
1399 event->header.size = size;
1400
1401 return size;
1402}
1403
75f5f1fc
AB		 1404static size_t zstd_compress(struct perf_session *session, struct mmap *map,
1405 void *dst, size_t dst_size, void *src, size_t src_size)
5d7f4116
AB		 1406{
1407 size_t compressed;
72932371 1408 size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed) - 1;
75f5f1fc 1409 struct zstd_data *zstd_data = &session->zstd_data;
5d7f4116 1410
75f5f1fc
AB		 1411 if (map && map->file)
1412 zstd_data = &map->zstd_data;
1413
1414 compressed = zstd_compress_stream_to_records(zstd_data, dst, dst_size, src, src_size,
5d7f4116
AB		 1415 max_record_size, process_comp_header);
1416
610fbc01
AB		 1417 if (map && map->file) {
1418 thread->bytes_transferred += src_size;
1419 thread->bytes_compressed += compressed;
1420 } else {
1421 session->bytes_transferred += src_size;
1422 session->bytes_compressed += compressed;
1423 }
5d7f4116
AB		 1424
1425 return compressed;
1426}
1427
63503dba 1428static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist,
470530bb 1429 bool overwrite, bool synch)
98402807 1430{
dcabb507 1431 u64 bytes_written = rec->bytes_written;
0e2e63dd 1432 int i;
8d3eca20 1433 int rc = 0;
396b626b
AB		 1434 int nr_mmaps;
1435 struct mmap **maps;
d3d1af6f 1436 int trace_fd = rec->data.file.fd;
ef781128 1437 off_t off = 0;
98402807 1438
cb21686b
WN		 1439 if (!evlist)
1440 return 0;
ef149c25 1441
396b626b
AB		 1442 nr_mmaps = thread->nr_mmaps;
1443 maps = overwrite ? thread->overwrite_maps : thread->maps;
1444
a4ea0ec4
WN		 1445 if (!maps)
1446 return 0;
1447
0b72d69a 1448 if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING)
54cc54de
WN		 1449 return 0;
1450
d3d1af6f
AB		 1451 if (record__aio_enabled(rec))
1452 off = record__aio_get_pos(trace_fd);
1453
396b626b 1454 for (i = 0; i < nr_mmaps; i++) {
470530bb 1455 u64 flush = 0;
396b626b 1456 struct mmap *map = maps[i];
cb21686b 1457
547740f7 1458 if (map->core.base) {
f13de660 1459 record__adjust_affinity(rec, map);
470530bb 1460 if (synch) {
65aa2e6b
JO		 1461 flush = map->core.flush;
1462 map->core.flush = 1;
470530bb 1463 }
d3d1af6f 1464 if (!record__aio_enabled(rec)) {
ef781128 1465 if (perf_mmap__push(map, rec, record__pushfn) < 0) {
470530bb 1466 if (synch)
65aa2e6b 1467 map->core.flush = flush;
d3d1af6f
AB		 1468 rc = -1;
1469 goto out;
1470 }
1471 } else {
ef781128 1472 if (record__aio_push(rec, map, &off) < 0) {
d3d1af6f 1473 record__aio_set_pos(trace_fd, off);
470530bb 1474 if (synch)
65aa2e6b 1475 map->core.flush = flush;
d3d1af6f
AB		 1476 rc = -1;
1477 goto out;
1478 }
8d3eca20 1479 }
470530bb 1480 if (synch)
65aa2e6b 1481 map->core.flush = flush;
8d3eca20 1482 }
ef149c25 1483
e035f4ca 1484 if (map->auxtrace_mmap.base && !rec->opts.auxtrace_snapshot_mode &&
c0a6de06 1485 !rec->opts.auxtrace_sample_mode &&
e035f4ca 1486 record__auxtrace_mmap_read(rec, map) != 0) {
ef149c25
AH		 1487 rc = -1;
1488 goto out;
1489 }
98402807
FW		 1490 }
1491
d3d1af6f
AB		 1492 if (record__aio_enabled(rec))
1493 record__aio_set_pos(trace_fd, off);
1494
dcabb507
JO		 1495 /*
1496 * Mark the round finished in case we wrote
1497 * at least one event.
56f735ff
AB		 1498 *
1499 * No need for round events in directory mode,
1500 * because per-cpu maps and files have data
1501 * sorted by kernel.
dcabb507 1502 */
56f735ff 1503 if (!record__threads_enabled(rec) && bytes_written != rec->bytes_written)
ded2b8fe 1504 rc = record__write(rec, NULL, &finished_round_event, sizeof(finished_round_event));
8d3eca20 1505
0b72d69a 1506 if (overwrite)
ade9d208 1507 evlist__toggle_bkw_mmap(evlist, BKW_MMAP_EMPTY);
8d3eca20
DA		 1508out:
1509 return rc;
98402807
FW		 1510}
1511
470530bb 1512static int record__mmap_read_all(struct record *rec, bool synch)
cb21686b
WN		 1513{
1514 int err;
1515
470530bb 1516 err = record__mmap_read_evlist(rec, rec->evlist, false, synch);
cb21686b
WN		 1517 if (err)
1518 return err;
1519
470530bb 1520 return record__mmap_read_evlist(rec, rec->evlist, true, synch);
cb21686b
WN		 1521}
1522
396b626b
AB		 1523static void record__thread_munmap_filtered(struct fdarray *fda, int fd,
1524 void *arg __maybe_unused)
1525{
1526 struct perf_mmap *map = fda->priv[fd].ptr;
1527
1528 if (map)
1529 perf_mmap__put(map);
1530}
1531
3217e9fe
AB		 1532static void *record__thread(void *arg)
1533{
1534 enum thread_msg msg = THREAD_MSG__READY;
1535 bool terminate = false;
1536 struct fdarray *pollfd;
1537 int err, ctlfd_pos;
1538
1539 thread = arg;
1540 thread->tid = gettid();
1541
1542 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1543 if (err == -1)
1544 pr_warning("threads[%d]: failed to notify on start: %s\n",
1545 thread->tid, strerror(errno));
1546
1547 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
1548
1549 pollfd = &thread->pollfd;
1550 ctlfd_pos = thread->ctlfd_pos;
1551
1552 for (;;) {
1553 unsigned long long hits = thread->samples;
1554
1555 if (record__mmap_read_all(thread->rec, false) < 0 || terminate)
1556 break;
1557
1558 if (hits == thread->samples) {
1559
1560 err = fdarray__poll(pollfd, -1);
1561 /*
1562 * Propagate error, only if there's any. Ignore positive
1563 * number of returned events and interrupt error.
1564 */
1565 if (err > 0 || (err < 0 && errno == EINTR))
1566 err = 0;
1567 thread->waking++;
1568
1569 if (fdarray__filter(pollfd, POLLERR | POLLHUP,
1570 record__thread_munmap_filtered, NULL) == 0)
1571 break;
1572 }
1573
1574 if (pollfd->entries[ctlfd_pos].revents & POLLHUP) {
1575 terminate = true;
1576 close(thread->pipes.msg[0]);
1577 thread->pipes.msg[0] = -1;
1578 pollfd->entries[ctlfd_pos].fd = -1;
1579 pollfd->entries[ctlfd_pos].events = 0;
1580 }
1581
1582 pollfd->entries[ctlfd_pos].revents = 0;
1583 }
1584 record__mmap_read_all(thread->rec, true);
1585
1586 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1587 if (err == -1)
1588 pr_warning("threads[%d]: failed to notify on termination: %s\n",
1589 thread->tid, strerror(errno));
1590
1591 return NULL;
1592}
1593
8c6f45a7 1594static void record__init_features(struct record *rec)
57706abc 1595{
57706abc
DA		 1596 struct perf_session *session = rec->session;
1597 int feat;
1598
1599 for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
1600 perf_header__set_feat(&session->header, feat);
1601
1602 if (rec->no_buildid)
1603 perf_header__clear_feat(&session->header, HEADER_BUILD_ID);
1604
ce9036a6 1605 if (!have_tracepoints(&rec->evlist->core.entries))
57706abc
DA		 1606 perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
1607
1608 if (!rec->opts.branch_stack)
1609 perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);
ef149c25
AH		 1610
1611 if (!rec->opts.full_auxtrace)
1612 perf_header__clear_feat(&session->header, HEADER_AUXTRACE);
ffa517ad 1613
cf790516
AB		 1614 if (!(rec->opts.use_clockid && rec->opts.clockid_res_ns))
1615 perf_header__clear_feat(&session->header, HEADER_CLOCKID);
1616
d1e325cf
JO		 1617 if (!rec->opts.use_clockid)
1618 perf_header__clear_feat(&session->header, HEADER_CLOCK_DATA);
1619
56f735ff
AB		 1620 if (!record__threads_enabled(rec))
1621 perf_header__clear_feat(&session->header, HEADER_DIR_FORMAT);
1622
42e1fd80
AB		 1623 if (!record__comp_enabled(rec))
1624 perf_header__clear_feat(&session->header, HEADER_COMPRESSED);
258031c0 1625
ffa517ad 1626 perf_header__clear_feat(&session->header, HEADER_STAT);
57706abc
DA		 1627}
1628
e1ab48ba
WN		 1629static void
1630record__finish_output(struct record *rec)
1631{
56f735ff 1632 int i;
8ceb41d7
JO		 1633 struct perf_data *data = &rec->data;
1634 int fd = perf_data__fd(data);
e1ab48ba 1635
8ceb41d7 1636 if (data->is_pipe)
e1ab48ba
WN		 1637 return;
1638
1639 rec->session->header.data_size += rec->bytes_written;
45112e89 1640 data->file.size = lseek(perf_data__fd(data), 0, SEEK_CUR);
56f735ff
AB		 1641 if (record__threads_enabled(rec)) {
1642 for (i = 0; i < data->dir.nr; i++)
1643 data->dir.files[i].size = lseek(data->dir.files[i].fd, 0, SEEK_CUR);
1644 }
e1ab48ba
WN		 1645
1646 if (!rec->no_buildid) {
1647 process_buildids(rec);
1648
1649 if (rec->buildid_all)
1650 dsos__hit_all(rec->session);
1651 }
1652 perf_session__write_header(rec->session, rec->evlist, fd, true);
1653
1654 return;
1655}
1656
4ea648ae 1657static int record__synthesize_workload(struct record *rec, bool tail)
be7b0c9e 1658{
9d6aae72 1659 int err;
9749b90e 1660 struct perf_thread_map *thread_map;
41b740b6 1661 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
be7b0c9e 1662
4ea648ae
WN		 1663 if (rec->opts.tail_synthesize != tail)
1664 return 0;
1665
9d6aae72
ACM		 1666 thread_map = thread_map__new_by_tid(rec->evlist->workload.pid);
1667 if (thread_map == NULL)
1668 return -1;
1669
1670 err = perf_event__synthesize_thread_map(&rec->tool, thread_map,
be7b0c9e
WN		 1671 process_synthesized_event,
1672 &rec->session->machines.host,
41b740b6 1673 needs_mmap,
3fcb10e4 1674 rec->opts.sample_address);
7836e52e 1675 perf_thread_map__put(thread_map);
9d6aae72 1676 return err;
be7b0c9e
WN		 1677}
1678
4ea648ae 1679static int record__synthesize(struct record *rec, bool tail);
3c1cb7e3 1680
ecfd7a9c
WN		 1681static int
1682record__switch_output(struct record *rec, bool at_exit)
1683{
8ceb41d7 1684 struct perf_data *data = &rec->data;
ecfd7a9c 1685 int fd, err;
03724b2e 1686 char *new_filename;
ecfd7a9c
WN		 1687
1688 /* Same Size: "2015122520103046"*/
1689 char timestamp[] = "InvalidTimestamp";
1690
d3d1af6f
AB		 1691 record__aio_mmap_read_sync(rec);
1692
4ea648ae
WN		 1693 record__synthesize(rec, true);
1694 if (target__none(&rec->opts.target))
1695 record__synthesize_workload(rec, true);
1696
ecfd7a9c
WN		 1697 rec->samples = 0;
1698 record__finish_output(rec);
1699 err = fetch_current_timestamp(timestamp, sizeof(timestamp));
1700 if (err) {
1701 pr_err("Failed to get current timestamp\n");
1702 return -EINVAL;
1703 }
1704
8ceb41d7 1705 fd = perf_data__switch(data, timestamp,
ecfd7a9c 1706 rec->session->header.data_offset,
03724b2e 1707 at_exit, &new_filename);
ecfd7a9c
WN		 1708 if (fd >= 0 && !at_exit) {
1709 rec->bytes_written = 0;
1710 rec->session->header.data_size = 0;
1711 }
1712
1713 if (!quiet)
1714 fprintf(stderr, "[ perf record: Dump %s.%s ]\n",
2d4f2799 1715 data->path, timestamp);
3c1cb7e3 1716
03724b2e
AK		 1717 if (rec->switch_output.num_files) {
1718 int n = rec->switch_output.cur_file + 1;
1719
1720 if (n >= rec->switch_output.num_files)
1721 n = 0;
1722 rec->switch_output.cur_file = n;
1723 if (rec->switch_output.filenames[n]) {
1724 remove(rec->switch_output.filenames[n]);
d8f9da24 1725 zfree(&rec->switch_output.filenames[n]);
03724b2e
AK		 1726 }
1727 rec->switch_output.filenames[n] = new_filename;
1728 } else {
1729 free(new_filename);
1730 }
1731
3c1cb7e3 1732 /* Output tracking events */
be7b0c9e 1733 if (!at_exit) {
4ea648ae 1734 record__synthesize(rec, false);
3c1cb7e3 1735
be7b0c9e
WN		 1736 /*
1737 * In 'perf record --switch-output' without -a,
1738 * record__synthesize() in record__switch_output() won't
1739 * generate tracking events because there's no thread_map
1740 * in evlist. Which causes newly created perf.data doesn't
1741 * contain map and comm information.
1742 * Create a fake thread_map and directly call
1743 * perf_event__synthesize_thread_map() for those events.
1744 */
1745 if (target__none(&rec->opts.target))
4ea648ae 1746 record__synthesize_workload(rec, false);
be7b0c9e 1747 }
ecfd7a9c
WN		 1748 return fd;
1749}
1750
f33cbe72
ACM		 1751static volatile int workload_exec_errno;
1752
1753/*
7b392ef0 1754 * evlist__prepare_workload will send a SIGUSR1
f33cbe72
ACM		 1755 * if the fork fails, since we asked by setting its
1756 * want_signal to true.
1757 */
45604710
NK		 1758static void workload_exec_failed_signal(int signo __maybe_unused,
1759 siginfo_t *info,
f33cbe72
ACM		 1760 void *ucontext __maybe_unused)
1761{
1762 workload_exec_errno = info->si_value.sival_int;
1763 done = 1;
f33cbe72
ACM		 1764 child_finished = 1;
1765}
1766
2dd6d8a1 1767static void snapshot_sig_handler(int sig);
bfacbe3b 1768static void alarm_sig_handler(int sig);
2dd6d8a1 1769
db0ea13c 1770static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist)
ee667f94 1771{
b2cb615d 1772 if (evlist) {
547740f7
JO		 1773 if (evlist->mmap && evlist->mmap[0].core.base)
1774 return evlist->mmap[0].core.base;
1775 if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base)
1776 return evlist->overwrite_mmap[0].core.base;
b2cb615d 1777 }
ee667f94
WN		 1778 return NULL;
1779}
1780
c45628b0
WN		 1781static const struct perf_event_mmap_page *record__pick_pc(struct record *rec)
1782{
db0ea13c 1783 const struct perf_event_mmap_page *pc = evlist__pick_pc(rec->evlist);
ee667f94
WN		 1784 if (pc)
1785 return pc;
c45628b0
WN		 1786 return NULL;
1787}
1788
4ea648ae 1789static int record__synthesize(struct record *rec, bool tail)
c45c86eb
WN		 1790{
1791 struct perf_session *session = rec->session;
1792 struct machine *machine = &session->machines.host;
8ceb41d7 1793 struct perf_data *data = &rec->data;
c45c86eb
WN		 1794 struct record_opts *opts = &rec->opts;
1795 struct perf_tool *tool = &rec->tool;
c45c86eb 1796 int err = 0;
d99c22ea 1797 event_op f = process_synthesized_event;
c45c86eb 1798
4ea648ae
WN		 1799 if (rec->opts.tail_synthesize != tail)
1800 return 0;
1801
8ceb41d7 1802 if (data->is_pipe) {
c3a057dc 1803 err = perf_event__synthesize_for_pipe(tool, session, data,
a2015516 1804 process_synthesized_event);
c3a057dc
NK		 1805 if (err < 0)
1806 goto out;
a2015516 1807
c3a057dc 1808 rec->bytes_written += err;
c45c86eb
WN		 1809 }
1810
c45628b0 1811 err = perf_event__synth_time_conv(record__pick_pc(rec), tool,
46bc29b9
AH		 1812 process_synthesized_event, machine);
1813 if (err)
1814 goto out;
1815
c0a6de06 1816 /* Synthesize id_index before auxtrace_info */
61750473 1817 if (rec->opts.auxtrace_sample_mode || rec->opts.full_auxtrace) {
c0a6de06
AH		 1818 err = perf_event__synthesize_id_index(tool,
1819 process_synthesized_event,
1820 session->evlist, machine);
1821 if (err)
1822 goto out;
1823 }
1824
c45c86eb
WN		 1825 if (rec->opts.full_auxtrace) {
1826 err = perf_event__synthesize_auxtrace_info(rec->itr, tool,
1827 session, process_synthesized_event);
1828 if (err)
1829 goto out;
1830 }
1831
78e1bc25 1832 if (!evlist__exclude_kernel(rec->evlist)) {
6c443954
ACM		 1833 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
1834 machine);
1835 WARN_ONCE(err < 0, "Couldn't record kernel reference relocation symbol\n"
1836 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
1837 "Check /proc/kallsyms permission or run as root.\n");
1838
1839 err = perf_event__synthesize_modules(tool, process_synthesized_event,
1840 machine);
1841 WARN_ONCE(err < 0, "Couldn't record kernel module information.\n"
1842 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
1843 "Check /proc/modules permission or run as root.\n");
1844 }
c45c86eb
WN		 1845
1846 if (perf_guest) {
1847 machines__process_guests(&session->machines,
1848 perf_event__synthesize_guest_os, tool);
1849 }
1850
bfd8f72c
AK		 1851 err = perf_event__synthesize_extra_attr(&rec->tool,
1852 rec->evlist,
1853 process_synthesized_event,
1854 data->is_pipe);
1855 if (err)
1856 goto out;
1857
03617c22 1858 err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads,
373565d2
AK		 1859 process_synthesized_event,
1860 NULL);
1861 if (err < 0) {
1862 pr_err("Couldn't synthesize thread map.\n");
1863 return err;
1864 }
1865
f72f901d 1866 err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.cpus,
373565d2
AK		 1867 process_synthesized_event, NULL);
1868 if (err < 0) {
1869 pr_err("Couldn't synthesize cpu map.\n");
1870 return err;
1871 }
1872
e5416950 1873 err = perf_event__synthesize_bpf_events(session, process_synthesized_event,
7b612e29
SL		 1874 machine, opts);
1875 if (err < 0)
1876 pr_warning("Couldn't synthesize bpf events.\n");
1877
41b740b6
NK		 1878 if (rec->opts.synth & PERF_SYNTH_CGROUP) {
1879 err = perf_event__synthesize_cgroups(tool, process_synthesized_event,
1880 machine);
1881 if (err < 0)
1882 pr_warning("Couldn't synthesize cgroup events.\n");
1883 }
ab64069f 1884
d99c22ea
SE		 1885 if (rec->opts.nr_threads_synthesize > 1) {
1886 perf_set_multithreaded();
1887 f = process_locked_synthesized_event;
1888 }
1889
41b740b6
NK		 1890 if (rec->opts.synth & PERF_SYNTH_TASK) {
1891 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
1892
1893 err = __machine__synthesize_threads(machine, tool, &opts->target,
1894 rec->evlist->core.threads,
1895 f, needs_mmap, opts->sample_address,
1896 rec->opts.nr_threads_synthesize);
1897 }
d99c22ea
SE		 1898
1899 if (rec->opts.nr_threads_synthesize > 1)
1900 perf_set_singlethreaded();
1901
c45c86eb
WN		 1902out:
1903 return err;
1904}
1905
899e5ffb
ACM		 1906static int record__process_signal_event(union perf_event *event __maybe_unused, void *data)
1907{
1908 struct record *rec = data;
1909 pthread_kill(rec->thread_id, SIGUSR2);
1910 return 0;
1911}
1912
23cbb41c
ACM		 1913static int record__setup_sb_evlist(struct record *rec)
1914{
1915 struct record_opts *opts = &rec->opts;
1916
1917 if (rec->sb_evlist != NULL) {
1918 /*
1919 * We get here if --switch-output-event populated the
1920 * sb_evlist, so associate a callback that will send a SIGUSR2
1921 * to the main thread.
1922 */
1923 evlist__set_cb(rec->sb_evlist, record__process_signal_event, rec);
1924 rec->thread_id = pthread_self();
1925 }
1101c872 1926#ifdef HAVE_LIBBPF_SUPPORT
23cbb41c
ACM		 1927 if (!opts->no_bpf_event) {
1928 if (rec->sb_evlist == NULL) {
1929 rec->sb_evlist = evlist__new();
1930
1931 if (rec->sb_evlist == NULL) {
1932 pr_err("Couldn't create side band evlist.\n.");
1933 return -1;
1934 }
1935 }
1936
1937 if (evlist__add_bpf_sb_event(rec->sb_evlist, &rec->session->header.env)) {
1938 pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
1939 return -1;
1940 }
1941 }
1101c872 1942#endif
08c83997 1943 if (evlist__start_sb_thread(rec->sb_evlist, &rec->opts.target)) {
23cbb41c
ACM		 1944 pr_debug("Couldn't start the BPF side band thread:\nBPF programs starting from now on won't be annotatable\n");
1945 opts->no_bpf_event = true;
1946 }
1947
1948 return 0;
1949}
1950
d1e325cf
JO		 1951static int record__init_clock(struct record *rec)
1952{
1953 struct perf_session *session = rec->session;
1954 struct timespec ref_clockid;
1955 struct timeval ref_tod;
1956 u64 ref;
1957
1958 if (!rec->opts.use_clockid)
1959 return 0;
1960
9d88a1a1
JO		 1961 if (rec->opts.use_clockid && rec->opts.clockid_res_ns)
1962 session->header.env.clock.clockid_res_ns = rec->opts.clockid_res_ns;
1963
d1e325cf
JO		 1964 session->header.env.clock.clockid = rec->opts.clockid;
1965
1966 if (gettimeofday(&ref_tod, NULL) != 0) {
1967 pr_err("gettimeofday failed, cannot set reference time.\n");
1968 return -1;
1969 }
1970
1971 if (clock_gettime(rec->opts.clockid, &ref_clockid)) {
1972 pr_err("clock_gettime failed, cannot set reference time.\n");
1973 return -1;
1974 }
1975
1976 ref = (u64) ref_tod.tv_sec * NSEC_PER_SEC +
1977 (u64) ref_tod.tv_usec * NSEC_PER_USEC;
1978
1979 session->header.env.clock.tod_ns = ref;
1980
1981 ref = (u64) ref_clockid.tv_sec * NSEC_PER_SEC +
1982 (u64) ref_clockid.tv_nsec;
1983
1984 session->header.env.clock.clockid_ns = ref;
1985 return 0;
1986}
1987
d20aff15
AH		 1988static void hit_auxtrace_snapshot_trigger(struct record *rec)
1989{
1990 if (trigger_is_ready(&auxtrace_snapshot_trigger)) {
1991 trigger_hit(&auxtrace_snapshot_trigger);
1992 auxtrace_record__snapshot_started = 1;
1993 if (auxtrace_record__snapshot_start(rec->itr))
1994 trigger_error(&auxtrace_snapshot_trigger);
1995 }
1996}
1997
91c0f5ec
JY		 1998static void record__uniquify_name(struct record *rec)
1999{
2000 struct evsel *pos;
2001 struct evlist *evlist = rec->evlist;
2002 char *new_name;
2003 int ret;
2004
2005 if (!perf_pmu__has_hybrid())
2006 return;
2007
2008 evlist__for_each_entry(evlist, pos) {
2009 if (!evsel__is_hybrid(pos))
2010 continue;
2011
2012 if (strchr(pos->name, '/'))
2013 continue;
2014
2015 ret = asprintf(&new_name, "%s/%s/",
2016 pos->pmu_name, pos->name);
2017 if (ret) {
2018 free(pos->name);
2019 pos->name = new_name;
2020 }
2021 }
2022}
2023
1e5de7d9
AB		 2024static int record__terminate_thread(struct record_thread *thread_data)
2025{
2026 int err;
2027 enum thread_msg ack = THREAD_MSG__UNDEFINED;
2028 pid_t tid = thread_data->tid;
2029
2030 close(thread_data->pipes.msg[1]);
2031 thread_data->pipes.msg[1] = -1;
2032 err = read(thread_data->pipes.ack[0], &ack, sizeof(ack));
2033 if (err > 0)
2034 pr_debug2("threads[%d]: sent %s\n", tid, thread_msg_tags[ack]);
2035 else
2036 pr_warning("threads[%d]: failed to receive termination notification from %d\n",
2037 thread->tid, tid);
2038
2039 return 0;
2040}
2041
396b626b
AB		 2042static int record__start_threads(struct record *rec)
2043{
3217e9fe 2044 int t, tt, err, ret = 0, nr_threads = rec->nr_threads;
396b626b 2045 struct record_thread *thread_data = rec->thread_data;
3217e9fe
AB		 2046 sigset_t full, mask;
2047 pthread_t handle;
2048 pthread_attr_t attrs;
396b626b
AB		 2049
2050 thread = &thread_data[0];
2051
3217e9fe
AB		 2052 if (!record__threads_enabled(rec))
2053 return 0;
2054
2055 sigfillset(&full);
2056 if (sigprocmask(SIG_SETMASK, &full, &mask)) {
2057 pr_err("Failed to block signals on threads start: %s\n", strerror(errno));
2058 return -1;
2059 }
2060
2061 pthread_attr_init(&attrs);
2062 pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED);
2063
2064 for (t = 1; t < nr_threads; t++) {
2065 enum thread_msg msg = THREAD_MSG__UNDEFINED;
2066
2067#ifdef HAVE_PTHREAD_ATTR_SETAFFINITY_NP
2068 pthread_attr_setaffinity_np(&attrs,
2069 MMAP_CPU_MASK_BYTES(&(thread_data[t].mask->affinity)),
2070 (cpu_set_t *)(thread_data[t].mask->affinity.bits));
2071#endif
2072 if (pthread_create(&handle, &attrs, record__thread, &thread_data[t])) {
2073 for (tt = 1; tt < t; tt++)
2074 record__terminate_thread(&thread_data[t]);
2075 pr_err("Failed to start threads: %s\n", strerror(errno));
2076 ret = -1;
2077 goto out_err;
2078 }
2079
2080 err = read(thread_data[t].pipes.ack[0], &msg, sizeof(msg));
2081 if (err > 0)
2082 pr_debug2("threads[%d]: sent %s\n", rec->thread_data[t].tid,
2083 thread_msg_tags[msg]);
2084 else
2085 pr_warning("threads[%d]: failed to receive start notification from %d\n",
2086 thread->tid, rec->thread_data[t].tid);
2087 }
2088
2089 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
2090 (cpu_set_t *)thread->mask->affinity.bits);
2091
396b626b
AB		 2092 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
2093
3217e9fe
AB		 2094out_err:
2095 pthread_attr_destroy(&attrs);
2096
2097 if (sigprocmask(SIG_SETMASK, &mask, NULL)) {
2098 pr_err("Failed to unblock signals on threads start: %s\n", strerror(errno));
2099 ret = -1;
2100 }
2101
2102 return ret;
396b626b
AB		 2103}
2104
2105static int record__stop_threads(struct record *rec)
2106{
2107 int t;
2108 struct record_thread *thread_data = rec->thread_data;
2109
1e5de7d9
AB		 2110 for (t = 1; t < rec->nr_threads; t++)
2111 record__terminate_thread(&thread_data[t]);
2112
610fbc01 2113 for (t = 0; t < rec->nr_threads; t++) {
396b626b 2114 rec->samples += thread_data[t].samples;
610fbc01
AB		 2115 if (!record__threads_enabled(rec))
2116 continue;
2117 rec->session->bytes_transferred += thread_data[t].bytes_transferred;
2118 rec->session->bytes_compressed += thread_data[t].bytes_compressed;
2119 pr_debug("threads[%d]: samples=%lld, wakes=%ld, ", thread_data[t].tid,
2120 thread_data[t].samples, thread_data[t].waking);
2121 if (thread_data[t].bytes_transferred && thread_data[t].bytes_compressed)
2122 pr_debug("transferred=%" PRIu64 ", compressed=%" PRIu64 "\n",
2123 thread_data[t].bytes_transferred, thread_data[t].bytes_compressed);
2124 else
2125 pr_debug("written=%" PRIu64 "\n", thread_data[t].bytes_written);
2126 }
396b626b
AB		 2127
2128 return 0;
2129}
2130
2131static unsigned long record__waking(struct record *rec)
2132{
2133 int t;
2134 unsigned long waking = 0;
2135 struct record_thread *thread_data = rec->thread_data;
2136
2137 for (t = 0; t < rec->nr_threads; t++)
2138 waking += thread_data[t].waking;
2139
2140 return waking;
2141}
2142
8c6f45a7 2143static int __cmd_record(struct record *rec, int argc, const char **argv)
16c8a109 2144{
57706abc 2145 int err;
45604710 2146 int status = 0;
46be604b 2147 const bool forks = argc > 0;
45694aa7 2148 struct perf_tool *tool = &rec->tool;
b4006796 2149 struct record_opts *opts = &rec->opts;
8ceb41d7 2150 struct perf_data *data = &rec->data;
d20deb64 2151 struct perf_session *session;
6dcf45ef 2152 bool disabled = false, draining = false;
42aa276f 2153 int fd;
d3c8c08e 2154 float ratio = 0;
acce0223 2155 enum evlist_ctl_cmd cmd = EVLIST_CTL_CMD_UNSUPPORTED;
de9ac07b 2156
45604710 2157 atexit(record__sig_exit);
f5970550
PZ		 2158 signal(SIGCHLD, sig_handler);
2159 signal(SIGINT, sig_handler);
804f7ac7 2160 signal(SIGTERM, sig_handler);
a074865e 2161 signal(SIGSEGV, sigsegv_handler);
c0bdc1c4 2162
f3b3614a
HB		 2163 if (rec->opts.record_namespaces)
2164 tool->namespace_events = true;
2165
8fb4b679
NK		 2166 if (rec->opts.record_cgroup) {
2167#ifdef HAVE_FILE_HANDLE
2168 tool->cgroup_events = true;
2169#else
2170 pr_err("cgroup tracking is not supported\n");
2171 return -1;
2172#endif
2173 }
2174
dc0c6127 2175 if (rec->opts.auxtrace_snapshot_mode || rec->switch_output.enabled) {
2dd6d8a1 2176 signal(SIGUSR2, snapshot_sig_handler);
3c1cb7e3
WN		 2177 if (rec->opts.auxtrace_snapshot_mode)
2178 trigger_on(&auxtrace_snapshot_trigger);
dc0c6127 2179 if (rec->switch_output.enabled)
3c1cb7e3 2180 trigger_on(&switch_output_trigger);
c0bdc1c4 2181 } else {
2dd6d8a1 2182 signal(SIGUSR2, SIG_IGN);
c0bdc1c4 2183 }
f5970550 2184
2681bd85 2185 session = perf_session__new(data, tool);
6ef81c55 2186 if (IS_ERR(session)) {
ffa91880 2187 pr_err("Perf session creation failed.\n");
6ef81c55 2188 return PTR_ERR(session);
a9a70bbc
ACM		 2189 }
2190
8ceb41d7 2191 fd = perf_data__fd(data);
d20deb64
ACM		 2192 rec->session = session;
2193
5d7f4116
AB		 2194 if (zstd_init(&session->zstd_data, rec->opts.comp_level) < 0) {
2195 pr_err("Compression initialization failed.\n");
2196 return -1;
2197 }
da231338
AM		 2198#ifdef HAVE_EVENTFD_SUPPORT
2199 done_fd = eventfd(0, EFD_NONBLOCK);
2200 if (done_fd < 0) {
2201 pr_err("Failed to create wakeup eventfd, error: %m\n");
2202 status = -1;
2203 goto out_delete_session;
2204 }
e16c2ce7 2205 err = evlist__add_wakeup_eventfd(rec->evlist, done_fd);
da231338
AM		 2206 if (err < 0) {
2207 pr_err("Failed to add wakeup eventfd to poll list\n");
2208 status = err;
2209 goto out_delete_session;
2210 }
2211#endif // HAVE_EVENTFD_SUPPORT
5d7f4116
AB		 2212
2213 session->header.env.comp_type = PERF_COMP_ZSTD;
2214 session->header.env.comp_level = rec->opts.comp_level;
2215
eeb399b5
AH		 2216 if (rec->opts.kcore &&
2217 !record__kcore_readable(&session->machines.host)) {
2218 pr_err("ERROR: kcore is not readable.\n");
2219 return -1;
2220 }
2221
d1e325cf
JO		 2222 if (record__init_clock(rec))
2223 return -1;
2224
8c6f45a7 2225 record__init_features(rec);
330aa675 2226
d4db3f16 2227 if (forks) {
7b392ef0
ACM		 2228 err = evlist__prepare_workload(rec->evlist, &opts->target, argv, data->is_pipe,
2229 workload_exec_failed_signal);
35b9d88e
ACM		 2230 if (err < 0) {
2231 pr_err("Couldn't run the workload!\n");
45604710 2232 status = err;
35b9d88e 2233 goto out_delete_session;
856e9660 2234 }
856e9660
PZ		 2235 }
2236
ad46e48c
JO		 2237 /*
2238 * If we have just single event and are sending data
2239 * through pipe, we need to force the ids allocation,
2240 * because we synthesize event name through the pipe
2241 * and need the id for that.
2242 */
6484d2f9 2243 if (data->is_pipe && rec->evlist->core.nr_entries == 1)
ad46e48c
JO		 2244 rec->opts.sample_id = true;
2245
91c0f5ec
JY		 2246 record__uniquify_name(rec);
2247
8c6f45a7 2248 if (record__open(rec) != 0) {
8d3eca20 2249 err = -1;
396b626b 2250 goto out_free_threads;
8d3eca20 2251 }
f6fa4375 2252 session->header.env.comp_mmap_len = session->evlist->core.mmap_len;
de9ac07b 2253
eeb399b5
AH		 2254 if (rec->opts.kcore) {
2255 err = record__kcore_copy(&session->machines.host, data);
2256 if (err) {
2257 pr_err("ERROR: Failed to copy kcore\n");
396b626b 2258 goto out_free_threads;
eeb399b5
AH		 2259 }
2260 }
2261
8690a2a7
WN		 2262 err = bpf__apply_obj_config();
2263 if (err) {
2264 char errbuf[BUFSIZ];
2265
2266 bpf__strerror_apply_obj_config(err, errbuf, sizeof(errbuf));
2267 pr_err("ERROR: Apply config to BPF failed: %s\n",
2268 errbuf);
396b626b 2269 goto out_free_threads;
8690a2a7
WN		 2270 }
2271
cca8482c
AH		 2272 /*
2273 * Normally perf_session__new would do this, but it doesn't have the
2274 * evlist.
2275 */
8cedf3a5 2276 if (rec->tool.ordered_events && !evlist__sample_id_all(rec->evlist)) {
cca8482c
AH		 2277 pr_warning("WARNING: No sample_id_all support, falling back to unordered processing\n");
2278 rec->tool.ordered_events = false;
2279 }
2280
3a683120 2281 if (!rec->evlist->core.nr_groups)
a8bb559b
NK		 2282 perf_header__clear_feat(&session->header, HEADER_GROUP_DESC);
2283
8ceb41d7 2284 if (data->is_pipe) {
42aa276f 2285 err = perf_header__write_pipe(fd);
529870e3 2286 if (err < 0)
396b626b 2287 goto out_free_threads;
563aecb2 2288 } else {
42aa276f 2289 err = perf_session__write_header(session, rec->evlist, fd, false);
d5eed904 2290 if (err < 0)
396b626b 2291 goto out_free_threads;
56b03f3c
ACM		 2292 }
2293
b38d85ef 2294 err = -1;
d3665498 2295 if (!rec->no_buildid
e20960c0 2296 && !perf_header__has_feat(&session->header, HEADER_BUILD_ID)) {
d3665498 2297 pr_err("Couldn't generate buildids. "
e20960c0 2298 "Use --no-buildid to profile anyway.\n");
396b626b 2299 goto out_free_threads;
e20960c0
RR		 2300 }
2301
23cbb41c
ACM		 2302 err = record__setup_sb_evlist(rec);
2303 if (err)
396b626b 2304 goto out_free_threads;
657ee553 2305
4ea648ae 2306 err = record__synthesize(rec, false);
c45c86eb 2307 if (err < 0)
396b626b 2308 goto out_free_threads;
8d3eca20 2309
d20deb64 2310 if (rec->realtime_prio) {
de9ac07b
PZ		 2311 struct sched_param param;
2312
d20deb64 2313 param.sched_priority = rec->realtime_prio;
de9ac07b 2314 if (sched_setscheduler(0, SCHED_FIFO, &param)) {
6beba7ad 2315 pr_err("Could not set realtime priority.\n");
8d3eca20 2316 err = -1;
396b626b 2317 goto out_free_threads;
de9ac07b
PZ		 2318 }
2319 }
2320
396b626b
AB		 2321 if (record__start_threads(rec))
2322 goto out_free_threads;
2323
774cb499
JO		 2324 /*
2325 * When perf is starting the traced process, all the events
2326 * (apart from group members) have enable_on_exec=1 set,
2327 * so don't spoil it by prematurely enabling them.
2328 */
6619a53e 2329 if (!target__none(&opts->target) && !opts->initial_delay)
1c87f165 2330 evlist__enable(rec->evlist);
764e16a3 2331
856e9660
PZ		 2332 /*
2333 * Let the child rip
2334 */
e803cf97 2335 if (forks) {
20a8a3cf 2336 struct machine *machine = &session->machines.host;
e5bed564 2337 union perf_event *event;
e907caf3 2338 pid_t tgid;
e5bed564
NK		 2339
2340 event = malloc(sizeof(event->comm) + machine->id_hdr_size);
2341 if (event == NULL) {
2342 err = -ENOMEM;
2343 goto out_child;
2344 }
2345
e803cf97
NK		 2346 /*
2347 * Some H/W events are generated before COMM event
2348 * which is emitted during exec(), so perf script
2349 * cannot see a correct process name for those events.
2350 * Synthesize COMM event to prevent it.
2351 */
e907caf3
HB		 2352 tgid = perf_event__synthesize_comm(tool, event,
2353 rec->evlist->workload.pid,
2354 process_synthesized_event,
2355 machine);
2356 free(event);
2357
2358 if (tgid == -1)
2359 goto out_child;
2360
2361 event = malloc(sizeof(event->namespaces) +
2362 (NR_NAMESPACES * sizeof(struct perf_ns_link_info)) +
2363 machine->id_hdr_size);
2364 if (event == NULL) {
2365 err = -ENOMEM;
2366 goto out_child;
2367 }
2368
2369 /*
2370 * Synthesize NAMESPACES event for the command specified.
2371 */
2372 perf_event__synthesize_namespaces(tool, event,
2373 rec->evlist->workload.pid,
2374 tgid, process_synthesized_event,
2375 machine);
e5bed564 2376 free(event);
e803cf97 2377
7b392ef0 2378 evlist__start_workload(rec->evlist);
e803cf97 2379 }
856e9660 2380
6619a53e 2381 if (opts->initial_delay) {
68cd3b45
AB		 2382 pr_info(EVLIST_DISABLED_MSG);
2383 if (opts->initial_delay > 0) {
2384 usleep(opts->initial_delay * USEC_PER_MSEC);
2385 evlist__enable(rec->evlist);
2386 pr_info(EVLIST_ENABLED_MSG);
2387 }
6619a53e
AK		 2388 }
2389
5f9cf599 2390 trigger_ready(&auxtrace_snapshot_trigger);
3c1cb7e3 2391 trigger_ready(&switch_output_trigger);
a074865e 2392 perf_hooks__invoke_record_start();
649c48a9 2393 for (;;) {
396b626b 2394 unsigned long long hits = thread->samples;
de9ac07b 2395
05737464
WN		 2396 /*
2397 * rec->evlist->bkw_mmap_state is possible to be
2398 * BKW_MMAP_EMPTY here: when done == true and
2399 * hits != rec->samples in previous round.
2400 *
ade9d208 2401 * evlist__toggle_bkw_mmap ensure we never
05737464
WN		 2402 * convert BKW_MMAP_EMPTY to BKW_MMAP_DATA_PENDING.
2403 */
2404 if (trigger_is_hit(&switch_output_trigger) || done || draining)
ade9d208 2405 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_DATA_PENDING);
05737464 2406
470530bb 2407 if (record__mmap_read_all(rec, false) < 0) {
5f9cf599 2408 trigger_error(&auxtrace_snapshot_trigger);
3c1cb7e3 2409 trigger_error(&switch_output_trigger);
8d3eca20 2410 err = -1;
45604710 2411 goto out_child;
8d3eca20 2412 }
de9ac07b 2413
2dd6d8a1
AH		 2414 if (auxtrace_record__snapshot_started) {
2415 auxtrace_record__snapshot_started = 0;
5f9cf599 2416 if (!trigger_is_error(&auxtrace_snapshot_trigger))
ce7b0e42 2417 record__read_auxtrace_snapshot(rec, false);
5f9cf599 2418 if (trigger_is_error(&auxtrace_snapshot_trigger)) {
2dd6d8a1
AH		 2419 pr_err("AUX area tracing snapshot failed\n");
2420 err = -1;
2421 goto out_child;
2422 }
2423 }
2424
3c1cb7e3 2425 if (trigger_is_hit(&switch_output_trigger)) {
05737464
WN		 2426 /*
2427 * If switch_output_trigger is hit, the data in
2428 * overwritable ring buffer should have been collected,
2429 * so bkw_mmap_state should be set to BKW_MMAP_EMPTY.
2430 *
2431 * If SIGUSR2 raise after or during record__mmap_read_all(),
2432 * record__mmap_read_all() didn't collect data from
2433 * overwritable ring buffer. Read again.
2434 */
2435 if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING)
2436 continue;
3c1cb7e3
WN		 2437 trigger_ready(&switch_output_trigger);
2438
05737464
WN		 2439 /*
2440 * Reenable events in overwrite ring buffer after
2441 * record__mmap_read_all(): we should have collected
2442 * data from it.
2443 */
ade9d208 2444 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_RUNNING);
05737464 2445
3c1cb7e3
WN		 2446 if (!quiet)
2447 fprintf(stderr, "[ perf record: dump data: Woken up %ld times ]\n",
396b626b
AB		 2448 record__waking(rec));
2449 thread->waking = 0;
3c1cb7e3
WN		 2450 fd = record__switch_output(rec, false);
2451 if (fd < 0) {
2452 pr_err("Failed to switch to new file\n");
2453 trigger_error(&switch_output_trigger);
2454 err = fd;
2455 goto out_child;
2456 }
bfacbe3b
JO		 2457
2458 /* re-arm the alarm */
2459 if (rec->switch_output.time)
2460 alarm(rec->switch_output.time);
3c1cb7e3
WN		 2461 }
2462
396b626b 2463 if (hits == thread->samples) {
6dcf45ef 2464 if (done || draining)
649c48a9 2465 break;
396b626b 2466 err = fdarray__poll(&thread->pollfd, -1);
a515114f
JO		 2467 /*
2468 * Propagate error, only if there's any. Ignore positive
2469 * number of returned events and interrupt error.
2470 */
2471 if (err > 0 || (err < 0 && errno == EINTR))
45604710 2472 err = 0;
396b626b 2473 thread->waking++;
6dcf45ef 2474
396b626b
AB		 2475 if (fdarray__filter(&thread->pollfd, POLLERR | POLLHUP,
2476 record__thread_munmap_filtered, NULL) == 0)
6dcf45ef 2477 draining = true;
396b626b
AB		 2478
2479 evlist__ctlfd_update(rec->evlist,
2480 &thread->pollfd.entries[thread->ctlfd_pos]);
8b412664
PZ		 2481 }
2482
acce0223
AB		 2483 if (evlist__ctlfd_process(rec->evlist, &cmd) > 0) {
2484 switch (cmd) {
d20aff15
AH		 2485 case EVLIST_CTL_CMD_SNAPSHOT:
2486 hit_auxtrace_snapshot_trigger(rec);
2487 evlist__ctlfd_ack(rec->evlist);
2488 break;
f186cd61
JO		 2489 case EVLIST_CTL_CMD_STOP:
2490 done = 1;
2491 break;
acce0223
AB		 2492 case EVLIST_CTL_CMD_ACK:
2493 case EVLIST_CTL_CMD_UNSUPPORTED:
991ae4eb
JO		 2494 case EVLIST_CTL_CMD_ENABLE:
2495 case EVLIST_CTL_CMD_DISABLE:
142544a9 2496 case EVLIST_CTL_CMD_EVLIST:
47fddcb4 2497 case EVLIST_CTL_CMD_PING:
acce0223
AB		 2498 default:
2499 break;
2500 }
2501 }
2502
774cb499
JO		 2503 /*
2504 * When perf is starting the traced process, at the end events
2505 * die with the process and we wait for that. Thus no need to
2506 * disable events in this case.
2507 */
602ad878 2508 if (done && !disabled && !target__none(&opts->target)) {
5f9cf599 2509 trigger_off(&auxtrace_snapshot_trigger);
e74676de 2510 evlist__disable(rec->evlist);
2711926a
JO		 2511 disabled = true;
2512 }
de9ac07b 2513 }
ce7b0e42 2514
5f9cf599 2515 trigger_off(&auxtrace_snapshot_trigger);
3c1cb7e3 2516 trigger_off(&switch_output_trigger);
de9ac07b 2517
ce7b0e42
AS		 2518 if (opts->auxtrace_snapshot_on_exit)
2519 record__auxtrace_snapshot_exit(rec);
2520
f33cbe72 2521 if (forks && workload_exec_errno) {
3535a696 2522 char msg[STRERR_BUFSIZE], strevsels[2048];
c8b5f2c9 2523 const char *emsg = str_error_r(workload_exec_errno, msg, sizeof(msg));
3535a696
ACM		 2524
2525 evlist__scnprintf_evsels(rec->evlist, sizeof(strevsels), strevsels);
2526
2527 pr_err("Failed to collect '%s' for the '%s' workload: %s\n",
2528 strevsels, argv[0], emsg);
f33cbe72 2529 err = -1;
45604710 2530 goto out_child;
f33cbe72
ACM		 2531 }
2532
e3d59112 2533 if (!quiet)
396b626b
AB		 2534 fprintf(stderr, "[ perf record: Woken up %ld times to write data ]\n",
2535 record__waking(rec));
b44308f5 2536
4ea648ae
WN		 2537 if (target__none(&rec->opts.target))
2538 record__synthesize_workload(rec, true);
2539
45604710 2540out_child:
396b626b 2541 record__stop_threads(rec);
470530bb 2542 record__mmap_read_all(rec, true);
396b626b 2543out_free_threads:
415ccb58 2544 record__free_thread_data(rec);
396b626b 2545 evlist__finalize_ctlfd(rec->evlist);
d3d1af6f
AB		 2546 record__aio_mmap_read_sync(rec);
2547
d3c8c08e
AB		 2548 if (rec->session->bytes_transferred && rec->session->bytes_compressed) {
2549 ratio = (float)rec->session->bytes_transferred/(float)rec->session->bytes_compressed;
2550 session->header.env.comp_ratio = ratio + 0.5;
2551 }
2552
45604710
NK		 2553 if (forks) {
2554 int exit_status;
addc2785 2555
45604710
NK		 2556 if (!child_finished)
2557 kill(rec->evlist->workload.pid, SIGTERM);
2558
2559 wait(&exit_status);
2560
2561 if (err < 0)
2562 status = err;
2563 else if (WIFEXITED(exit_status))
2564 status = WEXITSTATUS(exit_status);
2565 else if (WIFSIGNALED(exit_status))
2566 signr = WTERMSIG(exit_status);
2567 } else
2568 status = err;
2569
4ea648ae 2570 record__synthesize(rec, true);
e3d59112
NK		 2571 /* this will be recalculated during process_buildids() */
2572 rec->samples = 0;
2573
ecfd7a9c
WN		 2574 if (!err) {
2575 if (!rec->timestamp_filename) {
2576 record__finish_output(rec);
2577 } else {
2578 fd = record__switch_output(rec, true);
2579 if (fd < 0) {
2580 status = fd;
2581 goto out_delete_session;
2582 }
2583 }
2584 }
39d17dac 2585
a074865e
WN		 2586 perf_hooks__invoke_record_end();
2587
e3d59112
NK		 2588 if (!err && !quiet) {
2589 char samples[128];
ecfd7a9c
WN		 2590 const char *postfix = rec->timestamp_filename ?
2591 ".<timestamp>" : "";
e3d59112 2592
ef149c25 2593 if (rec->samples && !rec->opts.full_auxtrace)
e3d59112
NK		 2594 scnprintf(samples, sizeof(samples),
2595 " (%" PRIu64 " samples)", rec->samples);
2596 else
2597 samples[0] = '\0';
2598
d3c8c08e 2599 fprintf(stderr, "[ perf record: Captured and wrote %.3f MB %s%s%s",
8ceb41d7 2600 perf_data__size(data) / 1024.0 / 1024.0,
2d4f2799 2601 data->path, postfix, samples);
d3c8c08e
AB		 2602 if (ratio) {
2603 fprintf(stderr, ", compressed (original %.3f MB, ratio is %.3f)",
2604 rec->session->bytes_transferred / 1024.0 / 1024.0,
2605 ratio);
2606 }
2607 fprintf(stderr, " ]\n");
e3d59112
NK		 2608 }
2609
39d17dac 2610out_delete_session:
da231338
AM		 2611#ifdef HAVE_EVENTFD_SUPPORT
2612 if (done_fd >= 0)
2613 close(done_fd);
2614#endif
5d7f4116 2615 zstd_fini(&session->zstd_data);
39d17dac 2616 perf_session__delete(session);
657ee553
SL		 2617
2618 if (!opts->no_bpf_event)
08c83997 2619 evlist__stop_sb_thread(rec->sb_evlist);
45604710 2620 return status;
de9ac07b 2621}
0e9b20b8 2622
0883e820 2623static void callchain_debug(struct callchain_param *callchain)
09b0fd45 2624{
aad2b21c 2625 static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF", "LBR" };
a601fdff 2626
0883e820 2627 pr_debug("callchain: type %s\n", str[callchain->record_mode]);
26d33022 2628
0883e820 2629 if (callchain->record_mode == CALLCHAIN_DWARF)
09b0fd45 2630 pr_debug("callchain: stack dump size %d\n",
0883e820 2631 callchain->dump_size);
09b0fd45
JO		 2632}
2633
0883e820
ACM		 2634int record_opts__parse_callchain(struct record_opts *record,
2635 struct callchain_param *callchain,
2636 const char *arg, bool unset)
09b0fd45 2637{
09b0fd45 2638 int ret;
0883e820 2639 callchain->enabled = !unset;
eb853e80 2640
09b0fd45
JO		 2641 /* --no-call-graph */
2642 if (unset) {
0883e820 2643 callchain->record_mode = CALLCHAIN_NONE;
09b0fd45
JO		 2644 pr_debug("callchain: disabled\n");
2645 return 0;
2646 }
2647
0883e820 2648 ret = parse_callchain_record_opt(arg, callchain);
5c0cf224
JO		 2649 if (!ret) {
2650 /* Enable data address sampling for DWARF unwind. */
0883e820 2651 if (callchain->record_mode == CALLCHAIN_DWARF)
5c0cf224 2652 record->sample_address = true;
0883e820 2653 callchain_debug(callchain);
5c0cf224 2654 }
26d33022
JO		 2655
2656 return ret;
2657}
2658
0883e820
ACM		 2659int record_parse_callchain_opt(const struct option *opt,
2660 const char *arg,
2661 int unset)
2662{
2663 return record_opts__parse_callchain(opt->value, &callchain_param, arg, unset);
2664}
2665
c421e80b 2666int record_callchain_opt(const struct option *opt,
09b0fd45
JO		 2667 const char *arg __maybe_unused,
2668 int unset __maybe_unused)
2669{
2ddd5c04 2670 struct callchain_param *callchain = opt->value;
c421e80b 2671
2ddd5c04 2672 callchain->enabled = true;
09b0fd45 2673
2ddd5c04
ACM		 2674 if (callchain->record_mode == CALLCHAIN_NONE)
2675 callchain->record_mode = CALLCHAIN_FP;
eb853e80 2676
2ddd5c04 2677 callchain_debug(callchain);
09b0fd45
JO		 2678 return 0;
2679}
2680
eb853e80
JO		 2681static int perf_record_config(const char *var, const char *value, void *cb)
2682{
7a29c087
NK		 2683 struct record *rec = cb;
2684
2685 if (!strcmp(var, "record.build-id")) {
2686 if (!strcmp(value, "cache"))
2687 rec->no_buildid_cache = false;
2688 else if (!strcmp(value, "no-cache"))
2689 rec->no_buildid_cache = true;
2690 else if (!strcmp(value, "skip"))
2691 rec->no_buildid = true;
e29386c8
JO		 2692 else if (!strcmp(value, "mmap"))
2693 rec->buildid_mmap = true;
7a29c087
NK		 2694 else
2695 return -1;
2696 return 0;
2697 }
cff17205
YX		 2698 if (!strcmp(var, "record.call-graph")) {
2699 var = "call-graph.record-mode";
2700 return perf_default_config(var, value, cb);
2701 }
93f20c0f
AB		 2702#ifdef HAVE_AIO_SUPPORT
2703 if (!strcmp(var, "record.aio")) {
2704 rec->opts.nr_cblocks = strtol(value, NULL, 0);
2705 if (!rec->opts.nr_cblocks)
2706 rec->opts.nr_cblocks = nr_cblocks_default;
2707 }
2708#endif
9bce13ea
JO		 2709 if (!strcmp(var, "record.debuginfod")) {
2710 rec->debuginfod.urls = strdup(value);
2711 if (!rec->debuginfod.urls)
2712 return -ENOMEM;
2713 rec->debuginfod.set = true;
2714 }
eb853e80 2715
cff17205 2716 return 0;
eb853e80
JO		 2717}
2718
814c8c38 2719
f4fe11b7
AB		 2720static int record__parse_affinity(const struct option *opt, const char *str, int unset)
2721{
2722 struct record_opts *opts = (struct record_opts *)opt->value;
2723
2724 if (unset || !str)
2725 return 0;
2726
2727 if (!strcasecmp(str, "node"))
2728 opts->affinity = PERF_AFFINITY_NODE;
2729 else if (!strcasecmp(str, "cpu"))
2730 opts->affinity = PERF_AFFINITY_CPU;
2731
2732 return 0;
2733}
2734
7954f716
AB		 2735static int record__mmap_cpu_mask_alloc(struct mmap_cpu_mask *mask, int nr_bits)
2736{
2737 mask->nbits = nr_bits;
2738 mask->bits = bitmap_zalloc(mask->nbits);
2739 if (!mask->bits)
2740 return -ENOMEM;
2741
2742 return 0;
2743}
2744
2745static void record__mmap_cpu_mask_free(struct mmap_cpu_mask *mask)
2746{
2747 bitmap_free(mask->bits);
2748 mask->nbits = 0;
2749}
2750
2751static int record__thread_mask_alloc(struct thread_mask *mask, int nr_bits)
2752{
2753 int ret;
2754
2755 ret = record__mmap_cpu_mask_alloc(&mask->maps, nr_bits);
2756 if (ret) {
2757 mask->affinity.bits = NULL;
2758 return ret;
2759 }
2760
2761 ret = record__mmap_cpu_mask_alloc(&mask->affinity, nr_bits);
2762 if (ret) {
2763 record__mmap_cpu_mask_free(&mask->maps);
2764 mask->maps.bits = NULL;
2765 }
2766
2767 return ret;
2768}
2769
2770static void record__thread_mask_free(struct thread_mask *mask)
2771{
2772 record__mmap_cpu_mask_free(&mask->maps);
2773 record__mmap_cpu_mask_free(&mask->affinity);
2774}
2775
06380a84
AB		 2776static int record__parse_threads(const struct option *opt, const char *str, int unset)
2777{
2778 struct record_opts *opts = opt->value;
2779
2780 if (unset || !str || !strlen(str))
2781 opts->threads_spec = THREAD_SPEC__CPU;
2782
2783 return 0;
2784}
2785
6d575816
JS		 2786static int parse_output_max_size(const struct option *opt,
2787 const char *str, int unset)
2788{
2789 unsigned long *s = (unsigned long *)opt->value;
2790 static struct parse_tag tags_size[] = {
2791 { .tag = 'B', .mult = 1 },
2792 { .tag = 'K', .mult = 1 << 10 },
2793 { .tag = 'M', .mult = 1 << 20 },
2794 { .tag = 'G', .mult = 1 << 30 },
2795 { .tag = 0 },
2796 };
2797 unsigned long val;
2798
2799 if (unset) {
2800 *s = 0;
2801 return 0;
2802 }
2803
2804 val = parse_tag_value(str, tags_size);
2805 if (val != (unsigned long) -1) {
2806 *s = val;
2807 return 0;
2808 }
2809
2810 return -1;
2811}
2812
e9db1310
AH		 2813static int record__parse_mmap_pages(const struct option *opt,
2814 const char *str,
2815 int unset __maybe_unused)
2816{
2817 struct record_opts *opts = opt->value;
2818 char *s, *p;
2819 unsigned int mmap_pages;
2820 int ret;
2821
2822 if (!str)
2823 return -EINVAL;
2824
2825 s = strdup(str);
2826 if (!s)
2827 return -ENOMEM;
2828
2829 p = strchr(s, ',');
2830 if (p)
2831 *p = '\0';
2832
2833 if (*s) {
25f84702 2834 ret = __evlist__parse_mmap_pages(&mmap_pages, s);
e9db1310
AH		 2835 if (ret)
2836 goto out_free;
2837 opts->mmap_pages = mmap_pages;
2838 }
2839
2840 if (!p) {
2841 ret = 0;
2842 goto out_free;
2843 }
2844
25f84702 2845 ret = __evlist__parse_mmap_pages(&mmap_pages, p + 1);
e9db1310
AH		 2846 if (ret)
2847 goto out_free;
2848
2849 opts->auxtrace_mmap_pages = mmap_pages;
2850
2851out_free:
2852 free(s);
2853 return ret;
2854}
2855
7248e308
AT		 2856void __weak arch__add_leaf_frame_record_opts(struct record_opts *opts __maybe_unused)
2857{
2858}
2859
1d078ccb
AB		 2860static int parse_control_option(const struct option *opt,
2861 const char *str,
2862 int unset __maybe_unused)
2863{
9864a66d 2864 struct record_opts *opts = opt->value;
1d078ccb 2865
a8fcbd26
AH		 2866 return evlist__parse_control(str, &opts->ctl_fd, &opts->ctl_fd_ack, &opts->ctl_fd_close);
2867}
2868
0c582449
JO		 2869static void switch_output_size_warn(struct record *rec)
2870{
9521b5f2 2871 u64 wakeup_size = evlist__mmap_size(rec->opts.mmap_pages);
0c582449
JO		 2872 struct switch_output *s = &rec->switch_output;
2873
2874 wakeup_size /= 2;
2875
2876 if (s->size < wakeup_size) {
2877 char buf[100];
2878
2879 unit_number__scnprintf(buf, sizeof(buf), wakeup_size);
2880 pr_warning("WARNING: switch-output data size lower than "
2881 "wakeup kernel buffer size (%s) "
2882 "expect bigger perf.data sizes\n", buf);
2883 }
2884}
2885
cb4e1ebb
JO		 2886static int switch_output_setup(struct record *rec)
2887{
2888 struct switch_output *s = &rec->switch_output;
dc0c6127
JO		 2889 static struct parse_tag tags_size[] = {
2890 { .tag = 'B', .mult = 1 },
2891 { .tag = 'K', .mult = 1 << 10 },
2892 { .tag = 'M', .mult = 1 << 20 },
2893 { .tag = 'G', .mult = 1 << 30 },
2894 { .tag = 0 },
2895 };
bfacbe3b
JO		 2896 static struct parse_tag tags_time[] = {
2897 { .tag = 's', .mult = 1 },
2898 { .tag = 'm', .mult = 60 },
2899 { .tag = 'h', .mult = 60*60 },
2900 { .tag = 'd', .mult = 60*60*24 },
2901 { .tag = 0 },
2902 };
dc0c6127 2903 unsigned long val;
cb4e1ebb 2904
899e5ffb
ACM		 2905 /*
2906 * If we're using --switch-output-events, then we imply its
2907 * --switch-output=signal, as we'll send a SIGUSR2 from the side band
2908 * thread to its parent.
2909 */
2910 if (rec->switch_output_event_set)
2911 goto do_signal;
2912
cb4e1ebb
JO		 2913 if (!s->set)
2914 return 0;
2915
2916 if (!strcmp(s->str, "signal")) {
899e5ffb 2917do_signal:
cb4e1ebb
JO		 2918 s->signal = true;
2919 pr_debug("switch-output with SIGUSR2 signal\n");
dc0c6127
JO		 2920 goto enabled;
2921 }
2922
2923 val = parse_tag_value(s->str, tags_size);
2924 if (val != (unsigned long) -1) {
2925 s->size = val;
2926 pr_debug("switch-output with %s size threshold\n", s->str);
2927 goto enabled;
cb4e1ebb
JO		 2928 }
2929
bfacbe3b
JO		 2930 val = parse_tag_value(s->str, tags_time);
2931 if (val != (unsigned long) -1) {
2932 s->time = val;
2933 pr_debug("switch-output with %s time threshold (%lu seconds)\n",
2934 s->str, s->time);
2935 goto enabled;
2936 }
2937
cb4e1ebb 2938 return -1;
dc0c6127
JO		 2939
2940enabled:
2941 rec->timestamp_filename = true;
2942 s->enabled = true;
0c582449
JO		 2943
2944 if (s->size && !rec->opts.no_buffering)
2945 switch_output_size_warn(rec);
2946
dc0c6127 2947 return 0;
cb4e1ebb
JO		 2948}
2949
e5b2c207 2950static const char * const __record_usage[] = {
9e096753
MG		 2951 "perf record [<options>] [<command>]",
2952 "perf record [<options>] -- <command> [<options>]",
0e9b20b8
IM		 2953 NULL
2954};
e5b2c207 2955const char * const *record_usage = __record_usage;
0e9b20b8 2956
6e0a9b3d
ACM		 2957static int build_id__process_mmap(struct perf_tool *tool, union perf_event *event,
2958 struct perf_sample *sample, struct machine *machine)
2959{
2960 /*
2961 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
2962 * no need to add them twice.
2963 */
2964 if (!(event->header.misc & PERF_RECORD_MISC_USER))
2965 return 0;
2966 return perf_event__process_mmap(tool, event, sample, machine);
2967}
2968
2969static int build_id__process_mmap2(struct perf_tool *tool, union perf_event *event,
2970 struct perf_sample *sample, struct machine *machine)
2971{
2972 /*
2973 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
2974 * no need to add them twice.
2975 */
2976 if (!(event->header.misc & PERF_RECORD_MISC_USER))
2977 return 0;
2978
2979 return perf_event__process_mmap2(tool, event, sample, machine);
2980}
2981
66286ed3
AH		 2982static int process_timestamp_boundary(struct perf_tool *tool,
2983 union perf_event *event __maybe_unused,
2984 struct perf_sample *sample,
2985 struct machine *machine __maybe_unused)
2986{
2987 struct record *rec = container_of(tool, struct record, tool);
2988
2989 set_timestamp_boundary(rec, sample->time);
2990 return 0;
2991}
2992
41b740b6
NK		 2993static int parse_record_synth_option(const struct option *opt,
2994 const char *str,
2995 int unset __maybe_unused)
2996{
2997 struct record_opts *opts = opt->value;
2998 char *p = strdup(str);
2999
3000 if (p == NULL)
3001 return -1;
3002
3003 opts->synth = parse_synth_opt(p);
3004 free(p);
3005
3006 if (opts->synth < 0) {
3007 pr_err("Invalid synth option: %s\n", str);
3008 return -1;
3009 }
3010 return 0;
3011}
3012
d20deb64 3013/*
8c6f45a7
ACM		 3014 * XXX Ideally would be local to cmd_record() and passed to a record__new
3015 * because we need to have access to it in record__exit, that is called
d20deb64
ACM		 3016 * after cmd_record() exits, but since record_options need to be accessible to
3017 * builtin-script, leave it here.
3018 *
3019 * At least we don't ouch it in all the other functions here directly.
3020 *
3021 * Just say no to tons of global variables, sigh.
3022 */
8c6f45a7 3023static struct record record = {
d20deb64 3024 .opts = {
8affc2b8 3025 .sample_time = true,
d20deb64
ACM		 3026 .mmap_pages = UINT_MAX,
3027 .user_freq = UINT_MAX,
3028 .user_interval = ULLONG_MAX,
447a6013 3029 .freq = 4000,
d1cb9fce
NK		 3030 .target = {
3031 .uses_mmap = true,
3aa5939d 3032 .default_per_cpu = true,
d1cb9fce 3033 },
470530bb 3034 .mmap_flush = MMAP_FLUSH_DEFAULT,
d99c22ea 3035 .nr_threads_synthesize = 1,
1d078ccb
AB		 3036 .ctl_fd = -1,
3037 .ctl_fd_ack = -1,
41b740b6 3038 .synth = PERF_SYNTH_ALL,
d20deb64 3039 },
e3d59112
NK		 3040 .tool = {
3041 .sample = process_sample_event,
3042 .fork = perf_event__process_fork,
cca8482c 3043 .exit = perf_event__process_exit,
e3d59112 3044 .comm = perf_event__process_comm,
f3b3614a 3045 .namespaces = perf_event__process_namespaces,
6e0a9b3d
ACM		 3046 .mmap = build_id__process_mmap,
3047 .mmap2 = build_id__process_mmap2,
66286ed3
AH		 3048 .itrace_start = process_timestamp_boundary,
3049 .aux = process_timestamp_boundary,
cca8482c 3050 .ordered_events = true,
e3d59112 3051 },
d20deb64 3052};
7865e817 3053
76a26549
NK		 3054const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
3055 "\n\t\t\t\tDefault: fp";
61eaa3be 3056
0aab2136
WN		 3057static bool dry_run;
3058
d20deb64
ACM		 3059/*
3060 * XXX Will stay a global variable till we fix builtin-script.c to stop messing
3061 * with it and switch to use the library functions in perf_evlist that came
b4006796 3062 * from builtin-record.c, i.e. use record_opts,
7b392ef0 3063 * evlist__prepare_workload, etc instead of fork+exec'in 'perf record',
d20deb64
ACM		 3064 * using pipes, etc.
3065 */
efd21307 3066static struct option __record_options[] = {
d20deb64 3067 OPT_CALLBACK('e', "event", &record.evlist, "event",
86847b62 3068 "event selector. use 'perf list' to list available events",
f120f9d5 3069 parse_events_option),
d20deb64 3070 OPT_CALLBACK(0, "filter", &record.evlist, "filter",
c171b552 3071 "event filter", parse_filter),
4ba1faa1
WN		 3072 OPT_CALLBACK_NOOPT(0, "exclude-perf", &record.evlist,
3073 NULL, "don't record events from perf itself",
3074 exclude_perf),
bea03405 3075 OPT_STRING('p', "pid", &record.opts.target.pid, "pid",
d6d901c2 3076 "record events on existing process id"),
bea03405 3077 OPT_STRING('t', "tid", &record.opts.target.tid, "tid",
d6d901c2 3078 "record events on existing thread id"),
d20deb64 3079 OPT_INTEGER('r', "realtime", &record.realtime_prio,
0e9b20b8 3080 "collect data with this RT SCHED_FIFO priority"),
509051ea 3081 OPT_BOOLEAN(0, "no-buffering", &record.opts.no_buffering,
acac03fa 3082 "collect data without buffering"),
d20deb64 3083 OPT_BOOLEAN('R', "raw-samples", &record.opts.raw_samples,
daac07b2 3084 "collect raw sample records from all opened counters"),
bea03405 3085 OPT_BOOLEAN('a', "all-cpus", &record.opts.target.system_wide,
0e9b20b8 3086 "system-wide collection from all CPUs"),
bea03405 3087 OPT_STRING('C', "cpu", &record.opts.target.cpu_list, "cpu",
c45c6ea2 3088 "list of cpus to monitor"),
d20deb64 3089 OPT_U64('c', "count", &record.opts.user_interval, "event period to sample"),
2d4f2799 3090 OPT_STRING('o', "output", &record.data.path, "file",
abaff32a 3091 "output file name"),
69e7e5b0
AH		 3092 OPT_BOOLEAN_SET('i', "no-inherit", &record.opts.no_inherit,
3093 &record.opts.no_inherit_set,
3094 "child tasks do not inherit counters"),
4ea648ae
WN		 3095 OPT_BOOLEAN(0, "tail-synthesize", &record.opts.tail_synthesize,
3096 "synthesize non-sample events at the end of output"),
626a6b78 3097 OPT_BOOLEAN(0, "overwrite", &record.opts.overwrite, "use overwrite mode"),
a060c1f1 3098 OPT_BOOLEAN(0, "no-bpf-event", &record.opts.no_bpf_event, "do not record bpf events"),
b09c2364
ACM		 3099 OPT_BOOLEAN(0, "strict-freq", &record.opts.strict_freq,
3100 "Fail if the specified frequency can't be used"),
67230479
ACM		 3101 OPT_CALLBACK('F', "freq", &record.opts, "freq or 'max'",
3102 "profile at this frequency",
3103 record__parse_freq),
e9db1310
AH		 3104 OPT_CALLBACK('m', "mmap-pages", &record.opts, "pages[,pages]",
3105 "number of mmap data pages and AUX area tracing mmap pages",
3106 record__parse_mmap_pages),
470530bb
AB		 3107 OPT_CALLBACK(0, "mmap-flush", &record.opts, "number",
3108 "Minimal number of bytes that is extracted from mmap data pages (default: 1)",
3109 record__mmap_flush_parse),
d20deb64 3110 OPT_BOOLEAN(0, "group", &record.opts.group,
43bece79 3111 "put the counters into a counter group"),
2ddd5c04 3112 OPT_CALLBACK_NOOPT('g', NULL, &callchain_param,
09b0fd45
JO		 3113 NULL, "enables call-graph recording" ,
3114 &record_callchain_opt),
3115 OPT_CALLBACK(0, "call-graph", &record.opts,
76a26549 3116 "record_mode[,record_size]", record_callchain_help,
09b0fd45 3117 &record_parse_callchain_opt),
c0555642 3118 OPT_INCR('v', "verbose", &verbose,
3da297a6 3119 "be more verbose (show counter open errors, etc)"),
b44308f5 3120 OPT_BOOLEAN('q', "quiet", &quiet, "don't print any message"),
d20deb64 3121 OPT_BOOLEAN('s', "stat", &record.opts.inherit_stat,
649c48a9 3122 "per thread counts"),
56100321 3123 OPT_BOOLEAN('d', "data", &record.opts.sample_address, "Record the sample addresses"),
3b0a5daa
KL		 3124 OPT_BOOLEAN(0, "phys-data", &record.opts.sample_phys_addr,
3125 "Record the sample physical addresses"),
542b88fd
KL		 3126 OPT_BOOLEAN(0, "data-page-size", &record.opts.sample_data_page_size,
3127 "Record the sampled data address data page size"),
c1de7f3d
KL		 3128 OPT_BOOLEAN(0, "code-page-size", &record.opts.sample_code_page_size,
3129 "Record the sampled code address (ip) page size"),
b6f35ed7 3130 OPT_BOOLEAN(0, "sample-cpu", &record.opts.sample_cpu, "Record the sample cpu"),
3abebc55
AH		 3131 OPT_BOOLEAN_SET('T', "timestamp", &record.opts.sample_time,
3132 &record.opts.sample_time_set,
3133 "Record the sample timestamps"),
f290aa1f
JO		 3134 OPT_BOOLEAN_SET('P', "period", &record.opts.period, &record.opts.period_set,
3135 "Record the sample period"),
d20deb64 3136 OPT_BOOLEAN('n', "no-samples", &record.opts.no_samples,
649c48a9 3137 "don't sample"),
d2db9a98
WN		 3138 OPT_BOOLEAN_SET('N', "no-buildid-cache", &record.no_buildid_cache,
3139 &record.no_buildid_cache_set,
3140 "do not update the buildid cache"),
3141 OPT_BOOLEAN_SET('B', "no-buildid", &record.no_buildid,
3142 &record.no_buildid_set,
3143 "do not collect buildids in perf.data"),
d20deb64 3144 OPT_CALLBACK('G', "cgroup", &record.evlist, "name",
023695d9
SE		 3145 "monitor event in cgroup name only",
3146 parse_cgroups),
68cd3b45
AB		 3147 OPT_INTEGER('D', "delay", &record.opts.initial_delay,
3148 "ms to wait before starting measurement after program start (-1: start with events disabled)"),
eeb399b5 3149 OPT_BOOLEAN(0, "kcore", &record.opts.kcore, "copy /proc/kcore"),
bea03405
NK		 3150 OPT_STRING('u', "uid", &record.opts.target.uid_str, "user",
3151 "user to profile"),
a5aabdac
SE		 3152
3153 OPT_CALLBACK_NOOPT('b', "branch-any", &record.opts.branch_stack,
3154 "branch any", "sample any taken branches",
3155 parse_branch_stack),
3156
3157 OPT_CALLBACK('j', "branch-filter", &record.opts.branch_stack,
3158 "branch filter mask", "branch stack filter modes",
bdfebd84 3159 parse_branch_stack),
05484298
AK		 3160 OPT_BOOLEAN('W', "weight", &record.opts.sample_weight,
3161 "sample by weight (on special events only)"),
475eeab9
AK		 3162 OPT_BOOLEAN(0, "transaction", &record.opts.sample_transaction,
3163 "sample transaction flags (special events only)"),
3aa5939d
AH		 3164 OPT_BOOLEAN(0, "per-thread", &record.opts.target.per_thread,
3165 "use per-thread mmaps"),
bcc84ec6
SE		 3166 OPT_CALLBACK_OPTARG('I', "intr-regs", &record.opts.sample_intr_regs, NULL, "any register",
3167 "sample selected machine registers on interrupt,"
aeea9062 3168 " use '-I?' to list register names", parse_intr_regs),
84c41742
AK		 3169 OPT_CALLBACK_OPTARG(0, "user-regs", &record.opts.sample_user_regs, NULL, "any register",
3170 "sample selected machine registers on interrupt,"
aeea9062 3171 " use '--user-regs=?' to list register names", parse_user_regs),
85c273d2
AK		 3172 OPT_BOOLEAN(0, "running-time", &record.opts.running_time,
3173 "Record running/enabled time of read (:S) events"),
814c8c38
PZ		 3174 OPT_CALLBACK('k', "clockid", &record.opts,
3175 "clockid", "clockid to use for events, see clock_gettime()",
3176 parse_clockid),
2dd6d8a1
AH		 3177 OPT_STRING_OPTARG('S', "snapshot", &record.opts.auxtrace_snapshot_opts,
3178 "opts", "AUX area tracing Snapshot Mode", ""),
c0a6de06
AH		 3179 OPT_STRING_OPTARG(0, "aux-sample", &record.opts.auxtrace_sample_opts,
3180 "opts", "sample AUX area", ""),
3fcb10e4 3181 OPT_UINTEGER(0, "proc-map-timeout", &proc_map_timeout,
9d9cad76 3182 "per thread proc mmap processing timeout in ms"),
f3b3614a
HB		 3183 OPT_BOOLEAN(0, "namespaces", &record.opts.record_namespaces,
3184 "Record namespaces events"),
8fb4b679
NK		 3185 OPT_BOOLEAN(0, "all-cgroups", &record.opts.record_cgroup,
3186 "Record cgroup events"),
16b4b4e1
AH		 3187 OPT_BOOLEAN_SET(0, "switch-events", &record.opts.record_switch_events,
3188 &record.opts.record_switch_events_set,
3189 "Record context switch events"),
85723885
JO		 3190 OPT_BOOLEAN_FLAG(0, "all-kernel", &record.opts.all_kernel,
3191 "Configure all used events to run in kernel space.",
3192 PARSE_OPT_EXCLUSIVE),
3193 OPT_BOOLEAN_FLAG(0, "all-user", &record.opts.all_user,
3194 "Configure all used events to run in user space.",
3195 PARSE_OPT_EXCLUSIVE),
53651b28 3196 OPT_BOOLEAN(0, "kernel-callchains", &record.opts.kernel_callchains,
3197 "collect kernel callchains"),
3198 OPT_BOOLEAN(0, "user-callchains", &record.opts.user_callchains,
3199 "collect user callchains"),
71dc2326
WN		 3200 OPT_STRING(0, "clang-path", &llvm_param.clang_path, "clang path",
3201 "clang binary to use for compiling BPF scriptlets"),
3202 OPT_STRING(0, "clang-opt", &llvm_param.clang_opt, "clang options",
3203 "options passed to clang when compiling BPF scriptlets"),
7efe0e03
HK		 3204 OPT_STRING(0, "vmlinux", &symbol_conf.vmlinux_name,
3205 "file", "vmlinux pathname"),
6156681b
NK		 3206 OPT_BOOLEAN(0, "buildid-all", &record.buildid_all,
3207 "Record build-id of all DSOs regardless of hits"),
e29386c8
JO		 3208 OPT_BOOLEAN(0, "buildid-mmap", &record.buildid_mmap,
3209 "Record build-id in map events"),
ecfd7a9c
WN		 3210 OPT_BOOLEAN(0, "timestamp-filename", &record.timestamp_filename,
3211 "append timestamp to output filename"),
68588baf
JY		 3212 OPT_BOOLEAN(0, "timestamp-boundary", &record.timestamp_boundary,
3213 "Record timestamp boundary (time of first/last samples)"),
cb4e1ebb 3214 OPT_STRING_OPTARG_SET(0, "switch-output", &record.switch_output.str,
c38dab7d
AK		 3215 &record.switch_output.set, "signal or size[BKMG] or time[smhd]",
3216 "Switch output when receiving SIGUSR2 (signal) or cross a size or time threshold",
dc0c6127 3217 "signal"),
899e5ffb
ACM		 3218 OPT_CALLBACK_SET(0, "switch-output-event", &record.sb_evlist, &record.switch_output_event_set, "switch output event",
3219 "switch output event selector. use 'perf list' to list available events",
3220 parse_events_option_new_evlist),
03724b2e
AK		 3221 OPT_INTEGER(0, "switch-max-files", &record.switch_output.num_files,
3222 "Limit number of switch output generated files"),
0aab2136
WN		 3223 OPT_BOOLEAN(0, "dry-run", &dry_run,
3224 "Parse options then exit"),
d3d1af6f 3225#ifdef HAVE_AIO_SUPPORT
93f20c0f
AB		 3226 OPT_CALLBACK_OPTARG(0, "aio", &record.opts,
3227 &nr_cblocks_default, "n", "Use <n> control blocks in asynchronous trace writing mode (default: 1, max: 4)",
d3d1af6f
AB		 3228 record__aio_parse),
3229#endif
f4fe11b7
AB		 3230 OPT_CALLBACK(0, "affinity", &record.opts, "node|cpu",
3231 "Set affinity mask of trace reading thread to NUMA node cpu mask or cpu of processed mmap buffer",
3232 record__parse_affinity),
504c1ad1
AB		 3233#ifdef HAVE_ZSTD_SUPPORT
3234 OPT_CALLBACK_OPTARG('z', "compression-level", &record.opts, &comp_level_default,
3235 "n", "Compressed records using specified level (default: 1 - fastest compression, 22 - greatest compression)",
3236 record__parse_comp_level),
3237#endif
6d575816
JS		 3238 OPT_CALLBACK(0, "max-size", &record.output_max_size,
3239 "size", "Limit the maximum size of the output file", parse_output_max_size),
d99c22ea
SE		 3240 OPT_UINTEGER(0, "num-thread-synthesize",
3241 &record.opts.nr_threads_synthesize,
3242 "number of threads to run for event synthesis"),
70943490
SE		 3243#ifdef HAVE_LIBPFM
3244 OPT_CALLBACK(0, "pfm-events", &record.evlist, "event",
3245 "libpfm4 event selector. use 'perf list' to list available events",
3246 parse_libpfm_events_option),
3247#endif
a8fcbd26 3248 OPT_CALLBACK(0, "control", &record.opts, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]",
d20aff15
AH		 3249 "Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events,\n"
3250 "\t\t\t 'snapshot': AUX area tracing snapshot).\n"
a8fcbd26
AH		 3251 "\t\t\t Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n"
3252 "\t\t\t Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.",
1d078ccb 3253 parse_control_option),
41b740b6
NK		 3254 OPT_CALLBACK(0, "synth", &record.opts, "no|all|task|mmap|cgroup",
3255 "Fine-tune event synthesis: default=all", parse_record_synth_option),
9bce13ea
JO		 3256 OPT_STRING_OPTARG_SET(0, "debuginfod", &record.debuginfod.urls,
3257 &record.debuginfod.set, "debuginfod urls",
3258 "Enable debuginfod data retrieval from DEBUGINFOD_URLS or specified urls",
3259 "system"),
06380a84
AB		 3260 OPT_CALLBACK_OPTARG(0, "threads", &record.opts, NULL, "spec",
3261 "write collected trace data into several data files using parallel threads",
3262 record__parse_threads),
0e9b20b8
IM		 3263 OPT_END()
3264};
3265
e5b2c207
NK		 3266struct option *record_options = __record_options;
3267
7954f716
AB		 3268static void record__mmap_cpu_mask_init(struct mmap_cpu_mask *mask, struct perf_cpu_map *cpus)
3269{
3270 int c;
3271
3272 for (c = 0; c < cpus->nr; c++)
3273 set_bit(cpus->map[c].cpu, mask->bits);
3274}
3275
3276static void record__free_thread_masks(struct record *rec, int nr_threads)
3277{
3278 int t;
3279
3280 if (rec->thread_masks)
3281 for (t = 0; t < nr_threads; t++)
3282 record__thread_mask_free(&rec->thread_masks[t]);
3283
3284 zfree(&rec->thread_masks);
3285}
3286
3287static int record__alloc_thread_masks(struct record *rec, int nr_threads, int nr_bits)
3288{
3289 int t, ret;
3290
3291 rec->thread_masks = zalloc(nr_threads * sizeof(*(rec->thread_masks)));
3292 if (!rec->thread_masks) {
3293 pr_err("Failed to allocate thread masks\n");
3294 return -ENOMEM;
3295 }
3296
3297 for (t = 0; t < nr_threads; t++) {
3298 ret = record__thread_mask_alloc(&rec->thread_masks[t], nr_bits);
3299 if (ret) {
3300 pr_err("Failed to allocate thread masks[%d]\n", t);
3301 goto out_free;
3302 }
3303 }
3304
3305 return 0;
3306
3307out_free:
3308 record__free_thread_masks(rec, nr_threads);
3309
3310 return ret;
3311}
3312
06380a84
AB		 3313static int record__init_thread_cpu_masks(struct record *rec, struct perf_cpu_map *cpus)
3314{
3315 int t, ret, nr_cpus = perf_cpu_map__nr(cpus);
3316
3317 ret = record__alloc_thread_masks(rec, nr_cpus, cpu__max_cpu().cpu);
3318 if (ret)
3319 return ret;
3320
3321 rec->nr_threads = nr_cpus;
3322 pr_debug("nr_threads: %d\n", rec->nr_threads);
3323
3324 for (t = 0; t < rec->nr_threads; t++) {
3325 set_bit(cpus->map[t].cpu, rec->thread_masks[t].maps.bits);
3326 set_bit(cpus->map[t].cpu, rec->thread_masks[t].affinity.bits);
3327 if (verbose) {
3328 pr_debug("thread_masks[%d]: ", t);
3329 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].maps, "maps");
3330 pr_debug("thread_masks[%d]: ", t);
3331 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].affinity, "affinity");
3332 }
3333 }
3334
3335 return 0;
3336}
3337
7954f716
AB		 3338static int record__init_thread_default_masks(struct record *rec, struct perf_cpu_map *cpus)
3339{
3340 int ret;
3341
3342 ret = record__alloc_thread_masks(rec, 1, cpu__max_cpu().cpu);
3343 if (ret)
3344 return ret;
3345
3346 record__mmap_cpu_mask_init(&rec->thread_masks->maps, cpus);
3347
3348 rec->nr_threads = 1;
3349
3350 return 0;
3351}
3352
3353static int record__init_thread_masks(struct record *rec)
3354{
3355 struct perf_cpu_map *cpus = rec->evlist->core.cpus;
3356
06380a84
AB		 3357 if (!record__threads_enabled(rec))
3358 return record__init_thread_default_masks(rec, cpus);
3359
3360 return record__init_thread_cpu_masks(rec, cpus);
7954f716
AB		 3361}
3362
b0ad8ea6 3363int cmd_record(int argc, const char **argv)
0e9b20b8 3364{
ef149c25 3365 int err;
8c6f45a7 3366 struct record *rec = &record;
16ad2ffb 3367 char errbuf[BUFSIZ];
0e9b20b8 3368
67230479
ACM		 3369 setlocale(LC_ALL, "");
3370
48e1cab1
WN		 3371#ifndef HAVE_LIBBPF_SUPPORT
3372# define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, "NO_LIBBPF=1", c)
3373 set_nobuild('\0', "clang-path", true);
3374 set_nobuild('\0', "clang-opt", true);
3375# undef set_nobuild
7efe0e03
HK		 3376#endif
3377
3378#ifndef HAVE_BPF_PROLOGUE
3379# if !defined (HAVE_DWARF_SUPPORT)
3380# define REASON "NO_DWARF=1"
3381# elif !defined (HAVE_LIBBPF_SUPPORT)
3382# define REASON "NO_LIBBPF=1"
3383# else
3384# define REASON "this architecture doesn't support BPF prologue"
3385# endif
3386# define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, REASON, c)
3387 set_nobuild('\0', "vmlinux", true);
3388# undef set_nobuild
3389# undef REASON
48e1cab1
WN		 3390#endif
3391
9d2ed645
AB		 3392 rec->opts.affinity = PERF_AFFINITY_SYS;
3393
0f98b11c 3394 rec->evlist = evlist__new();
3e2be2da 3395 if (rec->evlist == NULL)
361c99a6
ACM		 3396 return -ENOMEM;
3397
ecc4c561
ACM		 3398 err = perf_config(perf_record_config, rec);
3399 if (err)
3400 return err;
eb853e80 3401
bca647aa 3402 argc = parse_options(argc, argv, record_options, record_usage,
655000e7 3403 PARSE_OPT_STOP_AT_NON_OPTION);
68ba3235
NK		 3404 if (quiet)
3405 perf_quiet_option();
483635a9 3406
7cc72553
JC		 3407 err = symbol__validate_sym_arguments();
3408 if (err)
3409 return err;
3410
9bce13ea
JO		 3411 perf_debuginfod_setup(&record.debuginfod);
3412
483635a9 3413 /* Make system wide (-a) the default target. */
602ad878 3414 if (!argc && target__none(&rec->opts.target))
483635a9 3415 rec->opts.target.system_wide = true;
0e9b20b8 3416
bea03405 3417 if (nr_cgroups && !rec->opts.target.system_wide) {
c7118369
NK		 3418 usage_with_options_msg(record_usage, record_options,
3419 "cgroup monitoring only available in system-wide mode");
3420
023695d9 3421 }
504c1ad1 3422
e29386c8
JO		 3423 if (rec->buildid_mmap) {
3424 if (!perf_can_record_build_id()) {
3425 pr_err("Failed: no support to record build id in mmap events, update your kernel.\n");
3426 err = -EINVAL;
3427 goto out_opts;
3428 }
3429 pr_debug("Enabling build id in mmap2 events.\n");
3430 /* Enable mmap build id synthesizing. */
3431 symbol_conf.buildid_mmap2 = true;
3432 /* Enable perf_event_attr::build_id bit. */
3433 rec->opts.build_id = true;
3434 /* Disable build id cache. */
3435 rec->no_buildid = true;
3436 }
3437
4f2abe91
NK		 3438 if (rec->opts.record_cgroup && !perf_can_record_cgroup()) {
3439 pr_err("Kernel has no cgroup sampling support.\n");
3440 err = -EINVAL;
3441 goto out_opts;
3442 }
3443
56f735ff 3444 if (rec->opts.kcore || record__threads_enabled(rec))
eeb399b5
AH		 3445 rec->data.is_dir = true;
3446
504c1ad1
AB		 3447 if (rec->opts.comp_level != 0) {
3448 pr_debug("Compression enabled, disabling build id collection at the end of the session.\n");
3449 rec->no_buildid = true;
3450 }
3451
b757bb09
AH		 3452 if (rec->opts.record_switch_events &&
3453 !perf_can_record_switch_events()) {
c7118369
NK		 3454 ui__error("kernel does not support recording context switch events\n");
3455 parse_options_usage(record_usage, record_options, "switch-events", 0);
a8fcbd26
AH		 3456 err = -EINVAL;
3457 goto out_opts;
b757bb09 3458 }
023695d9 3459
cb4e1ebb
JO		 3460 if (switch_output_setup(rec)) {
3461 parse_options_usage(record_usage, record_options, "switch-output", 0);
a8fcbd26
AH		 3462 err = -EINVAL;
3463 goto out_opts;
cb4e1ebb
JO		 3464 }
3465
bfacbe3b
JO		 3466 if (rec->switch_output.time) {
3467 signal(SIGALRM, alarm_sig_handler);
3468 alarm(rec->switch_output.time);
3469 }
3470
03724b2e
AK		 3471 if (rec->switch_output.num_files) {
3472 rec->switch_output.filenames = calloc(sizeof(char *),
3473 rec->switch_output.num_files);
a8fcbd26
AH		 3474 if (!rec->switch_output.filenames) {
3475 err = -EINVAL;
3476 goto out_opts;
3477 }
03724b2e
AK		 3478 }
3479
1b36c03e
AH		 3480 /*
3481 * Allow aliases to facilitate the lookup of symbols for address
3482 * filters. Refer to auxtrace_parse_filters().
3483 */
3484 symbol_conf.allow_aliases = true;
3485
3486 symbol__init(NULL);
3487
4b5ea3bd 3488 err = record__auxtrace_init(rec);
1b36c03e
AH		 3489 if (err)
3490 goto out;
3491
0aab2136 3492 if (dry_run)
5c01ad60 3493 goto out;
0aab2136 3494
d7888573
WN		 3495 err = bpf__setup_stdout(rec->evlist);
3496 if (err) {
3497 bpf__strerror_setup_stdout(rec->evlist, err, errbuf, sizeof(errbuf));
3498 pr_err("ERROR: Setup BPF stdout failed: %s\n",
3499 errbuf);
5c01ad60 3500 goto out;
d7888573
WN		 3501 }
3502
ef149c25
AH		 3503 err = -ENOMEM;
3504
0c1d46a8 3505 if (rec->no_buildid_cache || rec->no_buildid) {
a1ac1d3c 3506 disable_buildid_cache();
dc0c6127 3507 } else if (rec->switch_output.enabled) {
0c1d46a8
WN		 3508 /*
3509 * In 'perf record --switch-output', disable buildid
3510 * generation by default to reduce data file switching
3511 * overhead. Still generate buildid if they are required
3512 * explicitly using
3513 *
60437ac0 3514 * perf record --switch-output --no-no-buildid \
0c1d46a8
WN		 3515 * --no-no-buildid-cache
3516 *
3517 * Following code equals to:
3518 *
3519 * if ((rec->no_buildid || !rec->no_buildid_set) &&
3520 * (rec->no_buildid_cache || !rec->no_buildid_cache_set))
3521 * disable_buildid_cache();
3522 */
3523 bool disable = true;
3524
3525 if (rec->no_buildid_set && !rec->no_buildid)
3526 disable = false;
3527 if (rec->no_buildid_cache_set && !rec->no_buildid_cache)
3528 disable = false;
3529 if (disable) {
3530 rec->no_buildid = true;
3531 rec->no_buildid_cache = true;
3532 disable_buildid_cache();
3533 }
3534 }
655000e7 3535
4ea648ae
WN		 3536 if (record.opts.overwrite)
3537 record.opts.tail_synthesize = true;
3538
b53a0755
JY		 3539 if (rec->evlist->core.nr_entries == 0) {
3540 if (perf_pmu__has_hybrid()) {
3541 err = evlist__add_default_hybrid(rec->evlist,
3542 !record.opts.no_samples);
3543 } else {
3544 err = __evlist__add_default(rec->evlist,
3545 !record.opts.no_samples);
3546 }
3547
3548 if (err < 0) {
3549 pr_err("Not enough memory for event selector list\n");
3550 goto out;
3551 }
bbd36e5e 3552 }
0e9b20b8 3553
69e7e5b0
AH		 3554 if (rec->opts.target.tid && !rec->opts.no_inherit_set)
3555 rec->opts.no_inherit = true;
3556
602ad878 3557 err = target__validate(&rec->opts.target);
16ad2ffb 3558 if (err) {
602ad878 3559 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
c3dec27b 3560 ui__warning("%s\n", errbuf);
16ad2ffb
NK		 3561 }
3562
602ad878 3563 err = target__parse_uid(&rec->opts.target);
16ad2ffb
NK		 3564 if (err) {
3565 int saved_errno = errno;
4bd0f2d2 3566
602ad878 3567 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
3780f488 3568 ui__error("%s", errbuf);
16ad2ffb
NK		 3569
3570 err = -saved_errno;
394c01ed 3571 goto out;
16ad2ffb 3572 }
0d37aa34 3573
ca800068
MZ		 3574 /* Enable ignoring missing threads when -u/-p option is defined. */
3575 rec->opts.ignore_missing_thread = rec->opts.target.uid != UINT_MAX || rec->opts.target.pid;
23dc4f15 3576
1d3351e6
JY		 3577 if (evlist__fix_hybrid_cpus(rec->evlist, rec->opts.target.cpu_list)) {
3578 pr_err("failed to use cpu list %s\n",
3579 rec->opts.target.cpu_list);
3580 goto out;
3581 }
3582
3583 rec->opts.target.hybrid = perf_pmu__has_hybrid();
7248e308
AT		 3584
3585 if (callchain_param.enabled && callchain_param.record_mode == CALLCHAIN_FP)
3586 arch__add_leaf_frame_record_opts(&rec->opts);
3587
16ad2ffb 3588 err = -ENOMEM;
7748bb71 3589 if (evlist__create_maps(rec->evlist, &rec->opts.target) < 0)
dd7927f4 3590 usage_with_options(record_usage, record_options);
69aad6f1 3591
ef149c25
AH		 3592 err = auxtrace_record__options(rec->itr, rec->evlist, &rec->opts);
3593 if (err)
394c01ed 3594 goto out;
ef149c25 3595
6156681b
NK		 3596 /*
3597 * We take all buildids when the file contains
3598 * AUX area tracing data because we do not decode the
3599 * trace because it would take too long.
3600 */
3601 if (rec->opts.full_auxtrace)
3602 rec->buildid_all = true;
3603
246eba8e
AH		 3604 if (rec->opts.text_poke) {
3605 err = record__config_text_poke(rec->evlist);
3606 if (err) {
3607 pr_err("record__config_text_poke failed, error %d\n", err);
3608 goto out;
3609 }
3610 }
3611
b4006796 3612 if (record_opts__config(&rec->opts)) {
39d17dac 3613 err = -EINVAL;
394c01ed 3614 goto out;
7e4ff9e3
MG		 3615 }
3616
7954f716
AB		 3617 err = record__init_thread_masks(rec);
3618 if (err) {
3619 pr_err("Failed to initialize parallel data streaming masks\n");
3620 goto out;
3621 }
3622
93f20c0f
AB		 3623 if (rec->opts.nr_cblocks > nr_cblocks_max)
3624 rec->opts.nr_cblocks = nr_cblocks_max;
5d7f4116 3625 pr_debug("nr_cblocks: %d\n", rec->opts.nr_cblocks);
d3d1af6f 3626
9d2ed645 3627 pr_debug("affinity: %s\n", affinity_tags[rec->opts.affinity]);
470530bb 3628 pr_debug("mmap flush: %d\n", rec->opts.mmap_flush);
9d2ed645 3629
51255a8a
AB		 3630 if (rec->opts.comp_level > comp_level_max)
3631 rec->opts.comp_level = comp_level_max;
3632 pr_debug("comp level: %d\n", rec->opts.comp_level);
3633
d20deb64 3634 err = __cmd_record(&record, argc, argv);
394c01ed 3635out:
c12995a5 3636 evlist__delete(rec->evlist);
d65a458b 3637 symbol__exit();
ef149c25 3638 auxtrace_record__free(rec->itr);
a8fcbd26 3639out_opts:
7954f716
AB		 3640 record__free_thread_masks(rec, rec->nr_threads);
3641 rec->nr_threads = 0;
ee7fe31e 3642 evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close);
39d17dac 3643 return err;
0e9b20b8 3644}
2dd6d8a1
AH		 3645
3646static void snapshot_sig_handler(int sig __maybe_unused)
3647{
dc0c6127
JO		 3648 struct record *rec = &record;
3649
d20aff15 3650 hit_auxtrace_snapshot_trigger(rec);
3c1cb7e3 3651
dc0c6127 3652 if (switch_output_signal(rec))
3c1cb7e3 3653 trigger_hit(&switch_output_trigger);
2dd6d8a1 3654}
bfacbe3b
JO		 3655
3656static void alarm_sig_handler(int sig __maybe_unused)
3657{
3658 struct record *rec = &record;
3659
3660 if (switch_output_time(rec))
3661 trigger_hit(&switch_output_trigger);
3662}
Linux 6.x block layer and io_uring tree(s)