projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
perf evsel: Remove use_uncore_alias
[linux-2.6-block.git] / tools / perf / builtin-record.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
abaff32a 2/*
bf9e1876
IM		 3 * builtin-record.c
4 *
5 * Builtin record command: Record the profile of a workload
6 * (or a CPU, or a PID) into the perf.data output file - for
7 * later analysis via perf report.
abaff32a 8 */
16f762a2 9#include "builtin.h"
bf9e1876 10
6122e4e4 11#include "util/build-id.h"
4b6ab94e 12#include <subcmd/parse-options.h>
e3a23261 13#include <internal/xyarray.h>
8ad8db37 14#include "util/parse-events.h"
41840d21 15#include "util/config.h"
6eda5838 16
8f651eae 17#include "util/callchain.h"
f14d5707 18#include "util/cgroup.h"
7c6a1c65 19#include "util/header.h"
66e274f3 20#include "util/event.h"
361c99a6 21#include "util/evlist.h"
69aad6f1 22#include "util/evsel.h"
8f28827a 23#include "util/debug.h"
e0fcfb08 24#include "util/mmap.h"
49c670b1 25#include "util/mutex.h"
aeb00b1a 26#include "util/target.h"
94c744b6 27#include "util/session.h"
45694aa7 28#include "util/tool.h"
8d06367f 29#include "util/symbol.h"
aeb00b1a 30#include "util/record.h"
a12b51c4 31#include "util/cpumap.h"
fd78260b 32#include "util/thread_map.h"
f5fc1412 33#include "util/data.h"
bcc84ec6 34#include "util/perf_regs.h"
ef149c25 35#include "util/auxtrace.h"
46bc29b9 36#include "util/tsc.h"
f00898f4 37#include "util/parse-branch-options.h"
bcc84ec6 38#include "util/parse-regs-options.h"
40c7d246 39#include "util/perf_api_probe.h"
71dc2326 40#include "util/llvm-utils.h"
8690a2a7 41#include "util/bpf-loader.h"
5f9cf599 42#include "util/trigger.h"
a074865e 43#include "util/perf-hooks.h"
f13de660 44#include "util/cpu-set-sched.h"
ea49e01c 45#include "util/synthetic-events.h"
c5e4027e 46#include "util/time-utils.h"
58db1d6e 47#include "util/units.h"
7b612e29 48#include "util/bpf-event.h"
d99c22ea 49#include "util/util.h"
70943490 50#include "util/pfm.h"
6953beb4 51#include "util/clockid.h"
b53a0755
JY		 52#include "util/pmu-hybrid.h"
53#include "util/evlist-hybrid.h"
edc41a10 54#include "util/off_cpu.h"
d8871ea7 55#include "asm/bug.h"
c1a604df 56#include "perf.h"
f466e5ed 57#include "cputopo.h"
7c6a1c65 58
a43783ae 59#include <errno.h>
fd20e811 60#include <inttypes.h>
67230479 61#include <locale.h>
4208735d 62#include <poll.h>
d99c22ea 63#include <pthread.h>
97124d5e 64#include <unistd.h>
415ccb58
AB		 65#ifndef HAVE_GETTID
66#include <syscall.h>
67#endif
de9ac07b 68#include <sched.h>
9607ad3a 69#include <signal.h>
da231338
AM		 70#ifdef HAVE_EVENTFD_SUPPORT
71#include <sys/eventfd.h>
72#endif
a41794cd 73#include <sys/mman.h>
4208735d 74#include <sys/wait.h>
eeb399b5
AH		 75#include <sys/types.h>
76#include <sys/stat.h>
77#include <fcntl.h>
6ef81c55 78#include <linux/err.h>
8520a98d 79#include <linux/string.h>
0693e680 80#include <linux/time64.h>
d8f9da24 81#include <linux/zalloc.h>
8384a260 82#include <linux/bitmap.h>
d1e325cf 83#include <sys/time.h>
78da39fa 84
1b43b704 85struct switch_output {
dc0c6127 86 bool enabled;
1b43b704 87 bool signal;
dc0c6127 88 unsigned long size;
bfacbe3b 89 unsigned long time;
cb4e1ebb
JO		 90 const char *str;
91 bool set;
03724b2e
AK		 92 char **filenames;
93 int num_files;
94 int cur_file;
1b43b704
JO		 95};
96
7954f716
AB		 97struct thread_mask {
98 struct mmap_cpu_mask maps;
99 struct mmap_cpu_mask affinity;
100};
101
415ccb58
AB		 102struct record_thread {
103 pid_t tid;
104 struct thread_mask *mask;
105 struct {
106 int msg[2];
107 int ack[2];
108 } pipes;
109 struct fdarray pollfd;
110 int ctlfd_pos;
111 int nr_mmaps;
112 struct mmap **maps;
113 struct mmap **overwrite_maps;
114 struct record *rec;
396b626b
AB		 115 unsigned long long samples;
116 unsigned long waking;
ae9c7242 117 u64 bytes_written;
610fbc01
AB		 118 u64 bytes_transferred;
119 u64 bytes_compressed;
415ccb58
AB		 120};
121
396b626b
AB		 122static __thread struct record_thread *thread;
123
1e5de7d9
AB		 124enum thread_msg {
125 THREAD_MSG__UNDEFINED = 0,
126 THREAD_MSG__READY,
127 THREAD_MSG__MAX,
128};
129
130static const char *thread_msg_tags[THREAD_MSG__MAX] = {
131 "UNDEFINED", "READY"
132};
133
06380a84
AB		 134enum thread_spec {
135 THREAD_SPEC__UNDEFINED = 0,
136 THREAD_SPEC__CPU,
f466e5ed
AB		 137 THREAD_SPEC__CORE,
138 THREAD_SPEC__PACKAGE,
139 THREAD_SPEC__NUMA,
140 THREAD_SPEC__USER,
141 THREAD_SPEC__MAX,
142};
143
144static const char *thread_spec_tags[THREAD_SPEC__MAX] = {
145 "undefined", "cpu", "core", "package", "numa", "user"
06380a84
AB		 146};
147
6562c9ac
AH		 148struct pollfd_index_map {
149 int evlist_pollfd_index;
150 int thread_pollfd_index;
151};
152
8c6f45a7 153struct record {
45694aa7 154 struct perf_tool tool;
b4006796 155 struct record_opts opts;
d20deb64 156 u64 bytes_written;
91621be6 157 u64 thread_bytes_written;
8ceb41d7 158 struct perf_data data;
ef149c25 159 struct auxtrace_record *itr;
63503dba 160 struct evlist *evlist;
d20deb64 161 struct perf_session *session;
bc477d79 162 struct evlist *sb_evlist;
899e5ffb 163 pthread_t thread_id;
d20deb64 164 int realtime_prio;
899e5ffb 165 bool switch_output_event_set;
d20deb64 166 bool no_buildid;
d2db9a98 167 bool no_buildid_set;
d20deb64 168 bool no_buildid_cache;
d2db9a98 169 bool no_buildid_cache_set;
6156681b 170 bool buildid_all;
e29386c8 171 bool buildid_mmap;
ecfd7a9c 172 bool timestamp_filename;
68588baf 173 bool timestamp_boundary;
edc41a10 174 bool off_cpu;
1b43b704 175 struct switch_output switch_output;
9f065194 176 unsigned long long samples;
6d575816 177 unsigned long output_max_size; /* = 0: unlimited */
9bce13ea 178 struct perf_debuginfod debuginfod;
7954f716
AB		 179 int nr_threads;
180 struct thread_mask *thread_masks;
415ccb58 181 struct record_thread *thread_data;
6562c9ac
AH		 182 struct pollfd_index_map *index_map;
183 size_t index_map_sz;
184 size_t index_map_cnt;
0f82ebc4 185};
a21ca2ca 186
6d575816
JS		 187static volatile int done;
188
dc0c6127
JO		 189static volatile int auxtrace_record__snapshot_started;
190static DEFINE_TRIGGER(auxtrace_snapshot_trigger);
191static DEFINE_TRIGGER(switch_output_trigger);
192
9d2ed645
AB		 193static const char *affinity_tags[PERF_AFFINITY_MAX] = {
194 "SYS", "NODE", "CPU"
195};
196
415ccb58
AB		 197#ifndef HAVE_GETTID
198static inline pid_t gettid(void)
199{
200 return (pid_t)syscall(__NR_gettid);
201}
202#endif
203
3217e9fe
AB		 204static int record__threads_enabled(struct record *rec)
205{
206 return rec->opts.threads_spec;
207}
208
dc0c6127
JO		 209static bool switch_output_signal(struct record *rec)
210{
211 return rec->switch_output.signal &&
212 trigger_is_ready(&switch_output_trigger);
213}
214
215static bool switch_output_size(struct record *rec)
216{
217 return rec->switch_output.size &&
218 trigger_is_ready(&switch_output_trigger) &&
219 (rec->bytes_written >= rec->switch_output.size);
220}
221
bfacbe3b
JO		 222static bool switch_output_time(struct record *rec)
223{
224 return rec->switch_output.time &&
225 trigger_is_ready(&switch_output_trigger);
226}
227
ae9c7242
AB		 228static u64 record__bytes_written(struct record *rec)
229{
91621be6 230 return rec->bytes_written + rec->thread_bytes_written;
ae9c7242
AB		 231}
232
6d575816
JS		 233static bool record__output_max_size_exceeded(struct record *rec)
234{
235 return rec->output_max_size &&
ae9c7242 236 (record__bytes_written(rec) >= rec->output_max_size);
6d575816
JS		 237}
238
a5830532 239static int record__write(struct record *rec, struct mmap *map __maybe_unused,
ded2b8fe 240 void *bf, size_t size)
f5970550 241{
ded2b8fe
JO		 242 struct perf_data_file *file = &rec->session->data->file;
243
56f735ff
AB		 244 if (map && map->file)
245 file = map->file;
246
ded2b8fe 247 if (perf_data_file__write(file, bf, size) < 0) {
50a9b868
JO		 248 pr_err("failed to write perf data, error: %m\n");
249 return -1;
f5970550 250 }
8d3eca20 251
91621be6 252 if (map && map->file) {
ae9c7242 253 thread->bytes_written += size;
91621be6
YJ		 254 rec->thread_bytes_written += size;
255 } else {
56f735ff 256 rec->bytes_written += size;
91621be6 257 }
dc0c6127 258
6d575816
JS		 259 if (record__output_max_size_exceeded(rec) && !done) {
260 fprintf(stderr, "[ perf record: perf size limit reached (%" PRIu64 " KB),"
261 " stopping session ]\n",
ae9c7242 262 record__bytes_written(rec) >> 10);
6d575816
JS		 263 done = 1;
264 }
265
dc0c6127
JO		 266 if (switch_output_size(rec))
267 trigger_hit(&switch_output_trigger);
268
8d3eca20 269 return 0;
f5970550
PZ		 270}
271
ef781128
AB		 272static int record__aio_enabled(struct record *rec);
273static int record__comp_enabled(struct record *rec);
75f5f1fc
AB		 274static size_t zstd_compress(struct perf_session *session, struct mmap *map,
275 void *dst, size_t dst_size, void *src, size_t src_size);
5d7f4116 276
d3d1af6f
AB		 277#ifdef HAVE_AIO_SUPPORT
278static int record__aio_write(struct aiocb *cblock, int trace_fd,
279 void *buf, size_t size, off_t off)
280{
281 int rc;
282
283 cblock->aio_fildes = trace_fd;
284 cblock->aio_buf = buf;
285 cblock->aio_nbytes = size;
286 cblock->aio_offset = off;
287 cblock->aio_sigevent.sigev_notify = SIGEV_NONE;
288
289 do {
290 rc = aio_write(cblock);
291 if (rc == 0) {
292 break;
293 } else if (errno != EAGAIN) {
294 cblock->aio_fildes = -1;
295 pr_err("failed to queue perf data, error: %m\n");
296 break;
297 }
298 } while (1);
299
300 return rc;
301}
302
a5830532 303static int record__aio_complete(struct mmap *md, struct aiocb *cblock)
d3d1af6f
AB		 304{
305 void *rem_buf;
306 off_t rem_off;
307 size_t rem_size;
308 int rc, aio_errno;
309 ssize_t aio_ret, written;
310
311 aio_errno = aio_error(cblock);
312 if (aio_errno == EINPROGRESS)
313 return 0;
314
315 written = aio_ret = aio_return(cblock);
316 if (aio_ret < 0) {
317 if (aio_errno != EINTR)
318 pr_err("failed to write perf data, error: %m\n");
319 written = 0;
320 }
321
322 rem_size = cblock->aio_nbytes - written;
323
324 if (rem_size == 0) {
325 cblock->aio_fildes = -1;
326 /*
ef781128
AB		 327 * md->refcount is incremented in record__aio_pushfn() for
328 * every aio write request started in record__aio_push() so
329 * decrement it because the request is now complete.
d3d1af6f 330 */
80e53d11 331 perf_mmap__put(&md->core);
d3d1af6f
AB		 332 rc = 1;
333 } else {
334 /*
335 * aio write request may require restart with the
336 * reminder if the kernel didn't write whole
337 * chunk at once.
338 */
339 rem_off = cblock->aio_offset + written;
340 rem_buf = (void *)(cblock->aio_buf + written);
341 record__aio_write(cblock, cblock->aio_fildes,
342 rem_buf, rem_size, rem_off);
343 rc = 0;
344 }
345
346 return rc;
347}
348
a5830532 349static int record__aio_sync(struct mmap *md, bool sync_all)
d3d1af6f 350{
93f20c0f
AB		 351 struct aiocb **aiocb = md->aio.aiocb;
352 struct aiocb *cblocks = md->aio.cblocks;
d3d1af6f 353 struct timespec timeout = { 0, 1000 * 1000 * 1 }; /* 1ms */
93f20c0f 354 int i, do_suspend;
d3d1af6f
AB		 355
356 do {
93f20c0f
AB		 357 do_suspend = 0;
358 for (i = 0; i < md->aio.nr_cblocks; ++i) {
359 if (cblocks[i].aio_fildes == -1 || record__aio_complete(md, &cblocks[i])) {
360 if (sync_all)
361 aiocb[i] = NULL;
362 else
363 return i;
364 } else {
365 /*
366 * Started aio write is not complete yet
367 * so it has to be waited before the
368 * next allocation.
369 */
370 aiocb[i] = &cblocks[i];
371 do_suspend = 1;
372 }
373 }
374 if (!do_suspend)
375 return -1;
d3d1af6f 376
93f20c0f 377 while (aio_suspend((const struct aiocb **)aiocb, md->aio.nr_cblocks, &timeout)) {
d3d1af6f
AB		 378 if (!(errno == EAGAIN || errno == EINTR))
379 pr_err("failed to sync perf data, error: %m\n");
380 }
381 } while (1);
382}
383
ef781128
AB		 384struct record_aio {
385 struct record *rec;
386 void *data;
387 size_t size;
388};
389
a5830532 390static int record__aio_pushfn(struct mmap *map, void *to, void *buf, size_t size)
d3d1af6f 391{
ef781128 392 struct record_aio *aio = to;
d3d1af6f 393
ef781128 394 /*
547740f7 395 * map->core.base data pointed by buf is copied into free map->aio.data[] buffer
ef781128
AB		 396 * to release space in the kernel buffer as fast as possible, calling
397 * perf_mmap__consume() from perf_mmap__push() function.
398 *
399 * That lets the kernel to proceed with storing more profiling data into
400 * the kernel buffer earlier than other per-cpu kernel buffers are handled.
401 *
402 * Coping can be done in two steps in case the chunk of profiling data
403 * crosses the upper bound of the kernel buffer. In this case we first move
404 * part of data from map->start till the upper bound and then the reminder
405 * from the beginning of the kernel buffer till the end of the data chunk.
406 */
407
408 if (record__comp_enabled(aio->rec)) {
75f5f1fc 409 size = zstd_compress(aio->rec->session, NULL, aio->data + aio->size,
bf59b305 410 mmap__mmap_len(map) - aio->size,
ef781128
AB		 411 buf, size);
412 } else {
413 memcpy(aio->data + aio->size, buf, size);
414 }
415
416 if (!aio->size) {
417 /*
418 * Increment map->refcount to guard map->aio.data[] buffer
419 * from premature deallocation because map object can be
420 * released earlier than aio write request started on
421 * map->aio.data[] buffer is complete.
422 *
423 * perf_mmap__put() is done at record__aio_complete()
424 * after started aio request completion or at record__aio_push()
425 * if the request failed to start.
426 */
e75710f0 427 perf_mmap__get(&map->core);
ef781128
AB		 428 }
429
430 aio->size += size;
431
432 return size;
433}
434
a5830532 435static int record__aio_push(struct record *rec, struct mmap *map, off_t *off)
ef781128
AB		 436{
437 int ret, idx;
438 int trace_fd = rec->session->data->file.fd;
439 struct record_aio aio = { .rec = rec, .size = 0 };
d3d1af6f 440
ef781128
AB		 441 /*
442 * Call record__aio_sync() to wait till map->aio.data[] buffer
443 * becomes available after previous aio write operation.
444 */
445
446 idx = record__aio_sync(map, false);
447 aio.data = map->aio.data[idx];
448 ret = perf_mmap__push(map, &aio, record__aio_pushfn);
449 if (ret != 0) /* ret > 0 - no data, ret < 0 - error */
450 return ret;
451
452 rec->samples++;
453 ret = record__aio_write(&(map->aio.cblocks[idx]), trace_fd, aio.data, aio.size, *off);
d3d1af6f 454 if (!ret) {
ef781128
AB		 455 *off += aio.size;
456 rec->bytes_written += aio.size;
d3d1af6f
AB		 457 if (switch_output_size(rec))
458 trigger_hit(&switch_output_trigger);
ef781128
AB		 459 } else {
460 /*
461 * Decrement map->refcount incremented in record__aio_pushfn()
462 * back if record__aio_write() operation failed to start, otherwise
463 * map->refcount is decremented in record__aio_complete() after
464 * aio write operation finishes successfully.
465 */
80e53d11 466 perf_mmap__put(&map->core);
d3d1af6f
AB		 467 }
468
469 return ret;
470}
471
472static off_t record__aio_get_pos(int trace_fd)
473{
474 return lseek(trace_fd, 0, SEEK_CUR);
475}
476
477static void record__aio_set_pos(int trace_fd, off_t pos)
478{
479 lseek(trace_fd, pos, SEEK_SET);
480}
481
482static void record__aio_mmap_read_sync(struct record *rec)
483{
484 int i;
63503dba 485 struct evlist *evlist = rec->evlist;
a5830532 486 struct mmap *maps = evlist->mmap;
d3d1af6f 487
ef781128 488 if (!record__aio_enabled(rec))
d3d1af6f
AB		 489 return;
490
c976ee11 491 for (i = 0; i < evlist->core.nr_mmaps; i++) {
a5830532 492 struct mmap *map = &maps[i];
d3d1af6f 493
547740f7 494 if (map->core.base)
93f20c0f 495 record__aio_sync(map, true);
d3d1af6f
AB		 496 }
497}
498
499static int nr_cblocks_default = 1;
93f20c0f 500static int nr_cblocks_max = 4;
d3d1af6f
AB		 501
502static int record__aio_parse(const struct option *opt,
93f20c0f 503 const char *str,
d3d1af6f
AB		 504 int unset)
505{
506 struct record_opts *opts = (struct record_opts *)opt->value;
507
93f20c0f 508 if (unset) {
d3d1af6f 509 opts->nr_cblocks = 0;
93f20c0f
AB		 510 } else {
511 if (str)
512 opts->nr_cblocks = strtol(str, NULL, 0);
513 if (!opts->nr_cblocks)
514 opts->nr_cblocks = nr_cblocks_default;
515 }
d3d1af6f
AB		 516
517 return 0;
518}
519#else /* HAVE_AIO_SUPPORT */
93f20c0f
AB		 520static int nr_cblocks_max = 0;
521
a5830532 522static int record__aio_push(struct record *rec __maybe_unused, struct mmap *map __maybe_unused,
ef781128 523 off_t *off __maybe_unused)
d3d1af6f
AB		 524{
525 return -1;
526}
527
528static off_t record__aio_get_pos(int trace_fd __maybe_unused)
529{
530 return -1;
531}
532
533static void record__aio_set_pos(int trace_fd __maybe_unused, off_t pos __maybe_unused)
534{
535}
536
537static void record__aio_mmap_read_sync(struct record *rec __maybe_unused)
538{
539}
540#endif
541
542static int record__aio_enabled(struct record *rec)
543{
544 return rec->opts.nr_cblocks > 0;
545}
546
470530bb
AB		 547#define MMAP_FLUSH_DEFAULT 1
548static int record__mmap_flush_parse(const struct option *opt,
549 const char *str,
550 int unset)
551{
552 int flush_max;
553 struct record_opts *opts = (struct record_opts *)opt->value;
554 static struct parse_tag tags[] = {
555 { .tag = 'B', .mult = 1 },
556 { .tag = 'K', .mult = 1 << 10 },
557 { .tag = 'M', .mult = 1 << 20 },
558 { .tag = 'G', .mult = 1 << 30 },
559 { .tag = 0 },
560 };
561
562 if (unset)
563 return 0;
564
565 if (str) {
566 opts->mmap_flush = parse_tag_value(str, tags);
567 if (opts->mmap_flush == (int)-1)
568 opts->mmap_flush = strtol(str, NULL, 0);
569 }
570
571 if (!opts->mmap_flush)
572 opts->mmap_flush = MMAP_FLUSH_DEFAULT;
573
9521b5f2 574 flush_max = evlist__mmap_size(opts->mmap_pages);
470530bb
AB		 575 flush_max /= 4;
576 if (opts->mmap_flush > flush_max)
577 opts->mmap_flush = flush_max;
578
579 return 0;
580}
581
504c1ad1
AB		 582#ifdef HAVE_ZSTD_SUPPORT
583static unsigned int comp_level_default = 1;
584
585static int record__parse_comp_level(const struct option *opt, const char *str, int unset)
586{
587 struct record_opts *opts = opt->value;
588
589 if (unset) {
590 opts->comp_level = 0;
591 } else {
592 if (str)
593 opts->comp_level = strtol(str, NULL, 0);
594 if (!opts->comp_level)
595 opts->comp_level = comp_level_default;
596 }
597
598 return 0;
599}
600#endif
51255a8a
AB		 601static unsigned int comp_level_max = 22;
602
42e1fd80
AB		 603static int record__comp_enabled(struct record *rec)
604{
605 return rec->opts.comp_level > 0;
606}
607
45694aa7 608static int process_synthesized_event(struct perf_tool *tool,
d20deb64 609 union perf_event *event,
1d037ca1
IT		 610 struct perf_sample *sample __maybe_unused,
611 struct machine *machine __maybe_unused)
234fbbf5 612{
8c6f45a7 613 struct record *rec = container_of(tool, struct record, tool);
ded2b8fe 614 return record__write(rec, NULL, event, event->header.size);
234fbbf5
ACM		 615}
616
49c670b1
IR		 617static struct mutex synth_lock;
618
d99c22ea
SE		 619static int process_locked_synthesized_event(struct perf_tool *tool,
620 union perf_event *event,
621 struct perf_sample *sample __maybe_unused,
622 struct machine *machine __maybe_unused)
623{
d99c22ea
SE		 624 int ret;
625
49c670b1 626 mutex_lock(&synth_lock);
d99c22ea 627 ret = process_synthesized_event(tool, event, sample, machine);
49c670b1 628 mutex_unlock(&synth_lock);
d99c22ea
SE		 629 return ret;
630}
631
a5830532 632static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
d37f1586
ACM		 633{
634 struct record *rec = to;
635
5d7f4116 636 if (record__comp_enabled(rec)) {
75f5f1fc 637 size = zstd_compress(rec->session, map, map->data, mmap__mmap_len(map), bf, size);
5d7f4116
AB		 638 bf = map->data;
639 }
640
396b626b 641 thread->samples++;
ded2b8fe 642 return record__write(rec, map, bf, size);
d37f1586
ACM		 643}
644
8ed28c2b
IR		 645static volatile sig_atomic_t signr = -1;
646static volatile sig_atomic_t child_finished;
da231338 647#ifdef HAVE_EVENTFD_SUPPORT
8ed28c2b 648static volatile sig_atomic_t done_fd = -1;
da231338 649#endif
c0bdc1c4 650
2dd6d8a1
AH		 651static void sig_handler(int sig)
652{
653 if (sig == SIGCHLD)
654 child_finished = 1;
655 else
656 signr = sig;
657
658 done = 1;
da231338 659#ifdef HAVE_EVENTFD_SUPPORT
304f0a2f
IR		 660 if (done_fd >= 0) {
661 u64 tmp = 1;
662 int orig_errno = errno;
663
664 /*
665 * It is possible for this signal handler to run after done is
666 * checked in the main loop, but before the perf counter fds are
667 * polled. If this happens, the poll() will continue to wait
668 * even though done is set, and will only break out if either
669 * another signal is received, or the counters are ready for
670 * read. To ensure the poll() doesn't sleep when done is set,
671 * use an eventfd (done_fd) to wake up the poll().
672 */
673 if (write(done_fd, &tmp, sizeof(tmp)) < 0)
674 pr_err("failed to signal wakeup fd, error: %m\n");
675
676 errno = orig_errno;
677 }
da231338 678#endif // HAVE_EVENTFD_SUPPORT
2dd6d8a1
AH		 679}
680
a074865e
WN		 681static void sigsegv_handler(int sig)
682{
683 perf_hooks__recover();
684 sighandler_dump_stack(sig);
685}
686
2dd6d8a1
AH		 687static void record__sig_exit(void)
688{
689 if (signr == -1)
690 return;
691
692 signal(signr, SIG_DFL);
693 raise(signr);
694}
695
e31f0d01
AH		 696#ifdef HAVE_AUXTRACE_SUPPORT
697
ef149c25 698static int record__process_auxtrace(struct perf_tool *tool,
a5830532 699 struct mmap *map,
ef149c25
AH		 700 union perf_event *event, void *data1,
701 size_t len1, void *data2, size_t len2)
702{
703 struct record *rec = container_of(tool, struct record, tool);
8ceb41d7 704 struct perf_data *data = &rec->data;
ef149c25
AH		 705 size_t padding;
706 u8 pad[8] = {0};
707
46e201ef 708 if (!perf_data__is_pipe(data) && perf_data__is_single_file(data)) {
99fa2984 709 off_t file_offset;
8ceb41d7 710 int fd = perf_data__fd(data);
99fa2984
AH		 711 int err;
712
713 file_offset = lseek(fd, 0, SEEK_CUR);
714 if (file_offset == -1)
715 return -1;
716 err = auxtrace_index__auxtrace_event(&rec->session->auxtrace_index,
717 event, file_offset);
718 if (err)
719 return err;
720 }
721
ef149c25
AH		 722 /* event.auxtrace.size includes padding, see __auxtrace_mmap__read() */
723 padding = (len1 + len2) & 7;
724 if (padding)
725 padding = 8 - padding;
726
ded2b8fe
JO		 727 record__write(rec, map, event, event->header.size);
728 record__write(rec, map, data1, len1);
ef149c25 729 if (len2)
ded2b8fe
JO		 730 record__write(rec, map, data2, len2);
731 record__write(rec, map, &pad, padding);
ef149c25
AH		 732
733 return 0;
734}
735
736static int record__auxtrace_mmap_read(struct record *rec,
a5830532 737 struct mmap *map)
ef149c25
AH		 738{
739 int ret;
740
e035f4ca 741 ret = auxtrace_mmap__read(map, rec->itr, &rec->tool,
ef149c25
AH		 742 record__process_auxtrace);
743 if (ret < 0)
744 return ret;
745
746 if (ret)
747 rec->samples++;
748
749 return 0;
750}
751
2dd6d8a1 752static int record__auxtrace_mmap_read_snapshot(struct record *rec,
a5830532 753 struct mmap *map)
2dd6d8a1
AH		 754{
755 int ret;
756
e035f4ca 757 ret = auxtrace_mmap__read_snapshot(map, rec->itr, &rec->tool,
2dd6d8a1
AH		 758 record__process_auxtrace,
759 rec->opts.auxtrace_snapshot_size);
760 if (ret < 0)
761 return ret;
762
763 if (ret)
764 rec->samples++;
765
766 return 0;
767}
768
769static int record__auxtrace_read_snapshot_all(struct record *rec)
770{
771 int i;
772 int rc = 0;
773
c976ee11 774 for (i = 0; i < rec->evlist->core.nr_mmaps; i++) {
a5830532 775 struct mmap *map = &rec->evlist->mmap[i];
2dd6d8a1 776
e035f4ca 777 if (!map->auxtrace_mmap.base)
2dd6d8a1
AH		 778 continue;
779
e035f4ca 780 if (record__auxtrace_mmap_read_snapshot(rec, map) != 0) {
2dd6d8a1
AH		 781 rc = -1;
782 goto out;
783 }
784 }
785out:
786 return rc;
787}
788
ce7b0e42 789static void record__read_auxtrace_snapshot(struct record *rec, bool on_exit)
2dd6d8a1
AH		 790{
791 pr_debug("Recording AUX area tracing snapshot\n");
792 if (record__auxtrace_read_snapshot_all(rec) < 0) {
5f9cf599 793 trigger_error(&auxtrace_snapshot_trigger);
2dd6d8a1 794 } else {
ce7b0e42 795 if (auxtrace_record__snapshot_finish(rec->itr, on_exit))
5f9cf599
WN		 796 trigger_error(&auxtrace_snapshot_trigger);
797 else
798 trigger_ready(&auxtrace_snapshot_trigger);
2dd6d8a1
AH		 799 }
800}
801
ce7b0e42
AS		 802static int record__auxtrace_snapshot_exit(struct record *rec)
803{
804 if (trigger_is_error(&auxtrace_snapshot_trigger))
805 return 0;
806
807 if (!auxtrace_record__snapshot_started &&
808 auxtrace_record__snapshot_start(rec->itr))
809 return -1;
810
811 record__read_auxtrace_snapshot(rec, true);
812 if (trigger_is_error(&auxtrace_snapshot_trigger))
813 return -1;
814
815 return 0;
816}
817
4b5ea3bd
AH		 818static int record__auxtrace_init(struct record *rec)
819{
820 int err;
821
b5f2511d
AB		 822 if ((rec->opts.auxtrace_snapshot_opts || rec->opts.auxtrace_sample_opts)
823 && record__threads_enabled(rec)) {
824 pr_err("AUX area tracing options are not available in parallel streaming mode.\n");
825 return -EINVAL;
826 }
827
4b5ea3bd
AH		 828 if (!rec->itr) {
829 rec->itr = auxtrace_record__init(rec->evlist, &err);
830 if (err)
831 return err;
832 }
833
834 err = auxtrace_parse_snapshot_options(rec->itr, &rec->opts,
835 rec->opts.auxtrace_snapshot_opts);
836 if (err)
837 return err;
838
c0a6de06
AH		 839 err = auxtrace_parse_sample_options(rec->itr, rec->evlist, &rec->opts,
840 rec->opts.auxtrace_sample_opts);
841 if (err)
842 return err;
843
d58b3f7e
AH		 844 auxtrace_regroup_aux_output(rec->evlist);
845
4b5ea3bd
AH		 846 return auxtrace_parse_filters(rec->evlist);
847}
848
e31f0d01
AH		 849#else
850
851static inline
852int record__auxtrace_mmap_read(struct record *rec __maybe_unused,
a5830532 853 struct mmap *map __maybe_unused)
e31f0d01
AH		 854{
855 return 0;
856}
857
2dd6d8a1 858static inline
ce7b0e42
AS		 859void record__read_auxtrace_snapshot(struct record *rec __maybe_unused,
860 bool on_exit __maybe_unused)
de9ac07b 861{
f7b7c26e
PZ		 862}
863
2dd6d8a1
AH		 864static inline
865int auxtrace_record__snapshot_start(struct auxtrace_record *itr __maybe_unused)
f7b7c26e 866{
2dd6d8a1 867 return 0;
de9ac07b
PZ		 868}
869
ce7b0e42
AS		 870static inline
871int record__auxtrace_snapshot_exit(struct record *rec __maybe_unused)
872{
873 return 0;
874}
875
4b5ea3bd
AH		 876static int record__auxtrace_init(struct record *rec __maybe_unused)
877{
878 return 0;
879}
880
2dd6d8a1
AH		 881#endif
882
246eba8e
AH		 883static int record__config_text_poke(struct evlist *evlist)
884{
885 struct evsel *evsel;
246eba8e
AH		 886
887 /* Nothing to do if text poke is already configured */
888 evlist__for_each_entry(evlist, evsel) {
889 if (evsel->core.attr.text_poke)
890 return 0;
891 }
892
921e3be5
AH		 893 evsel = evlist__add_dummy_on_all_cpus(evlist);
894 if (!evsel)
895 return -ENOMEM;
246eba8e 896
246eba8e
AH		 897 evsel->core.attr.text_poke = 1;
898 evsel->core.attr.ksymbol = 1;
246eba8e 899 evsel->immediate = true;
246eba8e
AH		 900 evsel__set_sample_bit(evsel, TIME);
901
902 return 0;
903}
904
edc41a10
NK		 905static int record__config_off_cpu(struct record *rec)
906{
685439a7 907 return off_cpu_prepare(rec->evlist, &rec->opts.target, &rec->opts);
edc41a10
NK		 908}
909
eeb399b5
AH		 910static bool record__kcore_readable(struct machine *machine)
911{
912 char kcore[PATH_MAX];
913 int fd;
914
915 scnprintf(kcore, sizeof(kcore), "%s/proc/kcore", machine->root_dir);
916
917 fd = open(kcore, O_RDONLY);
918 if (fd < 0)
919 return false;
920
921 close(fd);
922
923 return true;
924}
925
926static int record__kcore_copy(struct machine *machine, struct perf_data *data)
927{
928 char from_dir[PATH_MAX];
929 char kcore_dir[PATH_MAX];
930 int ret;
931
932 snprintf(from_dir, sizeof(from_dir), "%s/proc", machine->root_dir);
933
934 ret = perf_data__make_kcore_dir(data, kcore_dir, sizeof(kcore_dir));
935 if (ret)
936 return ret;
937
938 return kcore_copy(from_dir, kcore_dir);
939}
940
415ccb58
AB		 941static void record__thread_data_init_pipes(struct record_thread *thread_data)
942{
943 thread_data->pipes.msg[0] = -1;
944 thread_data->pipes.msg[1] = -1;
945 thread_data->pipes.ack[0] = -1;
946 thread_data->pipes.ack[1] = -1;
947}
948
949static int record__thread_data_open_pipes(struct record_thread *thread_data)
950{
951 if (pipe(thread_data->pipes.msg))
952 return -EINVAL;
953
954 if (pipe(thread_data->pipes.ack)) {
955 close(thread_data->pipes.msg[0]);
956 thread_data->pipes.msg[0] = -1;
957 close(thread_data->pipes.msg[1]);
958 thread_data->pipes.msg[1] = -1;
959 return -EINVAL;
960 }
961
962 pr_debug2("thread_data[%p]: msg=[%d,%d], ack=[%d,%d]\n", thread_data,
963 thread_data->pipes.msg[0], thread_data->pipes.msg[1],
964 thread_data->pipes.ack[0], thread_data->pipes.ack[1]);
965
966 return 0;
967}
968
969static void record__thread_data_close_pipes(struct record_thread *thread_data)
970{
971 if (thread_data->pipes.msg[0] != -1) {
972 close(thread_data->pipes.msg[0]);
973 thread_data->pipes.msg[0] = -1;
974 }
975 if (thread_data->pipes.msg[1] != -1) {
976 close(thread_data->pipes.msg[1]);
977 thread_data->pipes.msg[1] = -1;
978 }
979 if (thread_data->pipes.ack[0] != -1) {
980 close(thread_data->pipes.ack[0]);
981 thread_data->pipes.ack[0] = -1;
982 }
983 if (thread_data->pipes.ack[1] != -1) {
984 close(thread_data->pipes.ack[1]);
985 thread_data->pipes.ack[1] = -1;
986 }
987}
988
7be1fedd
AH		 989static bool evlist__per_thread(struct evlist *evlist)
990{
991 return cpu_map__is_dummy(evlist->core.user_requested_cpus);
992}
993
415ccb58
AB		 994static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist)
995{
996 int m, tm, nr_mmaps = evlist->core.nr_mmaps;
997 struct mmap *mmap = evlist->mmap;
998 struct mmap *overwrite_mmap = evlist->overwrite_mmap;
7be1fedd
AH		 999 struct perf_cpu_map *cpus = evlist->core.all_cpus;
1000 bool per_thread = evlist__per_thread(evlist);
415ccb58 1001
7be1fedd 1002 if (per_thread)
23380e4d
AB		 1003 thread_data->nr_mmaps = nr_mmaps;
1004 else
1005 thread_data->nr_mmaps = bitmap_weight(thread_data->mask->maps.bits,
1006 thread_data->mask->maps.nbits);
415ccb58
AB		 1007 if (mmap) {
1008 thread_data->maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
1009 if (!thread_data->maps)
1010 return -ENOMEM;
1011 }
1012 if (overwrite_mmap) {
1013 thread_data->overwrite_maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
1014 if (!thread_data->overwrite_maps) {
1015 zfree(&thread_data->maps);
1016 return -ENOMEM;
1017 }
1018 }
1019 pr_debug2("thread_data[%p]: nr_mmaps=%d, maps=%p, ow_maps=%p\n", thread_data,
1020 thread_data->nr_mmaps, thread_data->maps, thread_data->overwrite_maps);
1021
1022 for (m = 0, tm = 0; m < nr_mmaps && tm < thread_data->nr_mmaps; m++) {
7be1fedd 1023 if (per_thread ||
0255571a 1024 test_bit(perf_cpu_map__cpu(cpus, m).cpu, thread_data->mask->maps.bits)) {
415ccb58
AB		 1025 if (thread_data->maps) {
1026 thread_data->maps[tm] = &mmap[m];
1027 pr_debug2("thread_data[%p]: cpu%d: maps[%d] -> mmap[%d]\n",
23380e4d 1028 thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
415ccb58
AB		 1029 }
1030 if (thread_data->overwrite_maps) {
1031 thread_data->overwrite_maps[tm] = &overwrite_mmap[m];
1032 pr_debug2("thread_data[%p]: cpu%d: ow_maps[%d] -> ow_mmap[%d]\n",
23380e4d 1033 thread_data, perf_cpu_map__cpu(cpus, m).cpu, tm, m);
415ccb58
AB		 1034 }
1035 tm++;
1036 }
1037 }
1038
1039 return 0;
1040}
1041
1042static int record__thread_data_init_pollfd(struct record_thread *thread_data, struct evlist *evlist)
1043{
1044 int f, tm, pos;
1045 struct mmap *map, *overwrite_map;
1046
1047 fdarray__init(&thread_data->pollfd, 64);
1048
1049 for (tm = 0; tm < thread_data->nr_mmaps; tm++) {
1050 map = thread_data->maps ? thread_data->maps[tm] : NULL;
1051 overwrite_map = thread_data->overwrite_maps ?
1052 thread_data->overwrite_maps[tm] : NULL;
1053
1054 for (f = 0; f < evlist->core.pollfd.nr; f++) {
1055 void *ptr = evlist->core.pollfd.priv[f].ptr;
1056
1057 if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) {
1058 pos = fdarray__dup_entry_from(&thread_data->pollfd, f,
1059 &evlist->core.pollfd);
1060 if (pos < 0)
1061 return pos;
1062 pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n",
1063 thread_data, pos, evlist->core.pollfd.entries[f].fd);
1064 }
1065 }
1066 }
1067
1068 return 0;
1069}
1070
1071static void record__free_thread_data(struct record *rec)
1072{
1073 int t;
1074 struct record_thread *thread_data = rec->thread_data;
1075
1076 if (thread_data == NULL)
1077 return;
1078
1079 for (t = 0; t < rec->nr_threads; t++) {
1080 record__thread_data_close_pipes(&thread_data[t]);
1081 zfree(&thread_data[t].maps);
1082 zfree(&thread_data[t].overwrite_maps);
1083 fdarray__exit(&thread_data[t].pollfd);
1084 }
1085
1086 zfree(&rec->thread_data);
1087}
1088
6562c9ac
AH		 1089static int record__map_thread_evlist_pollfd_indexes(struct record *rec,
1090 int evlist_pollfd_index,
1091 int thread_pollfd_index)
1092{
1093 size_t x = rec->index_map_cnt;
1094
1095 if (realloc_array_as_needed(rec->index_map, rec->index_map_sz, x, NULL))
1096 return -ENOMEM;
1097 rec->index_map[x].evlist_pollfd_index = evlist_pollfd_index;
1098 rec->index_map[x].thread_pollfd_index = thread_pollfd_index;
1099 rec->index_map_cnt += 1;
1100 return 0;
1101}
1102
1103static int record__update_evlist_pollfd_from_thread(struct record *rec,
1104 struct evlist *evlist,
1105 struct record_thread *thread_data)
1106{
1107 struct pollfd *e_entries = evlist->core.pollfd.entries;
1108 struct pollfd *t_entries = thread_data->pollfd.entries;
1109 int err = 0;
1110 size_t i;
1111
1112 for (i = 0; i < rec->index_map_cnt; i++) {
1113 int e_pos = rec->index_map[i].evlist_pollfd_index;
1114 int t_pos = rec->index_map[i].thread_pollfd_index;
1115
1116 if (e_entries[e_pos].fd != t_entries[t_pos].fd ||
1117 e_entries[e_pos].events != t_entries[t_pos].events) {
1118 pr_err("Thread and evlist pollfd index mismatch\n");
1119 err = -EINVAL;
1120 continue;
1121 }
1122 e_entries[e_pos].revents = t_entries[t_pos].revents;
1123 }
1124 return err;
1125}
1126
1127static int record__dup_non_perf_events(struct record *rec,
1128 struct evlist *evlist,
1129 struct record_thread *thread_data)
1130{
1131 struct fdarray *fda = &evlist->core.pollfd;
1132 int i, ret;
1133
1134 for (i = 0; i < fda->nr; i++) {
1135 if (!(fda->priv[i].flags & fdarray_flag__non_perf_event))
1136 continue;
1137 ret = fdarray__dup_entry_from(&thread_data->pollfd, i, fda);
1138 if (ret < 0) {
1139 pr_err("Failed to duplicate descriptor in main thread pollfd\n");
1140 return ret;
1141 }
1142 pr_debug2("thread_data[%p]: pollfd[%d] <- non_perf_event fd=%d\n",
1143 thread_data, ret, fda->entries[i].fd);
1144 ret = record__map_thread_evlist_pollfd_indexes(rec, i, ret);
1145 if (ret < 0) {
1146 pr_err("Failed to map thread and evlist pollfd indexes\n");
1147 return ret;
1148 }
1149 }
1150 return 0;
1151}
1152
415ccb58
AB		 1153static int record__alloc_thread_data(struct record *rec, struct evlist *evlist)
1154{
1155 int t, ret;
1156 struct record_thread *thread_data;
1157
1158 rec->thread_data = zalloc(rec->nr_threads * sizeof(*(rec->thread_data)));
1159 if (!rec->thread_data) {
1160 pr_err("Failed to allocate thread data\n");
1161 return -ENOMEM;
1162 }
1163 thread_data = rec->thread_data;
1164
1165 for (t = 0; t < rec->nr_threads; t++)
1166 record__thread_data_init_pipes(&thread_data[t]);
1167
1168 for (t = 0; t < rec->nr_threads; t++) {
1169 thread_data[t].rec = rec;
1170 thread_data[t].mask = &rec->thread_masks[t];
1171 ret = record__thread_data_init_maps(&thread_data[t], evlist);
1172 if (ret) {
1173 pr_err("Failed to initialize thread[%d] maps\n", t);
1174 goto out_free;
1175 }
1176 ret = record__thread_data_init_pollfd(&thread_data[t], evlist);
1177 if (ret) {
1178 pr_err("Failed to initialize thread[%d] pollfd\n", t);
1179 goto out_free;
1180 }
1181 if (t) {
1182 thread_data[t].tid = -1;
1183 ret = record__thread_data_open_pipes(&thread_data[t]);
1184 if (ret) {
1185 pr_err("Failed to open thread[%d] communication pipes\n", t);
1186 goto out_free;
1187 }
1188 ret = fdarray__add(&thread_data[t].pollfd, thread_data[t].pipes.msg[0],
1189 POLLIN | POLLERR | POLLHUP, fdarray_flag__nonfilterable);
1190 if (ret < 0) {
1191 pr_err("Failed to add descriptor to thread[%d] pollfd\n", t);
1192 goto out_free;
1193 }
1194 thread_data[t].ctlfd_pos = ret;
1195 pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
1196 thread_data, thread_data[t].ctlfd_pos,
1197 thread_data[t].pipes.msg[0]);
1198 } else {
1199 thread_data[t].tid = gettid();
6562c9ac
AH		 1200
1201 ret = record__dup_non_perf_events(rec, evlist, &thread_data[t]);
1202 if (ret < 0)
1203 goto out_free;
1204
feff0b61 1205 thread_data[t].ctlfd_pos = -1; /* Not used */
415ccb58
AB		 1206 }
1207 }
1208
1209 return 0;
1210
1211out_free:
1212 record__free_thread_data(rec);
1213
1214 return ret;
1215}
1216
cda57a8c 1217static int record__mmap_evlist(struct record *rec,
63503dba 1218 struct evlist *evlist)
cda57a8c 1219{
56f735ff 1220 int i, ret;
cda57a8c 1221 struct record_opts *opts = &rec->opts;
c0a6de06
AH		 1222 bool auxtrace_overwrite = opts->auxtrace_snapshot_mode ||
1223 opts->auxtrace_sample_mode;
cda57a8c
WN		 1224 char msg[512];
1225
f13de660
AB		 1226 if (opts->affinity != PERF_AFFINITY_SYS)
1227 cpu__setup_cpunode_map();
1228
9521b5f2 1229 if (evlist__mmap_ex(evlist, opts->mmap_pages,
cda57a8c 1230 opts->auxtrace_mmap_pages,
c0a6de06 1231 auxtrace_overwrite,
470530bb 1232 opts->nr_cblocks, opts->affinity,
51255a8a 1233 opts->mmap_flush, opts->comp_level) < 0) {
cda57a8c
WN		 1234 if (errno == EPERM) {
1235 pr_err("Permission error mapping pages.\n"
1236 "Consider increasing "
1237 "/proc/sys/kernel/perf_event_mlock_kb,\n"
1238 "or try again with a smaller value of -m/--mmap_pages.\n"
1239 "(current value: %u,%u)\n",
1240 opts->mmap_pages, opts->auxtrace_mmap_pages);
1241 return -errno;
1242 } else {
1243 pr_err("failed to mmap with %d (%s)\n", errno,
c8b5f2c9 1244 str_error_r(errno, msg, sizeof(msg)));
cda57a8c
WN		 1245 if (errno)
1246 return -errno;
1247 else
1248 return -EINVAL;
1249 }
1250 }
415ccb58
AB		 1251
1252 if (evlist__initialize_ctlfd(evlist, opts->ctl_fd, opts->ctl_fd_ack))
1253 return -1;
1254
1255 ret = record__alloc_thread_data(rec, evlist);
1256 if (ret)
1257 return ret;
1258
56f735ff
AB		 1259 if (record__threads_enabled(rec)) {
1260 ret = perf_data__create_dir(&rec->data, evlist->core.nr_mmaps);
65e7c963
AB		 1261 if (ret) {
1262 pr_err("Failed to create data directory: %s\n", strerror(-ret));
56f735ff 1263 return ret;
65e7c963 1264 }
56f735ff
AB		 1265 for (i = 0; i < evlist->core.nr_mmaps; i++) {
1266 if (evlist->mmap)
1267 evlist->mmap[i].file = &rec->data.dir.files[i];
1268 if (evlist->overwrite_mmap)
1269 evlist->overwrite_mmap[i].file = &rec->data.dir.files[i];
1270 }
1271 }
1272
cda57a8c
WN		 1273 return 0;
1274}
1275
1276static int record__mmap(struct record *rec)
1277{
1278 return record__mmap_evlist(rec, rec->evlist);
1279}
1280
8c6f45a7 1281static int record__open(struct record *rec)
dd7927f4 1282{
d6195a6a 1283 char msg[BUFSIZ];
32dcd021 1284 struct evsel *pos;
63503dba 1285 struct evlist *evlist = rec->evlist;
d20deb64 1286 struct perf_session *session = rec->session;
b4006796 1287 struct record_opts *opts = &rec->opts;
8d3eca20 1288 int rc = 0;
dd7927f4 1289
d3dbf43c 1290 /*
b91e5492
KL		 1291 * For initial_delay, system wide or a hybrid system, we need to add a
1292 * dummy event so that we can track PERF_RECORD_MMAP to cover the delay
1293 * of waiting or event synthesis.
d3dbf43c 1294 */
cb4b9e68 1295 if (opts->target.initial_delay || target__has_cpu(&opts->target) ||
b91e5492 1296 perf_pmu__has_hybrid()) {
e80db255 1297 pos = evlist__get_tracking_event(evlist);
442ad225
AH		 1298 if (!evsel__is_dummy_event(pos)) {
1299 /* Set up dummy event. */
facbf0b9 1300 if (evlist__add_dummy(evlist))
442ad225
AH		 1301 return -ENOMEM;
1302 pos = evlist__last(evlist);
e80db255 1303 evlist__set_tracking_event(evlist, pos);
442ad225
AH		 1304 }
1305
0a892c1c
IR		 1306 /*
1307 * Enable the dummy event when the process is forked for
1308 * initial_delay, immediately for system wide.
1309 */
cb4b9e68 1310 if (opts->target.initial_delay && !pos->immediate &&
bb07d62e 1311 !target__has_cpu(&opts->target))
0a892c1c
IR		 1312 pos->core.attr.enable_on_exec = 1;
1313 else
1314 pos->immediate = 1;
d3dbf43c
ACM		 1315 }
1316
78e1bc25 1317 evlist__config(evlist, opts, &callchain_param);
cac21425 1318
e5cadb93 1319 evlist__for_each_entry(evlist, pos) {
dd7927f4 1320try_again:
af663bd0 1321 if (evsel__open(pos, pos->core.cpus, pos->core.threads) < 0) {
ae430892 1322 if (evsel__fallback(pos, errno, msg, sizeof(msg))) {
bb963e16 1323 if (verbose > 0)
c0a54341 1324 ui__warning("%s\n", msg);
d6d901c2
ZY		 1325 goto try_again;
1326 }
cf99ad14 1327 if ((errno == EINVAL || errno == EBADF) &&
fba7c866 1328 pos->core.leader != &pos->core &&
cf99ad14 1329 pos->weak_group) {
64b4778b 1330 pos = evlist__reset_weak_group(evlist, pos, true);
cf99ad14
AK		 1331 goto try_again;
1332 }
56e52e85 1333 rc = -errno;
2bb72dbb 1334 evsel__open_strerror(pos, &opts->target, errno, msg, sizeof(msg));
56e52e85 1335 ui__error("%s\n", msg);
8d3eca20 1336 goto out;
c171b552 1337 }
bfd8f72c
AK		 1338
1339 pos->supported = true;
c171b552 1340 }
a43d3f08 1341
78e1bc25 1342 if (symbol_conf.kptr_restrict && !evlist__exclude_kernel(evlist)) {
c8b567c8
ACM		 1343 pr_warning(
1344"WARNING: Kernel address maps (/proc/{kallsyms,modules}) are restricted,\n"
1345"check /proc/sys/kernel/kptr_restrict and /proc/sys/kernel/perf_event_paranoid.\n\n"
1346"Samples in kernel functions may not be resolved if a suitable vmlinux\n"
1347"file is not found in the buildid cache or in the vmlinux path.\n\n"
1348"Samples in kernel modules won't be resolved at all.\n\n"
1349"If some relocation was applied (e.g. kexec) symbols may be misresolved\n"
1350"even with a suitable vmlinux or kallsyms file.\n\n");
1351 }
1352
24bf91a7 1353 if (evlist__apply_filters(evlist, &pos)) {
62d94b00 1354 pr_err("failed to set filter \"%s\" on event %s with %d (%s)\n",
8ab2e96d 1355 pos->filter, evsel__name(pos), errno,
c8b5f2c9 1356 str_error_r(errno, msg, sizeof(msg)));
8d3eca20 1357 rc = -1;
5d8bb1ec
MP		 1358 goto out;
1359 }
1360
cda57a8c
WN		 1361 rc = record__mmap(rec);
1362 if (rc)
8d3eca20 1363 goto out;
0a27d7f9 1364
563aecb2 1365 session->evlist = evlist;
7b56cce2 1366 perf_session__set_id_hdr_size(session);
8d3eca20
DA		 1367out:
1368 return rc;
16c8a109
PZ		 1369}
1370
66286ed3
AH		 1371static void set_timestamp_boundary(struct record *rec, u64 sample_time)
1372{
1373 if (rec->evlist->first_sample_time == 0)
1374 rec->evlist->first_sample_time = sample_time;
1375
1376 if (sample_time)
1377 rec->evlist->last_sample_time = sample_time;
1378}
1379
e3d59112
NK		 1380static int process_sample_event(struct perf_tool *tool,
1381 union perf_event *event,
1382 struct perf_sample *sample,
32dcd021 1383 struct evsel *evsel,
e3d59112
NK		 1384 struct machine *machine)
1385{
1386 struct record *rec = container_of(tool, struct record, tool);
1387
66286ed3 1388 set_timestamp_boundary(rec, sample->time);
e3d59112 1389
68588baf
JY		 1390 if (rec->buildid_all)
1391 return 0;
1392
1393 rec->samples++;
e3d59112
NK		 1394 return build_id__mark_dso_hit(tool, event, sample, evsel, machine);
1395}
1396
8c6f45a7 1397static int process_buildids(struct record *rec)
6122e4e4 1398{
f5fc1412 1399 struct perf_session *session = rec->session;
6122e4e4 1400
45112e89 1401 if (perf_data__size(&rec->data) == 0)
9f591fd7
ACM		 1402 return 0;
1403
00dc8657
NK		 1404 /*
1405 * During this process, it'll load kernel map and replace the
1406 * dso->long_name to a real pathname it found. In this case
1407 * we prefer the vmlinux path like
1408 * /lib/modules/3.16.4/build/vmlinux
1409 *
1410 * rather than build-id path (in debug directory).
1411 * $HOME/.debug/.build-id/f0/6e17aa50adf4d00b88925e03775de107611551
1412 */
1413 symbol_conf.ignore_vmlinux_buildid = true;
1414
6156681b
NK		 1415 /*
1416 * If --buildid-all is given, it marks all DSO regardless of hits,
68588baf
JY		 1417 * so no need to process samples. But if timestamp_boundary is enabled,
1418 * it still needs to walk on all samples to get the timestamps of
1419 * first/last samples.
6156681b 1420 */
68588baf 1421 if (rec->buildid_all && !rec->timestamp_boundary)
6156681b
NK		 1422 rec->tool.sample = NULL;
1423
b7b61cbe 1424 return perf_session__process_events(session);
6122e4e4
ACM		 1425}
1426
8115d60c 1427static void perf_event__synthesize_guest_os(struct machine *machine, void *data)
a1645ce1
ZY		 1428{
1429 int err;
45694aa7 1430 struct perf_tool *tool = data;
a1645ce1
ZY		 1431 /*
1432 *As for guest kernel when processing subcommand record&report,
1433 *we arrange module mmap prior to guest kernel mmap and trigger
1434 *a preload dso because default guest module symbols are loaded
1435 *from guest kallsyms instead of /lib/modules/XXX/XXX. This
1436 *method is used to avoid symbol missing when the first addr is
1437 *in module instead of in guest kernel.
1438 */
45694aa7 1439 err = perf_event__synthesize_modules(tool, process_synthesized_event,
743eb868 1440 machine);
a1645ce1
ZY		 1441 if (err < 0)
1442 pr_err("Couldn't record guest kernel [%d]'s reference"
23346f21 1443 " relocation symbol.\n", machine->pid);
a1645ce1 1444
a1645ce1
ZY		 1445 /*
1446 * We use _stext for guest kernel because guest kernel's /proc/kallsyms
1447 * have no _text sometimes.
1448 */
45694aa7 1449 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
0ae617be 1450 machine);
a1645ce1
ZY		 1451 if (err < 0)
1452 pr_err("Couldn't record guest kernel [%d]'s reference"
23346f21 1453 " relocation symbol.\n", machine->pid);
a1645ce1
ZY		 1454}
1455
98402807
FW		 1456static struct perf_event_header finished_round_event = {
1457 .size = sizeof(struct perf_event_header),
1458 .type = PERF_RECORD_FINISHED_ROUND,
1459};
1460
3812d298
AH		 1461static struct perf_event_header finished_init_event = {
1462 .size = sizeof(struct perf_event_header),
1463 .type = PERF_RECORD_FINISHED_INIT,
1464};
1465
a5830532 1466static void record__adjust_affinity(struct record *rec, struct mmap *map)
f13de660
AB		 1467{
1468 if (rec->opts.affinity != PERF_AFFINITY_SYS &&
396b626b
AB		 1469 !bitmap_equal(thread->mask->affinity.bits, map->affinity_mask.bits,
1470 thread->mask->affinity.nbits)) {
1471 bitmap_zero(thread->mask->affinity.bits, thread->mask->affinity.nbits);
1472 bitmap_or(thread->mask->affinity.bits, thread->mask->affinity.bits,
1473 map->affinity_mask.bits, thread->mask->affinity.nbits);
1474 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
1475 (cpu_set_t *)thread->mask->affinity.bits);
1476 if (verbose == 2) {
1477 pr_debug("threads[%d]: running on cpu%d: ", thread->tid, sched_getcpu());
1478 mmap_cpu_mask__scnprintf(&thread->mask->affinity, "affinity");
1479 }
f13de660
AB		 1480 }
1481}
1482
5d7f4116
AB		 1483static size_t process_comp_header(void *record, size_t increment)
1484{
72932371 1485 struct perf_record_compressed *event = record;
5d7f4116
AB		 1486 size_t size = sizeof(*event);
1487
1488 if (increment) {
1489 event->header.size += increment;
1490 return increment;
1491 }
1492
1493 event->header.type = PERF_RECORD_COMPRESSED;
1494 event->header.size = size;
1495
1496 return size;
1497}
1498
75f5f1fc
AB		 1499static size_t zstd_compress(struct perf_session *session, struct mmap *map,
1500 void *dst, size_t dst_size, void *src, size_t src_size)
5d7f4116
AB		 1501{
1502 size_t compressed;
72932371 1503 size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed) - 1;
75f5f1fc 1504 struct zstd_data *zstd_data = &session->zstd_data;
5d7f4116 1505
75f5f1fc
AB		 1506 if (map && map->file)
1507 zstd_data = &map->zstd_data;
1508
1509 compressed = zstd_compress_stream_to_records(zstd_data, dst, dst_size, src, src_size,
5d7f4116
AB		 1510 max_record_size, process_comp_header);
1511
610fbc01
AB		 1512 if (map && map->file) {
1513 thread->bytes_transferred += src_size;
1514 thread->bytes_compressed += compressed;
1515 } else {
1516 session->bytes_transferred += src_size;
1517 session->bytes_compressed += compressed;
1518 }
5d7f4116
AB		 1519
1520 return compressed;
1521}
1522
63503dba 1523static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist,
470530bb 1524 bool overwrite, bool synch)
98402807 1525{
dcabb507 1526 u64 bytes_written = rec->bytes_written;
0e2e63dd 1527 int i;
8d3eca20 1528 int rc = 0;
396b626b
AB		 1529 int nr_mmaps;
1530 struct mmap **maps;
d3d1af6f 1531 int trace_fd = rec->data.file.fd;
ef781128 1532 off_t off = 0;
98402807 1533
cb21686b
WN		 1534 if (!evlist)
1535 return 0;
ef149c25 1536
396b626b
AB		 1537 nr_mmaps = thread->nr_mmaps;
1538 maps = overwrite ? thread->overwrite_maps : thread->maps;
1539
a4ea0ec4
WN		 1540 if (!maps)
1541 return 0;
1542
0b72d69a 1543 if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING)
54cc54de
WN		 1544 return 0;
1545
d3d1af6f
AB		 1546 if (record__aio_enabled(rec))
1547 off = record__aio_get_pos(trace_fd);
1548
396b626b 1549 for (i = 0; i < nr_mmaps; i++) {
470530bb 1550 u64 flush = 0;
396b626b 1551 struct mmap *map = maps[i];
cb21686b 1552
547740f7 1553 if (map->core.base) {
f13de660 1554 record__adjust_affinity(rec, map);
470530bb 1555 if (synch) {
65aa2e6b
JO		 1556 flush = map->core.flush;
1557 map->core.flush = 1;
470530bb 1558 }
d3d1af6f 1559 if (!record__aio_enabled(rec)) {
ef781128 1560 if (perf_mmap__push(map, rec, record__pushfn) < 0) {
470530bb 1561 if (synch)
65aa2e6b 1562 map->core.flush = flush;
d3d1af6f
AB		 1563 rc = -1;
1564 goto out;
1565 }
1566 } else {
ef781128 1567 if (record__aio_push(rec, map, &off) < 0) {
d3d1af6f 1568 record__aio_set_pos(trace_fd, off);
470530bb 1569 if (synch)
65aa2e6b 1570 map->core.flush = flush;
d3d1af6f
AB		 1571 rc = -1;
1572 goto out;
1573 }
8d3eca20 1574 }
470530bb 1575 if (synch)
65aa2e6b 1576 map->core.flush = flush;
8d3eca20 1577 }
ef149c25 1578
e035f4ca 1579 if (map->auxtrace_mmap.base && !rec->opts.auxtrace_snapshot_mode &&
c0a6de06 1580 !rec->opts.auxtrace_sample_mode &&
e035f4ca 1581 record__auxtrace_mmap_read(rec, map) != 0) {
ef149c25
AH		 1582 rc = -1;
1583 goto out;
1584 }
98402807
FW		 1585 }
1586
d3d1af6f
AB		 1587 if (record__aio_enabled(rec))
1588 record__aio_set_pos(trace_fd, off);
1589
dcabb507
JO		 1590 /*
1591 * Mark the round finished in case we wrote
1592 * at least one event.
56f735ff
AB		 1593 *
1594 * No need for round events in directory mode,
1595 * because per-cpu maps and files have data
1596 * sorted by kernel.
dcabb507 1597 */
56f735ff 1598 if (!record__threads_enabled(rec) && bytes_written != rec->bytes_written)
ded2b8fe 1599 rc = record__write(rec, NULL, &finished_round_event, sizeof(finished_round_event));
8d3eca20 1600
0b72d69a 1601 if (overwrite)
ade9d208 1602 evlist__toggle_bkw_mmap(evlist, BKW_MMAP_EMPTY);
8d3eca20
DA		 1603out:
1604 return rc;
98402807
FW		 1605}
1606
470530bb 1607static int record__mmap_read_all(struct record *rec, bool synch)
cb21686b
WN		 1608{
1609 int err;
1610
470530bb 1611 err = record__mmap_read_evlist(rec, rec->evlist, false, synch);
cb21686b
WN		 1612 if (err)
1613 return err;
1614
470530bb 1615 return record__mmap_read_evlist(rec, rec->evlist, true, synch);
cb21686b
WN		 1616}
1617
396b626b
AB		 1618static void record__thread_munmap_filtered(struct fdarray *fda, int fd,
1619 void *arg __maybe_unused)
1620{
1621 struct perf_mmap *map = fda->priv[fd].ptr;
1622
1623 if (map)
1624 perf_mmap__put(map);
1625}
1626
3217e9fe
AB		 1627static void *record__thread(void *arg)
1628{
1629 enum thread_msg msg = THREAD_MSG__READY;
1630 bool terminate = false;
1631 struct fdarray *pollfd;
1632 int err, ctlfd_pos;
1633
1634 thread = arg;
1635 thread->tid = gettid();
1636
1637 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1638 if (err == -1)
1639 pr_warning("threads[%d]: failed to notify on start: %s\n",
1640 thread->tid, strerror(errno));
1641
1642 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
1643
1644 pollfd = &thread->pollfd;
1645 ctlfd_pos = thread->ctlfd_pos;
1646
1647 for (;;) {
1648 unsigned long long hits = thread->samples;
1649
1650 if (record__mmap_read_all(thread->rec, false) < 0 || terminate)
1651 break;
1652
1653 if (hits == thread->samples) {
1654
1655 err = fdarray__poll(pollfd, -1);
1656 /*
1657 * Propagate error, only if there's any. Ignore positive
1658 * number of returned events and interrupt error.
1659 */
1660 if (err > 0 || (err < 0 && errno == EINTR))
1661 err = 0;
1662 thread->waking++;
1663
1664 if (fdarray__filter(pollfd, POLLERR | POLLHUP,
1665 record__thread_munmap_filtered, NULL) == 0)
1666 break;
1667 }
1668
1669 if (pollfd->entries[ctlfd_pos].revents & POLLHUP) {
1670 terminate = true;
1671 close(thread->pipes.msg[0]);
1672 thread->pipes.msg[0] = -1;
1673 pollfd->entries[ctlfd_pos].fd = -1;
1674 pollfd->entries[ctlfd_pos].events = 0;
1675 }
1676
1677 pollfd->entries[ctlfd_pos].revents = 0;
1678 }
1679 record__mmap_read_all(thread->rec, true);
1680
1681 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1682 if (err == -1)
1683 pr_warning("threads[%d]: failed to notify on termination: %s\n",
1684 thread->tid, strerror(errno));
1685
1686 return NULL;
1687}
1688
8c6f45a7 1689static void record__init_features(struct record *rec)
57706abc 1690{
57706abc
DA		 1691 struct perf_session *session = rec->session;
1692 int feat;
1693
1694 for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
1695 perf_header__set_feat(&session->header, feat);
1696
1697 if (rec->no_buildid)
1698 perf_header__clear_feat(&session->header, HEADER_BUILD_ID);
1699
378ef0f5 1700#ifdef HAVE_LIBTRACEEVENT
ce9036a6 1701 if (!have_tracepoints(&rec->evlist->core.entries))
57706abc 1702 perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
378ef0f5 1703#endif
57706abc
DA		 1704
1705 if (!rec->opts.branch_stack)
1706 perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);
ef149c25
AH		 1707
1708 if (!rec->opts.full_auxtrace)
1709 perf_header__clear_feat(&session->header, HEADER_AUXTRACE);
ffa517ad 1710
cf790516
AB		 1711 if (!(rec->opts.use_clockid && rec->opts.clockid_res_ns))
1712 perf_header__clear_feat(&session->header, HEADER_CLOCKID);
1713
d1e325cf
JO		 1714 if (!rec->opts.use_clockid)
1715 perf_header__clear_feat(&session->header, HEADER_CLOCK_DATA);
1716
56f735ff
AB		 1717 if (!record__threads_enabled(rec))
1718 perf_header__clear_feat(&session->header, HEADER_DIR_FORMAT);
1719
42e1fd80
AB		 1720 if (!record__comp_enabled(rec))
1721 perf_header__clear_feat(&session->header, HEADER_COMPRESSED);
258031c0 1722
ffa517ad 1723 perf_header__clear_feat(&session->header, HEADER_STAT);
57706abc
DA		 1724}
1725
e1ab48ba
WN		 1726static void
1727record__finish_output(struct record *rec)
1728{
56f735ff 1729 int i;
8ceb41d7
JO		 1730 struct perf_data *data = &rec->data;
1731 int fd = perf_data__fd(data);
e1ab48ba 1732
8ceb41d7 1733 if (data->is_pipe)
e1ab48ba
WN		 1734 return;
1735
1736 rec->session->header.data_size += rec->bytes_written;
45112e89 1737 data->file.size = lseek(perf_data__fd(data), 0, SEEK_CUR);
56f735ff
AB		 1738 if (record__threads_enabled(rec)) {
1739 for (i = 0; i < data->dir.nr; i++)
1740 data->dir.files[i].size = lseek(data->dir.files[i].fd, 0, SEEK_CUR);
1741 }
e1ab48ba
WN		 1742
1743 if (!rec->no_buildid) {
1744 process_buildids(rec);
1745
1746 if (rec->buildid_all)
1747 dsos__hit_all(rec->session);
1748 }
1749 perf_session__write_header(rec->session, rec->evlist, fd, true);
1750
1751 return;
1752}
1753
4ea648ae 1754static int record__synthesize_workload(struct record *rec, bool tail)
be7b0c9e 1755{
9d6aae72 1756 int err;
9749b90e 1757 struct perf_thread_map *thread_map;
41b740b6 1758 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
be7b0c9e 1759
4ea648ae
WN		 1760 if (rec->opts.tail_synthesize != tail)
1761 return 0;
1762
9d6aae72
ACM		 1763 thread_map = thread_map__new_by_tid(rec->evlist->workload.pid);
1764 if (thread_map == NULL)
1765 return -1;
1766
1767 err = perf_event__synthesize_thread_map(&rec->tool, thread_map,
be7b0c9e
WN		 1768 process_synthesized_event,
1769 &rec->session->machines.host,
41b740b6 1770 needs_mmap,
3fcb10e4 1771 rec->opts.sample_address);
7836e52e 1772 perf_thread_map__put(thread_map);
9d6aae72 1773 return err;
be7b0c9e
WN		 1774}
1775
3812d298
AH		 1776static int write_finished_init(struct record *rec, bool tail)
1777{
1778 if (rec->opts.tail_synthesize != tail)
1779 return 0;
1780
1781 return record__write(rec, NULL, &finished_init_event, sizeof(finished_init_event));
1782}
1783
4ea648ae 1784static int record__synthesize(struct record *rec, bool tail);
3c1cb7e3 1785
ecfd7a9c
WN		 1786static int
1787record__switch_output(struct record *rec, bool at_exit)
1788{
8ceb41d7 1789 struct perf_data *data = &rec->data;
ecfd7a9c 1790 int fd, err;
03724b2e 1791 char *new_filename;
ecfd7a9c
WN		 1792
1793 /* Same Size: "2015122520103046"*/
1794 char timestamp[] = "InvalidTimestamp";
1795
d3d1af6f
AB		 1796 record__aio_mmap_read_sync(rec);
1797
3812d298
AH		 1798 write_finished_init(rec, true);
1799
4ea648ae
WN		 1800 record__synthesize(rec, true);
1801 if (target__none(&rec->opts.target))
1802 record__synthesize_workload(rec, true);
1803
ecfd7a9c
WN		 1804 rec->samples = 0;
1805 record__finish_output(rec);
1806 err = fetch_current_timestamp(timestamp, sizeof(timestamp));
1807 if (err) {
1808 pr_err("Failed to get current timestamp\n");
1809 return -EINVAL;
1810 }
1811
8ceb41d7 1812 fd = perf_data__switch(data, timestamp,
ecfd7a9c 1813 rec->session->header.data_offset,
03724b2e 1814 at_exit, &new_filename);
ecfd7a9c
WN		 1815 if (fd >= 0 && !at_exit) {
1816 rec->bytes_written = 0;
1817 rec->session->header.data_size = 0;
1818 }
1819
1820 if (!quiet)
1821 fprintf(stderr, "[ perf record: Dump %s.%s ]\n",
2d4f2799 1822 data->path, timestamp);
3c1cb7e3 1823
03724b2e
AK		 1824 if (rec->switch_output.num_files) {
1825 int n = rec->switch_output.cur_file + 1;
1826
1827 if (n >= rec->switch_output.num_files)
1828 n = 0;
1829 rec->switch_output.cur_file = n;
1830 if (rec->switch_output.filenames[n]) {
1831 remove(rec->switch_output.filenames[n]);
d8f9da24 1832 zfree(&rec->switch_output.filenames[n]);
03724b2e
AK		 1833 }
1834 rec->switch_output.filenames[n] = new_filename;
1835 } else {
1836 free(new_filename);
1837 }
1838
3c1cb7e3 1839 /* Output tracking events */
be7b0c9e 1840 if (!at_exit) {
4ea648ae 1841 record__synthesize(rec, false);
3c1cb7e3 1842
be7b0c9e
WN		 1843 /*
1844 * In 'perf record --switch-output' without -a,
1845 * record__synthesize() in record__switch_output() won't
1846 * generate tracking events because there's no thread_map
1847 * in evlist. Which causes newly created perf.data doesn't
1848 * contain map and comm information.
1849 * Create a fake thread_map and directly call
1850 * perf_event__synthesize_thread_map() for those events.
1851 */
1852 if (target__none(&rec->opts.target))
4ea648ae 1853 record__synthesize_workload(rec, false);
3812d298 1854 write_finished_init(rec, false);
be7b0c9e 1855 }
ecfd7a9c
WN		 1856 return fd;
1857}
1858
e3a23261
NK		 1859static void __record__read_lost_samples(struct record *rec, struct evsel *evsel,
1860 struct perf_record_lost_samples *lost,
1861 int cpu_idx, int thread_idx)
1862{
1863 struct perf_counts_values count;
1864 struct perf_sample_id *sid;
1865 struct perf_sample sample = {};
1866 int id_hdr_size;
1867
1868 if (perf_evsel__read(&evsel->core, cpu_idx, thread_idx, &count) < 0) {
07d85ba9 1869 pr_debug("read LOST count failed\n");
e3a23261
NK		 1870 return;
1871 }
1872
1873 if (count.lost == 0)
1874 return;
1875
1876 lost->lost = count.lost;
1877 if (evsel->core.ids) {
1878 sid = xyarray__entry(evsel->core.sample_id, cpu_idx, thread_idx);
1879 sample.id = sid->id;
1880 }
1881
1882 id_hdr_size = perf_event__synthesize_id_sample((void *)(lost + 1),
1883 evsel->core.attr.sample_type, &sample);
1884 lost->header.size = sizeof(*lost) + id_hdr_size;
1885 record__write(rec, NULL, lost, lost->header.size);
1886}
1887
1888static void record__read_lost_samples(struct record *rec)
1889{
1890 struct perf_session *session = rec->session;
1891 struct perf_record_lost_samples *lost;
1892 struct evsel *evsel;
1893
d031a00a
NK		 1894 /* there was an error during record__open */
1895 if (session->evlist == NULL)
1896 return;
1897
e3a23261
NK		 1898 lost = zalloc(PERF_SAMPLE_MAX_SIZE);
1899 if (lost == NULL) {
1900 pr_debug("Memory allocation failed\n");
1901 return;
1902 }
1903
1904 lost->header.type = PERF_RECORD_LOST_SAMPLES;
1905
1906 evlist__for_each_entry(session->evlist, evsel) {
1907 struct xyarray *xy = evsel->core.sample_id;
1908
d031a00a
NK		 1909 if (xy == NULL || evsel->core.fd == NULL)
1910 continue;
e3a23261
NK		 1911 if (xyarray__max_x(evsel->core.fd) != xyarray__max_x(xy) ||
1912 xyarray__max_y(evsel->core.fd) != xyarray__max_y(xy)) {
1913 pr_debug("Unmatched FD vs. sample ID: skip reading LOST count\n");
1914 continue;
1915 }
1916
1917 for (int x = 0; x < xyarray__max_x(xy); x++) {
1918 for (int y = 0; y < xyarray__max_y(xy); y++) {
1919 __record__read_lost_samples(rec, evsel, lost, x, y);
1920 }
1921 }
1922 }
1923 free(lost);
1924
1925}
1926
8ed28c2b 1927static volatile sig_atomic_t workload_exec_errno;
f33cbe72
ACM		 1928
1929/*
7b392ef0 1930 * evlist__prepare_workload will send a SIGUSR1
f33cbe72
ACM		 1931 * if the fork fails, since we asked by setting its
1932 * want_signal to true.
1933 */
45604710
NK		 1934static void workload_exec_failed_signal(int signo __maybe_unused,
1935 siginfo_t *info,
f33cbe72
ACM		 1936 void *ucontext __maybe_unused)
1937{
1938 workload_exec_errno = info->si_value.sival_int;
1939 done = 1;
f33cbe72
ACM		 1940 child_finished = 1;
1941}
1942
2dd6d8a1 1943static void snapshot_sig_handler(int sig);
bfacbe3b 1944static void alarm_sig_handler(int sig);
2dd6d8a1 1945
db0ea13c 1946static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist)
ee667f94 1947{
b2cb615d 1948 if (evlist) {
547740f7
JO		 1949 if (evlist->mmap && evlist->mmap[0].core.base)
1950 return evlist->mmap[0].core.base;
1951 if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base)
1952 return evlist->overwrite_mmap[0].core.base;
b2cb615d 1953 }
ee667f94
WN		 1954 return NULL;
1955}
1956
c45628b0
WN		 1957static const struct perf_event_mmap_page *record__pick_pc(struct record *rec)
1958{
db0ea13c 1959 const struct perf_event_mmap_page *pc = evlist__pick_pc(rec->evlist);
ee667f94
WN		 1960 if (pc)
1961 return pc;
c45628b0
WN		 1962 return NULL;
1963}
1964
4ea648ae 1965static int record__synthesize(struct record *rec, bool tail)
c45c86eb
WN		 1966{
1967 struct perf_session *session = rec->session;
1968 struct machine *machine = &session->machines.host;
8ceb41d7 1969 struct perf_data *data = &rec->data;
c45c86eb
WN		 1970 struct record_opts *opts = &rec->opts;
1971 struct perf_tool *tool = &rec->tool;
c45c86eb 1972 int err = 0;
d99c22ea 1973 event_op f = process_synthesized_event;
c45c86eb 1974
4ea648ae
WN		 1975 if (rec->opts.tail_synthesize != tail)
1976 return 0;
1977
8ceb41d7 1978 if (data->is_pipe) {
c3a057dc 1979 err = perf_event__synthesize_for_pipe(tool, session, data,
a2015516 1980 process_synthesized_event);
c3a057dc
NK		 1981 if (err < 0)
1982 goto out;
a2015516 1983
c3a057dc 1984 rec->bytes_written += err;
c45c86eb
WN		 1985 }
1986
c45628b0 1987 err = perf_event__synth_time_conv(record__pick_pc(rec), tool,
46bc29b9
AH		 1988 process_synthesized_event, machine);
1989 if (err)
1990 goto out;
1991
c0a6de06 1992 /* Synthesize id_index before auxtrace_info */
6b080312
AH		 1993 err = perf_event__synthesize_id_index(tool,
1994 process_synthesized_event,
1995 session->evlist, machine);
1996 if (err)
1997 goto out;
c0a6de06 1998
c45c86eb
WN		 1999 if (rec->opts.full_auxtrace) {
2000 err = perf_event__synthesize_auxtrace_info(rec->itr, tool,
2001 session, process_synthesized_event);
2002 if (err)
2003 goto out;
2004 }
2005
78e1bc25 2006 if (!evlist__exclude_kernel(rec->evlist)) {
6c443954
ACM		 2007 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
2008 machine);
2009 WARN_ONCE(err < 0, "Couldn't record kernel reference relocation symbol\n"
2010 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
2011 "Check /proc/kallsyms permission or run as root.\n");
2012
2013 err = perf_event__synthesize_modules(tool, process_synthesized_event,
2014 machine);
2015 WARN_ONCE(err < 0, "Couldn't record kernel module information.\n"
2016 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
2017 "Check /proc/modules permission or run as root.\n");
2018 }
c45c86eb
WN		 2019
2020 if (perf_guest) {
2021 machines__process_guests(&session->machines,
2022 perf_event__synthesize_guest_os, tool);
2023 }
2024
bfd8f72c
AK		 2025 err = perf_event__synthesize_extra_attr(&rec->tool,
2026 rec->evlist,
2027 process_synthesized_event,
2028 data->is_pipe);
2029 if (err)
2030 goto out;
2031
03617c22 2032 err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads,
373565d2
AK		 2033 process_synthesized_event,
2034 NULL);
2035 if (err < 0) {
2036 pr_err("Couldn't synthesize thread map.\n");
2037 return err;
2038 }
2039
7be1fedd 2040 err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.all_cpus,
373565d2
AK		 2041 process_synthesized_event, NULL);
2042 if (err < 0) {
2043 pr_err("Couldn't synthesize cpu map.\n");
2044 return err;
2045 }
2046
e5416950 2047 err = perf_event__synthesize_bpf_events(session, process_synthesized_event,
7b612e29 2048 machine, opts);
faf59ec8 2049 if (err < 0) {
7b612e29 2050 pr_warning("Couldn't synthesize bpf events.\n");
faf59ec8
AH		 2051 err = 0;
2052 }
7b612e29 2053
41b740b6
NK		 2054 if (rec->opts.synth & PERF_SYNTH_CGROUP) {
2055 err = perf_event__synthesize_cgroups(tool, process_synthesized_event,
2056 machine);
faf59ec8 2057 if (err < 0) {
41b740b6 2058 pr_warning("Couldn't synthesize cgroup events.\n");
faf59ec8
AH		 2059 err = 0;
2060 }
41b740b6 2061 }
ab64069f 2062
d99c22ea 2063 if (rec->opts.nr_threads_synthesize > 1) {
49c670b1 2064 mutex_init(&synth_lock);
d99c22ea
SE		 2065 perf_set_multithreaded();
2066 f = process_locked_synthesized_event;
2067 }
2068
41b740b6
NK		 2069 if (rec->opts.synth & PERF_SYNTH_TASK) {
2070 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
2071
2072 err = __machine__synthesize_threads(machine, tool, &opts->target,
2073 rec->evlist->core.threads,
2074 f, needs_mmap, opts->sample_address,
2075 rec->opts.nr_threads_synthesize);
2076 }
d99c22ea 2077
49c670b1 2078 if (rec->opts.nr_threads_synthesize > 1) {
d99c22ea 2079 perf_set_singlethreaded();
49c670b1
IR		 2080 mutex_destroy(&synth_lock);
2081 }
d99c22ea 2082
c45c86eb
WN		 2083out:
2084 return err;
2085}
2086
899e5ffb
ACM		 2087static int record__process_signal_event(union perf_event *event __maybe_unused, void *data)
2088{
2089 struct record *rec = data;
2090 pthread_kill(rec->thread_id, SIGUSR2);
2091 return 0;
2092}
2093
23cbb41c
ACM		 2094static int record__setup_sb_evlist(struct record *rec)
2095{
2096 struct record_opts *opts = &rec->opts;
2097
2098 if (rec->sb_evlist != NULL) {
2099 /*
2100 * We get here if --switch-output-event populated the
2101 * sb_evlist, so associate a callback that will send a SIGUSR2
2102 * to the main thread.
2103 */
2104 evlist__set_cb(rec->sb_evlist, record__process_signal_event, rec);
2105 rec->thread_id = pthread_self();
2106 }
1101c872 2107#ifdef HAVE_LIBBPF_SUPPORT
23cbb41c
ACM		 2108 if (!opts->no_bpf_event) {
2109 if (rec->sb_evlist == NULL) {
2110 rec->sb_evlist = evlist__new();
2111
2112 if (rec->sb_evlist == NULL) {
2113 pr_err("Couldn't create side band evlist.\n.");
2114 return -1;
2115 }
2116 }
2117
2118 if (evlist__add_bpf_sb_event(rec->sb_evlist, &rec->session->header.env)) {
2119 pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
2120 return -1;
2121 }
2122 }
1101c872 2123#endif
08c83997 2124 if (evlist__start_sb_thread(rec->sb_evlist, &rec->opts.target)) {
23cbb41c
ACM		 2125 pr_debug("Couldn't start the BPF side band thread:\nBPF programs starting from now on won't be annotatable\n");
2126 opts->no_bpf_event = true;
2127 }
2128
2129 return 0;
2130}
2131
d1e325cf
JO		 2132static int record__init_clock(struct record *rec)
2133{
2134 struct perf_session *session = rec->session;
2135 struct timespec ref_clockid;
2136 struct timeval ref_tod;
2137 u64 ref;
2138
2139 if (!rec->opts.use_clockid)
2140 return 0;
2141
9d88a1a1
JO		 2142 if (rec->opts.use_clockid && rec->opts.clockid_res_ns)
2143 session->header.env.clock.clockid_res_ns = rec->opts.clockid_res_ns;
2144
d1e325cf
JO		 2145 session->header.env.clock.clockid = rec->opts.clockid;
2146
2147 if (gettimeofday(&ref_tod, NULL) != 0) {
2148 pr_err("gettimeofday failed, cannot set reference time.\n");
2149 return -1;
2150 }
2151
2152 if (clock_gettime(rec->opts.clockid, &ref_clockid)) {
2153 pr_err("clock_gettime failed, cannot set reference time.\n");
2154 return -1;
2155 }
2156
2157 ref = (u64) ref_tod.tv_sec * NSEC_PER_SEC +
2158 (u64) ref_tod.tv_usec * NSEC_PER_USEC;
2159
2160 session->header.env.clock.tod_ns = ref;
2161
2162 ref = (u64) ref_clockid.tv_sec * NSEC_PER_SEC +
2163 (u64) ref_clockid.tv_nsec;
2164
2165 session->header.env.clock.clockid_ns = ref;
2166 return 0;
2167}
2168
d20aff15
AH		 2169static void hit_auxtrace_snapshot_trigger(struct record *rec)
2170{
2171 if (trigger_is_ready(&auxtrace_snapshot_trigger)) {
2172 trigger_hit(&auxtrace_snapshot_trigger);
2173 auxtrace_record__snapshot_started = 1;
2174 if (auxtrace_record__snapshot_start(rec->itr))
2175 trigger_error(&auxtrace_snapshot_trigger);
2176 }
2177}
2178
91c0f5ec
JY		 2179static void record__uniquify_name(struct record *rec)
2180{
2181 struct evsel *pos;
2182 struct evlist *evlist = rec->evlist;
2183 char *new_name;
2184 int ret;
2185
2186 if (!perf_pmu__has_hybrid())
2187 return;
2188
2189 evlist__for_each_entry(evlist, pos) {
2190 if (!evsel__is_hybrid(pos))
2191 continue;
2192
2193 if (strchr(pos->name, '/'))
2194 continue;
2195
2196 ret = asprintf(&new_name, "%s/%s/",
2197 pos->pmu_name, pos->name);
2198 if (ret) {
2199 free(pos->name);
2200 pos->name = new_name;
2201 }
2202 }
2203}
2204
1e5de7d9
AB		 2205static int record__terminate_thread(struct record_thread *thread_data)
2206{
2207 int err;
2208 enum thread_msg ack = THREAD_MSG__UNDEFINED;
2209 pid_t tid = thread_data->tid;
2210
2211 close(thread_data->pipes.msg[1]);
2212 thread_data->pipes.msg[1] = -1;
2213 err = read(thread_data->pipes.ack[0], &ack, sizeof(ack));
2214 if (err > 0)
2215 pr_debug2("threads[%d]: sent %s\n", tid, thread_msg_tags[ack]);
2216 else
2217 pr_warning("threads[%d]: failed to receive termination notification from %d\n",
2218 thread->tid, tid);
2219
2220 return 0;
2221}
2222
396b626b
AB		 2223static int record__start_threads(struct record *rec)
2224{
3217e9fe 2225 int t, tt, err, ret = 0, nr_threads = rec->nr_threads;
396b626b 2226 struct record_thread *thread_data = rec->thread_data;
3217e9fe
AB		 2227 sigset_t full, mask;
2228 pthread_t handle;
2229 pthread_attr_t attrs;
396b626b
AB		 2230
2231 thread = &thread_data[0];
2232
3217e9fe
AB		 2233 if (!record__threads_enabled(rec))
2234 return 0;
2235
2236 sigfillset(&full);
2237 if (sigprocmask(SIG_SETMASK, &full, &mask)) {
2238 pr_err("Failed to block signals on threads start: %s\n", strerror(errno));
2239 return -1;
2240 }
2241
2242 pthread_attr_init(&attrs);
2243 pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED);
2244
2245 for (t = 1; t < nr_threads; t++) {
2246 enum thread_msg msg = THREAD_MSG__UNDEFINED;
2247
2248#ifdef HAVE_PTHREAD_ATTR_SETAFFINITY_NP
2249 pthread_attr_setaffinity_np(&attrs,
2250 MMAP_CPU_MASK_BYTES(&(thread_data[t].mask->affinity)),
2251 (cpu_set_t *)(thread_data[t].mask->affinity.bits));
2252#endif
2253 if (pthread_create(&handle, &attrs, record__thread, &thread_data[t])) {
2254 for (tt = 1; tt < t; tt++)
2255 record__terminate_thread(&thread_data[t]);
2256 pr_err("Failed to start threads: %s\n", strerror(errno));
2257 ret = -1;
2258 goto out_err;
2259 }
2260
2261 err = read(thread_data[t].pipes.ack[0], &msg, sizeof(msg));
2262 if (err > 0)
2263 pr_debug2("threads[%d]: sent %s\n", rec->thread_data[t].tid,
2264 thread_msg_tags[msg]);
2265 else
2266 pr_warning("threads[%d]: failed to receive start notification from %d\n",
2267 thread->tid, rec->thread_data[t].tid);
2268 }
2269
2270 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
2271 (cpu_set_t *)thread->mask->affinity.bits);
2272
396b626b
AB		 2273 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
2274
3217e9fe
AB		 2275out_err:
2276 pthread_attr_destroy(&attrs);
2277
2278 if (sigprocmask(SIG_SETMASK, &mask, NULL)) {
2279 pr_err("Failed to unblock signals on threads start: %s\n", strerror(errno));
2280 ret = -1;
2281 }
2282
2283 return ret;
396b626b
AB		 2284}
2285
2286static int record__stop_threads(struct record *rec)
2287{
2288 int t;
2289 struct record_thread *thread_data = rec->thread_data;
2290
1e5de7d9
AB		 2291 for (t = 1; t < rec->nr_threads; t++)
2292 record__terminate_thread(&thread_data[t]);
2293
610fbc01 2294 for (t = 0; t < rec->nr_threads; t++) {
396b626b 2295 rec->samples += thread_data[t].samples;
610fbc01
AB		 2296 if (!record__threads_enabled(rec))
2297 continue;
2298 rec->session->bytes_transferred += thread_data[t].bytes_transferred;
2299 rec->session->bytes_compressed += thread_data[t].bytes_compressed;
2300 pr_debug("threads[%d]: samples=%lld, wakes=%ld, ", thread_data[t].tid,
2301 thread_data[t].samples, thread_data[t].waking);
2302 if (thread_data[t].bytes_transferred && thread_data[t].bytes_compressed)
2303 pr_debug("transferred=%" PRIu64 ", compressed=%" PRIu64 "\n",
2304 thread_data[t].bytes_transferred, thread_data[t].bytes_compressed);
2305 else
2306 pr_debug("written=%" PRIu64 "\n", thread_data[t].bytes_written);
2307 }
396b626b
AB		 2308
2309 return 0;
2310}
2311
2312static unsigned long record__waking(struct record *rec)
2313{
2314 int t;
2315 unsigned long waking = 0;
2316 struct record_thread *thread_data = rec->thread_data;
2317
2318 for (t = 0; t < rec->nr_threads; t++)
2319 waking += thread_data[t].waking;
2320
2321 return waking;
2322}
2323
8c6f45a7 2324static int __cmd_record(struct record *rec, int argc, const char **argv)
16c8a109 2325{
57706abc 2326 int err;
45604710 2327 int status = 0;
46be604b 2328 const bool forks = argc > 0;
45694aa7 2329 struct perf_tool *tool = &rec->tool;
b4006796 2330 struct record_opts *opts = &rec->opts;
8ceb41d7 2331 struct perf_data *data = &rec->data;
d20deb64 2332 struct perf_session *session;
6dcf45ef 2333 bool disabled = false, draining = false;
42aa276f 2334 int fd;
d3c8c08e 2335 float ratio = 0;
acce0223 2336 enum evlist_ctl_cmd cmd = EVLIST_CTL_CMD_UNSUPPORTED;
de9ac07b 2337
45604710 2338 atexit(record__sig_exit);
f5970550
PZ		 2339 signal(SIGCHLD, sig_handler);
2340 signal(SIGINT, sig_handler);
804f7ac7 2341 signal(SIGTERM, sig_handler);
a074865e 2342 signal(SIGSEGV, sigsegv_handler);
c0bdc1c4 2343
f3b3614a
HB		 2344 if (rec->opts.record_namespaces)
2345 tool->namespace_events = true;
2346
8fb4b679
NK		 2347 if (rec->opts.record_cgroup) {
2348#ifdef HAVE_FILE_HANDLE
2349 tool->cgroup_events = true;
2350#else
2351 pr_err("cgroup tracking is not supported\n");
2352 return -1;
2353#endif
2354 }
2355
dc0c6127 2356 if (rec->opts.auxtrace_snapshot_mode || rec->switch_output.enabled) {
2dd6d8a1 2357 signal(SIGUSR2, snapshot_sig_handler);
3c1cb7e3
WN		 2358 if (rec->opts.auxtrace_snapshot_mode)
2359 trigger_on(&auxtrace_snapshot_trigger);
dc0c6127 2360 if (rec->switch_output.enabled)
3c1cb7e3 2361 trigger_on(&switch_output_trigger);
c0bdc1c4 2362 } else {
2dd6d8a1 2363 signal(SIGUSR2, SIG_IGN);
c0bdc1c4 2364 }
f5970550 2365
2681bd85 2366 session = perf_session__new(data, tool);
6ef81c55 2367 if (IS_ERR(session)) {
ffa91880 2368 pr_err("Perf session creation failed.\n");
6ef81c55 2369 return PTR_ERR(session);
a9a70bbc
ACM		 2370 }
2371
b5f2511d
AB		 2372 if (record__threads_enabled(rec)) {
2373 if (perf_data__is_pipe(&rec->data)) {
2374 pr_err("Parallel trace streaming is not available in pipe mode.\n");
2375 return -1;
2376 }
2377 if (rec->opts.full_auxtrace) {
2378 pr_err("Parallel trace streaming is not available in AUX area tracing mode.\n");
2379 return -1;
2380 }
2381 }
2382
8ceb41d7 2383 fd = perf_data__fd(data);
d20deb64
ACM		 2384 rec->session = session;
2385
5d7f4116
AB		 2386 if (zstd_init(&session->zstd_data, rec->opts.comp_level) < 0) {
2387 pr_err("Compression initialization failed.\n");
2388 return -1;
2389 }
da231338
AM		 2390#ifdef HAVE_EVENTFD_SUPPORT
2391 done_fd = eventfd(0, EFD_NONBLOCK);
2392 if (done_fd < 0) {
2393 pr_err("Failed to create wakeup eventfd, error: %m\n");
2394 status = -1;
2395 goto out_delete_session;
2396 }
e16c2ce7 2397 err = evlist__add_wakeup_eventfd(rec->evlist, done_fd);
da231338
AM		 2398 if (err < 0) {
2399 pr_err("Failed to add wakeup eventfd to poll list\n");
2400 status = err;
2401 goto out_delete_session;
2402 }
2403#endif // HAVE_EVENTFD_SUPPORT
5d7f4116
AB		 2404
2405 session->header.env.comp_type = PERF_COMP_ZSTD;
2406 session->header.env.comp_level = rec->opts.comp_level;
2407
eeb399b5
AH		 2408 if (rec->opts.kcore &&
2409 !record__kcore_readable(&session->machines.host)) {
2410 pr_err("ERROR: kcore is not readable.\n");
2411 return -1;
2412 }
2413
d1e325cf
JO		 2414 if (record__init_clock(rec))
2415 return -1;
2416
8c6f45a7 2417 record__init_features(rec);
330aa675 2418
d4db3f16 2419 if (forks) {
7b392ef0
ACM		 2420 err = evlist__prepare_workload(rec->evlist, &opts->target, argv, data->is_pipe,
2421 workload_exec_failed_signal);
35b9d88e
ACM		 2422 if (err < 0) {
2423 pr_err("Couldn't run the workload!\n");
45604710 2424 status = err;
35b9d88e 2425 goto out_delete_session;
856e9660 2426 }
856e9660
PZ		 2427 }
2428
ad46e48c
JO		 2429 /*
2430 * If we have just single event and are sending data
2431 * through pipe, we need to force the ids allocation,
2432 * because we synthesize event name through the pipe
2433 * and need the id for that.
2434 */
6484d2f9 2435 if (data->is_pipe && rec->evlist->core.nr_entries == 1)
ad46e48c
JO		 2436 rec->opts.sample_id = true;
2437
91c0f5ec
JY		 2438 record__uniquify_name(rec);
2439
da406202
AH		 2440 /* Debug message used by test scripts */
2441 pr_debug3("perf record opening and mmapping events\n");
8c6f45a7 2442 if (record__open(rec) != 0) {
8d3eca20 2443 err = -1;
396b626b 2444 goto out_free_threads;
8d3eca20 2445 }
da406202
AH		 2446 /* Debug message used by test scripts */
2447 pr_debug3("perf record done opening and mmapping events\n");
f6fa4375 2448 session->header.env.comp_mmap_len = session->evlist->core.mmap_len;
de9ac07b 2449
eeb399b5
AH		 2450 if (rec->opts.kcore) {
2451 err = record__kcore_copy(&session->machines.host, data);
2452 if (err) {
2453 pr_err("ERROR: Failed to copy kcore\n");
396b626b 2454 goto out_free_threads;
eeb399b5
AH		 2455 }
2456 }
2457
8690a2a7
WN		 2458 err = bpf__apply_obj_config();
2459 if (err) {
2460 char errbuf[BUFSIZ];
2461
2462 bpf__strerror_apply_obj_config(err, errbuf, sizeof(errbuf));
2463 pr_err("ERROR: Apply config to BPF failed: %s\n",
2464 errbuf);
396b626b 2465 goto out_free_threads;
8690a2a7
WN		 2466 }
2467
cca8482c
AH		 2468 /*
2469 * Normally perf_session__new would do this, but it doesn't have the
2470 * evlist.
2471 */
8cedf3a5 2472 if (rec->tool.ordered_events && !evlist__sample_id_all(rec->evlist)) {
cca8482c
AH		 2473 pr_warning("WARNING: No sample_id_all support, falling back to unordered processing\n");
2474 rec->tool.ordered_events = false;
2475 }
2476
3a683120 2477 if (!rec->evlist->core.nr_groups)
a8bb559b
NK		 2478 perf_header__clear_feat(&session->header, HEADER_GROUP_DESC);
2479
8ceb41d7 2480 if (data->is_pipe) {
42aa276f 2481 err = perf_header__write_pipe(fd);
529870e3 2482 if (err < 0)
396b626b 2483 goto out_free_threads;
563aecb2 2484 } else {
42aa276f 2485 err = perf_session__write_header(session, rec->evlist, fd, false);
d5eed904 2486 if (err < 0)
396b626b 2487 goto out_free_threads;
56b03f3c
ACM		 2488 }
2489
b38d85ef 2490 err = -1;
d3665498 2491 if (!rec->no_buildid
e20960c0 2492 && !perf_header__has_feat(&session->header, HEADER_BUILD_ID)) {
d3665498 2493 pr_err("Couldn't generate buildids. "
e20960c0 2494 "Use --no-buildid to profile anyway.\n");
396b626b 2495 goto out_free_threads;
e20960c0
RR		 2496 }
2497
23cbb41c
ACM		 2498 err = record__setup_sb_evlist(rec);
2499 if (err)
396b626b 2500 goto out_free_threads;
657ee553 2501
4ea648ae 2502 err = record__synthesize(rec, false);
c45c86eb 2503 if (err < 0)
396b626b 2504 goto out_free_threads;
8d3eca20 2505
d20deb64 2506 if (rec->realtime_prio) {
de9ac07b
PZ		 2507 struct sched_param param;
2508
d20deb64 2509 param.sched_priority = rec->realtime_prio;
de9ac07b 2510 if (sched_setscheduler(0, SCHED_FIFO, &param)) {
6beba7ad 2511 pr_err("Could not set realtime priority.\n");
8d3eca20 2512 err = -1;
396b626b 2513 goto out_free_threads;
de9ac07b
PZ		 2514 }
2515 }
2516
396b626b
AB		 2517 if (record__start_threads(rec))
2518 goto out_free_threads;
2519
774cb499
JO		 2520 /*
2521 * When perf is starting the traced process, all the events
2522 * (apart from group members) have enable_on_exec=1 set,
2523 * so don't spoil it by prematurely enabling them.
2524 */
cb4b9e68 2525 if (!target__none(&opts->target) && !opts->target.initial_delay)
1c87f165 2526 evlist__enable(rec->evlist);
764e16a3 2527
856e9660
PZ		 2528 /*
2529 * Let the child rip
2530 */
e803cf97 2531 if (forks) {
20a8a3cf 2532 struct machine *machine = &session->machines.host;
e5bed564 2533 union perf_event *event;
e907caf3 2534 pid_t tgid;
e5bed564
NK		 2535
2536 event = malloc(sizeof(event->comm) + machine->id_hdr_size);
2537 if (event == NULL) {
2538 err = -ENOMEM;
2539 goto out_child;
2540 }
2541
e803cf97
NK		 2542 /*
2543 * Some H/W events are generated before COMM event
2544 * which is emitted during exec(), so perf script
2545 * cannot see a correct process name for those events.
2546 * Synthesize COMM event to prevent it.
2547 */
e907caf3
HB		 2548 tgid = perf_event__synthesize_comm(tool, event,
2549 rec->evlist->workload.pid,
2550 process_synthesized_event,
2551 machine);
2552 free(event);
2553
2554 if (tgid == -1)
2555 goto out_child;
2556
2557 event = malloc(sizeof(event->namespaces) +
2558 (NR_NAMESPACES * sizeof(struct perf_ns_link_info)) +
2559 machine->id_hdr_size);
2560 if (event == NULL) {
2561 err = -ENOMEM;
2562 goto out_child;
2563 }
2564
2565 /*
2566 * Synthesize NAMESPACES event for the command specified.
2567 */
2568 perf_event__synthesize_namespaces(tool, event,
2569 rec->evlist->workload.pid,
2570 tgid, process_synthesized_event,
2571 machine);
e5bed564 2572 free(event);
e803cf97 2573
7b392ef0 2574 evlist__start_workload(rec->evlist);
e803cf97 2575 }
856e9660 2576
cb4b9e68 2577 if (opts->target.initial_delay) {
68cd3b45 2578 pr_info(EVLIST_DISABLED_MSG);
cb4b9e68
CD		 2579 if (opts->target.initial_delay > 0) {
2580 usleep(opts->target.initial_delay * USEC_PER_MSEC);
68cd3b45
AB		 2581 evlist__enable(rec->evlist);
2582 pr_info(EVLIST_ENABLED_MSG);
2583 }
6619a53e
AK		 2584 }
2585
6657a099
AH		 2586 err = event_enable_timer__start(rec->evlist->eet);
2587 if (err)
2588 goto out_child;
2589
da406202
AH		 2590 /* Debug message used by test scripts */
2591 pr_debug3("perf record has started\n");
2592 fflush(stderr);
2593
5f9cf599 2594 trigger_ready(&auxtrace_snapshot_trigger);
3c1cb7e3 2595 trigger_ready(&switch_output_trigger);
a074865e 2596 perf_hooks__invoke_record_start();
3812d298
AH		 2597
2598 /*
2599 * Must write FINISHED_INIT so it will be seen after all other
2600 * synthesized user events, but before any regular events.
2601 */
2602 err = write_finished_init(rec, false);
2603 if (err < 0)
2604 goto out_child;
2605
649c48a9 2606 for (;;) {
396b626b 2607 unsigned long long hits = thread->samples;
de9ac07b 2608
05737464
WN		 2609 /*
2610 * rec->evlist->bkw_mmap_state is possible to be
2611 * BKW_MMAP_EMPTY here: when done == true and
2612 * hits != rec->samples in previous round.
2613 *
ade9d208 2614 * evlist__toggle_bkw_mmap ensure we never
05737464
WN		 2615 * convert BKW_MMAP_EMPTY to BKW_MMAP_DATA_PENDING.
2616 */
2617 if (trigger_is_hit(&switch_output_trigger) || done || draining)
ade9d208 2618 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_DATA_PENDING);
05737464 2619
470530bb 2620 if (record__mmap_read_all(rec, false) < 0) {
5f9cf599 2621 trigger_error(&auxtrace_snapshot_trigger);
3c1cb7e3 2622 trigger_error(&switch_output_trigger);
8d3eca20 2623 err = -1;
45604710 2624 goto out_child;
8d3eca20 2625 }
de9ac07b 2626
2dd6d8a1
AH		 2627 if (auxtrace_record__snapshot_started) {
2628 auxtrace_record__snapshot_started = 0;
5f9cf599 2629 if (!trigger_is_error(&auxtrace_snapshot_trigger))
ce7b0e42 2630 record__read_auxtrace_snapshot(rec, false);
5f9cf599 2631 if (trigger_is_error(&auxtrace_snapshot_trigger)) {
2dd6d8a1
AH		 2632 pr_err("AUX area tracing snapshot failed\n");
2633 err = -1;
2634 goto out_child;
2635 }
2636 }
2637
3c1cb7e3 2638 if (trigger_is_hit(&switch_output_trigger)) {
05737464
WN		 2639 /*
2640 * If switch_output_trigger is hit, the data in
2641 * overwritable ring buffer should have been collected,
2642 * so bkw_mmap_state should be set to BKW_MMAP_EMPTY.
2643 *
2644 * If SIGUSR2 raise after or during record__mmap_read_all(),
2645 * record__mmap_read_all() didn't collect data from
2646 * overwritable ring buffer. Read again.
2647 */
2648 if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING)
2649 continue;
3c1cb7e3
WN		 2650 trigger_ready(&switch_output_trigger);
2651
05737464
WN		 2652 /*
2653 * Reenable events in overwrite ring buffer after
2654 * record__mmap_read_all(): we should have collected
2655 * data from it.
2656 */
ade9d208 2657 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_RUNNING);
05737464 2658
3c1cb7e3
WN		 2659 if (!quiet)
2660 fprintf(stderr, "[ perf record: dump data: Woken up %ld times ]\n",
396b626b
AB		 2661 record__waking(rec));
2662 thread->waking = 0;
3c1cb7e3
WN		 2663 fd = record__switch_output(rec, false);
2664 if (fd < 0) {
2665 pr_err("Failed to switch to new file\n");
2666 trigger_error(&switch_output_trigger);
2667 err = fd;
2668 goto out_child;
2669 }
bfacbe3b
JO		 2670
2671 /* re-arm the alarm */
2672 if (rec->switch_output.time)
2673 alarm(rec->switch_output.time);
3c1cb7e3
WN		 2674 }
2675
396b626b 2676 if (hits == thread->samples) {
6dcf45ef 2677 if (done || draining)
649c48a9 2678 break;
396b626b 2679 err = fdarray__poll(&thread->pollfd, -1);
a515114f
JO		 2680 /*
2681 * Propagate error, only if there's any. Ignore positive
2682 * number of returned events and interrupt error.
2683 */
2684 if (err > 0 || (err < 0 && errno == EINTR))
45604710 2685 err = 0;
396b626b 2686 thread->waking++;
6dcf45ef 2687
396b626b
AB		 2688 if (fdarray__filter(&thread->pollfd, POLLERR | POLLHUP,
2689 record__thread_munmap_filtered, NULL) == 0)
6dcf45ef 2690 draining = true;
396b626b 2691
6562c9ac
AH		 2692 err = record__update_evlist_pollfd_from_thread(rec, rec->evlist, thread);
2693 if (err)
2694 goto out_child;
8b412664
PZ		 2695 }
2696
acce0223
AB		 2697 if (evlist__ctlfd_process(rec->evlist, &cmd) > 0) {
2698 switch (cmd) {
d20aff15
AH		 2699 case EVLIST_CTL_CMD_SNAPSHOT:
2700 hit_auxtrace_snapshot_trigger(rec);
2701 evlist__ctlfd_ack(rec->evlist);
2702 break;
f186cd61
JO		 2703 case EVLIST_CTL_CMD_STOP:
2704 done = 1;
2705 break;
acce0223
AB		 2706 case EVLIST_CTL_CMD_ACK:
2707 case EVLIST_CTL_CMD_UNSUPPORTED:
991ae4eb
JO		 2708 case EVLIST_CTL_CMD_ENABLE:
2709 case EVLIST_CTL_CMD_DISABLE:
142544a9 2710 case EVLIST_CTL_CMD_EVLIST:
47fddcb4 2711 case EVLIST_CTL_CMD_PING:
acce0223
AB		 2712 default:
2713 break;
2714 }
2715 }
2716
6657a099
AH		 2717 err = event_enable_timer__process(rec->evlist->eet);
2718 if (err < 0)
2719 goto out_child;
2720 if (err) {
2721 err = 0;
2722 done = 1;
2723 }
2724
774cb499
JO		 2725 /*
2726 * When perf is starting the traced process, at the end events
2727 * die with the process and we wait for that. Thus no need to
2728 * disable events in this case.
2729 */
602ad878 2730 if (done && !disabled && !target__none(&opts->target)) {
5f9cf599 2731 trigger_off(&auxtrace_snapshot_trigger);
e74676de 2732 evlist__disable(rec->evlist);
2711926a
JO		 2733 disabled = true;
2734 }
de9ac07b 2735 }
ce7b0e42 2736
5f9cf599 2737 trigger_off(&auxtrace_snapshot_trigger);
3c1cb7e3 2738 trigger_off(&switch_output_trigger);
de9ac07b 2739
ce7b0e42
AS		 2740 if (opts->auxtrace_snapshot_on_exit)
2741 record__auxtrace_snapshot_exit(rec);
2742
f33cbe72 2743 if (forks && workload_exec_errno) {
3535a696 2744 char msg[STRERR_BUFSIZE], strevsels[2048];
c8b5f2c9 2745 const char *emsg = str_error_r(workload_exec_errno, msg, sizeof(msg));
3535a696
ACM		 2746
2747 evlist__scnprintf_evsels(rec->evlist, sizeof(strevsels), strevsels);
2748
2749 pr_err("Failed to collect '%s' for the '%s' workload: %s\n",
2750 strevsels, argv[0], emsg);
f33cbe72 2751 err = -1;
45604710 2752 goto out_child;
f33cbe72
ACM		 2753 }
2754
e3d59112 2755 if (!quiet)
396b626b
AB		 2756 fprintf(stderr, "[ perf record: Woken up %ld times to write data ]\n",
2757 record__waking(rec));
b44308f5 2758
3812d298
AH		 2759 write_finished_init(rec, true);
2760
4ea648ae
WN		 2761 if (target__none(&rec->opts.target))
2762 record__synthesize_workload(rec, true);
2763
45604710 2764out_child:
396b626b 2765 record__stop_threads(rec);
470530bb 2766 record__mmap_read_all(rec, true);
396b626b 2767out_free_threads:
415ccb58 2768 record__free_thread_data(rec);
396b626b 2769 evlist__finalize_ctlfd(rec->evlist);
d3d1af6f
AB		 2770 record__aio_mmap_read_sync(rec);
2771
d3c8c08e
AB		 2772 if (rec->session->bytes_transferred && rec->session->bytes_compressed) {
2773 ratio = (float)rec->session->bytes_transferred/(float)rec->session->bytes_compressed;
2774 session->header.env.comp_ratio = ratio + 0.5;
2775 }
2776
45604710
NK		 2777 if (forks) {
2778 int exit_status;
addc2785 2779
45604710
NK		 2780 if (!child_finished)
2781 kill(rec->evlist->workload.pid, SIGTERM);
2782
2783 wait(&exit_status);
2784
2785 if (err < 0)
2786 status = err;
2787 else if (WIFEXITED(exit_status))
2788 status = WEXITSTATUS(exit_status);
2789 else if (WIFSIGNALED(exit_status))
2790 signr = WTERMSIG(exit_status);
2791 } else
2792 status = err;
2793
edc41a10
NK		 2794 if (rec->off_cpu)
2795 rec->bytes_written += off_cpu_write(rec->session);
2796
e3a23261 2797 record__read_lost_samples(rec);
4ea648ae 2798 record__synthesize(rec, true);
e3d59112
NK		 2799 /* this will be recalculated during process_buildids() */
2800 rec->samples = 0;
2801
ecfd7a9c
WN		 2802 if (!err) {
2803 if (!rec->timestamp_filename) {
2804 record__finish_output(rec);
2805 } else {
2806 fd = record__switch_output(rec, true);
2807 if (fd < 0) {
2808 status = fd;
2809 goto out_delete_session;
2810 }
2811 }
2812 }
39d17dac 2813
a074865e
WN		 2814 perf_hooks__invoke_record_end();
2815
e3d59112
NK		 2816 if (!err && !quiet) {
2817 char samples[128];
ecfd7a9c
WN		 2818 const char *postfix = rec->timestamp_filename ?
2819 ".<timestamp>" : "";
e3d59112 2820
ef149c25 2821 if (rec->samples && !rec->opts.full_auxtrace)
e3d59112
NK		 2822 scnprintf(samples, sizeof(samples),
2823 " (%" PRIu64 " samples)", rec->samples);
2824 else
2825 samples[0] = '\0';
2826
d3c8c08e 2827 fprintf(stderr, "[ perf record: Captured and wrote %.3f MB %s%s%s",
8ceb41d7 2828 perf_data__size(data) / 1024.0 / 1024.0,
2d4f2799 2829 data->path, postfix, samples);
d3c8c08e
AB		 2830 if (ratio) {
2831 fprintf(stderr, ", compressed (original %.3f MB, ratio is %.3f)",
2832 rec->session->bytes_transferred / 1024.0 / 1024.0,
2833 ratio);
2834 }
2835 fprintf(stderr, " ]\n");
e3d59112
NK		 2836 }
2837
39d17dac 2838out_delete_session:
da231338 2839#ifdef HAVE_EVENTFD_SUPPORT
304f0a2f
IR		 2840 if (done_fd >= 0) {
2841 fd = done_fd;
2842 done_fd = -1;
2843
2844 close(fd);
2845 }
da231338 2846#endif
5d7f4116 2847 zstd_fini(&session->zstd_data);
39d17dac 2848 perf_session__delete(session);
657ee553
SL		 2849
2850 if (!opts->no_bpf_event)
08c83997 2851 evlist__stop_sb_thread(rec->sb_evlist);
45604710 2852 return status;
de9ac07b 2853}
0e9b20b8 2854
0883e820 2855static void callchain_debug(struct callchain_param *callchain)
09b0fd45 2856{
aad2b21c 2857 static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF", "LBR" };
a601fdff 2858
0883e820 2859 pr_debug("callchain: type %s\n", str[callchain->record_mode]);
26d33022 2860
0883e820 2861 if (callchain->record_mode == CALLCHAIN_DWARF)
09b0fd45 2862 pr_debug("callchain: stack dump size %d\n",
0883e820 2863 callchain->dump_size);
09b0fd45
JO		 2864}
2865
0883e820
ACM		 2866int record_opts__parse_callchain(struct record_opts *record,
2867 struct callchain_param *callchain,
2868 const char *arg, bool unset)
09b0fd45 2869{
09b0fd45 2870 int ret;
0883e820 2871 callchain->enabled = !unset;
eb853e80 2872
09b0fd45
JO		 2873 /* --no-call-graph */
2874 if (unset) {
0883e820 2875 callchain->record_mode = CALLCHAIN_NONE;
09b0fd45
JO		 2876 pr_debug("callchain: disabled\n");
2877 return 0;
2878 }
2879
0883e820 2880 ret = parse_callchain_record_opt(arg, callchain);
5c0cf224
JO		 2881 if (!ret) {
2882 /* Enable data address sampling for DWARF unwind. */
0883e820 2883 if (callchain->record_mode == CALLCHAIN_DWARF)
5c0cf224 2884 record->sample_address = true;
0883e820 2885 callchain_debug(callchain);
5c0cf224 2886 }
26d33022
JO		 2887
2888 return ret;
2889}
2890
0883e820
ACM		 2891int record_parse_callchain_opt(const struct option *opt,
2892 const char *arg,
2893 int unset)
2894{
2895 return record_opts__parse_callchain(opt->value, &callchain_param, arg, unset);
2896}
2897
c421e80b 2898int record_callchain_opt(const struct option *opt,
09b0fd45
JO		 2899 const char *arg __maybe_unused,
2900 int unset __maybe_unused)
2901{
2ddd5c04 2902 struct callchain_param *callchain = opt->value;
c421e80b 2903
2ddd5c04 2904 callchain->enabled = true;
09b0fd45 2905
2ddd5c04
ACM		 2906 if (callchain->record_mode == CALLCHAIN_NONE)
2907 callchain->record_mode = CALLCHAIN_FP;
eb853e80 2908
2ddd5c04 2909 callchain_debug(callchain);
09b0fd45
JO		 2910 return 0;
2911}
2912
eb853e80
JO		 2913static int perf_record_config(const char *var, const char *value, void *cb)
2914{
7a29c087
NK		 2915 struct record *rec = cb;
2916
2917 if (!strcmp(var, "record.build-id")) {
2918 if (!strcmp(value, "cache"))
2919 rec->no_buildid_cache = false;
2920 else if (!strcmp(value, "no-cache"))
2921 rec->no_buildid_cache = true;
2922 else if (!strcmp(value, "skip"))
2923 rec->no_buildid = true;
e29386c8
JO		 2924 else if (!strcmp(value, "mmap"))
2925 rec->buildid_mmap = true;
7a29c087
NK		 2926 else
2927 return -1;
2928 return 0;
2929 }
cff17205
YX		 2930 if (!strcmp(var, "record.call-graph")) {
2931 var = "call-graph.record-mode";
2932 return perf_default_config(var, value, cb);
2933 }
93f20c0f
AB		 2934#ifdef HAVE_AIO_SUPPORT
2935 if (!strcmp(var, "record.aio")) {
2936 rec->opts.nr_cblocks = strtol(value, NULL, 0);
2937 if (!rec->opts.nr_cblocks)
2938 rec->opts.nr_cblocks = nr_cblocks_default;
2939 }
2940#endif
9bce13ea
JO		 2941 if (!strcmp(var, "record.debuginfod")) {
2942 rec->debuginfod.urls = strdup(value);
2943 if (!rec->debuginfod.urls)
2944 return -ENOMEM;
2945 rec->debuginfod.set = true;
2946 }
eb853e80 2947
cff17205 2948 return 0;
eb853e80
JO		 2949}
2950
6657a099
AH		 2951static int record__parse_event_enable_time(const struct option *opt, const char *str, int unset)
2952{
2953 struct record *rec = (struct record *)opt->value;
2954
2955 return evlist__parse_event_enable_time(rec->evlist, &rec->opts, str, unset);
2956}
814c8c38 2957
f4fe11b7
AB		 2958static int record__parse_affinity(const struct option *opt, const char *str, int unset)
2959{
2960 struct record_opts *opts = (struct record_opts *)opt->value;
2961
2962 if (unset || !str)
2963 return 0;
2964
2965 if (!strcasecmp(str, "node"))
2966 opts->affinity = PERF_AFFINITY_NODE;
2967 else if (!strcasecmp(str, "cpu"))
2968 opts->affinity = PERF_AFFINITY_CPU;
2969
2970 return 0;
2971}
2972
7954f716
AB		 2973static int record__mmap_cpu_mask_alloc(struct mmap_cpu_mask *mask, int nr_bits)
2974{
2975 mask->nbits = nr_bits;
2976 mask->bits = bitmap_zalloc(mask->nbits);
2977 if (!mask->bits)
2978 return -ENOMEM;
2979
2980 return 0;
2981}
2982
2983static void record__mmap_cpu_mask_free(struct mmap_cpu_mask *mask)
2984{
2985 bitmap_free(mask->bits);
2986 mask->nbits = 0;
2987}
2988
2989static int record__thread_mask_alloc(struct thread_mask *mask, int nr_bits)
2990{
2991 int ret;
2992
2993 ret = record__mmap_cpu_mask_alloc(&mask->maps, nr_bits);
2994 if (ret) {
2995 mask->affinity.bits = NULL;
2996 return ret;
2997 }
2998
2999 ret = record__mmap_cpu_mask_alloc(&mask->affinity, nr_bits);
3000 if (ret) {
3001 record__mmap_cpu_mask_free(&mask->maps);
3002 mask->maps.bits = NULL;
3003 }
3004
3005 return ret;
3006}
3007
3008static void record__thread_mask_free(struct thread_mask *mask)
3009{
3010 record__mmap_cpu_mask_free(&mask->maps);
3011 record__mmap_cpu_mask_free(&mask->affinity);
3012}
3013
06380a84
AB		 3014static int record__parse_threads(const struct option *opt, const char *str, int unset)
3015{
f466e5ed 3016 int s;
06380a84
AB		 3017 struct record_opts *opts = opt->value;
3018
f466e5ed 3019 if (unset || !str || !strlen(str)) {
06380a84 3020 opts->threads_spec = THREAD_SPEC__CPU;
f466e5ed
AB		 3021 } else {
3022 for (s = 1; s < THREAD_SPEC__MAX; s++) {
3023 if (s == THREAD_SPEC__USER) {
3024 opts->threads_user_spec = strdup(str);
3025 if (!opts->threads_user_spec)
3026 return -ENOMEM;
3027 opts->threads_spec = THREAD_SPEC__USER;
3028 break;
3029 }
3030 if (!strncasecmp(str, thread_spec_tags[s], strlen(thread_spec_tags[s]))) {
3031 opts->threads_spec = s;
3032 break;
3033 }
3034 }
3035 }
3036
3037 if (opts->threads_spec == THREAD_SPEC__USER)
3038 pr_debug("threads_spec: %s\n", opts->threads_user_spec);
3039 else
3040 pr_debug("threads_spec: %s\n", thread_spec_tags[opts->threads_spec]);
06380a84
AB		 3041
3042 return 0;
3043}
3044
6d575816
JS		 3045static int parse_output_max_size(const struct option *opt,
3046 const char *str, int unset)
3047{
3048 unsigned long *s = (unsigned long *)opt->value;
3049 static struct parse_tag tags_size[] = {
3050 { .tag = 'B', .mult = 1 },
3051 { .tag = 'K', .mult = 1 << 10 },
3052 { .tag = 'M', .mult = 1 << 20 },
3053 { .tag = 'G', .mult = 1 << 30 },
3054 { .tag = 0 },
3055 };
3056 unsigned long val;
3057
3058 if (unset) {
3059 *s = 0;
3060 return 0;
3061 }
3062
3063 val = parse_tag_value(str, tags_size);
3064 if (val != (unsigned long) -1) {
3065 *s = val;
3066 return 0;
3067 }
3068
3069 return -1;
3070}
3071
e9db1310
AH		 3072static int record__parse_mmap_pages(const struct option *opt,
3073 const char *str,
3074 int unset __maybe_unused)
3075{
3076 struct record_opts *opts = opt->value;
3077 char *s, *p;
3078 unsigned int mmap_pages;
3079 int ret;
3080
3081 if (!str)
3082 return -EINVAL;
3083
3084 s = strdup(str);
3085 if (!s)
3086 return -ENOMEM;
3087
3088 p = strchr(s, ',');
3089 if (p)
3090 *p = '\0';
3091
3092 if (*s) {
25f84702 3093 ret = __evlist__parse_mmap_pages(&mmap_pages, s);
e9db1310
AH		 3094 if (ret)
3095 goto out_free;
3096 opts->mmap_pages = mmap_pages;
3097 }
3098
3099 if (!p) {
3100 ret = 0;
3101 goto out_free;
3102 }
3103
25f84702 3104 ret = __evlist__parse_mmap_pages(&mmap_pages, p + 1);
e9db1310
AH		 3105 if (ret)
3106 goto out_free;
3107
3108 opts->auxtrace_mmap_pages = mmap_pages;
3109
3110out_free:
3111 free(s);
3112 return ret;
3113}
3114
7248e308
AT		 3115void __weak arch__add_leaf_frame_record_opts(struct record_opts *opts __maybe_unused)
3116{
3117}
3118
1d078ccb
AB		 3119static int parse_control_option(const struct option *opt,
3120 const char *str,
3121 int unset __maybe_unused)
3122{
9864a66d 3123 struct record_opts *opts = opt->value;
1d078ccb 3124
a8fcbd26
AH		 3125 return evlist__parse_control(str, &opts->ctl_fd, &opts->ctl_fd_ack, &opts->ctl_fd_close);
3126}
3127
0c582449
JO		 3128static void switch_output_size_warn(struct record *rec)
3129{
9521b5f2 3130 u64 wakeup_size = evlist__mmap_size(rec->opts.mmap_pages);
0c582449
JO		 3131 struct switch_output *s = &rec->switch_output;
3132
3133 wakeup_size /= 2;
3134
3135 if (s->size < wakeup_size) {
3136 char buf[100];
3137
3138 unit_number__scnprintf(buf, sizeof(buf), wakeup_size);
3139 pr_warning("WARNING: switch-output data size lower than "
3140 "wakeup kernel buffer size (%s) "
3141 "expect bigger perf.data sizes\n", buf);
3142 }
3143}
3144
cb4e1ebb
JO		 3145static int switch_output_setup(struct record *rec)
3146{
3147 struct switch_output *s = &rec->switch_output;
dc0c6127
JO		 3148 static struct parse_tag tags_size[] = {
3149 { .tag = 'B', .mult = 1 },
3150 { .tag = 'K', .mult = 1 << 10 },
3151 { .tag = 'M', .mult = 1 << 20 },
3152 { .tag = 'G', .mult = 1 << 30 },
3153 { .tag = 0 },
3154 };
bfacbe3b
JO		 3155 static struct parse_tag tags_time[] = {
3156 { .tag = 's', .mult = 1 },
3157 { .tag = 'm', .mult = 60 },
3158 { .tag = 'h', .mult = 60*60 },
3159 { .tag = 'd', .mult = 60*60*24 },
3160 { .tag = 0 },
3161 };
dc0c6127 3162 unsigned long val;
cb4e1ebb 3163
899e5ffb
ACM		 3164 /*
3165 * If we're using --switch-output-events, then we imply its
3166 * --switch-output=signal, as we'll send a SIGUSR2 from the side band
3167 * thread to its parent.
3168 */
b5f2511d
AB		 3169 if (rec->switch_output_event_set) {
3170 if (record__threads_enabled(rec)) {
3171 pr_warning("WARNING: --switch-output-event option is not available in parallel streaming mode.\n");
3172 return 0;
3173 }
899e5ffb 3174 goto do_signal;
b5f2511d 3175 }
899e5ffb 3176
cb4e1ebb
JO		 3177 if (!s->set)
3178 return 0;
3179
b5f2511d
AB		 3180 if (record__threads_enabled(rec)) {
3181 pr_warning("WARNING: --switch-output option is not available in parallel streaming mode.\n");
3182 return 0;
3183 }
3184
cb4e1ebb 3185 if (!strcmp(s->str, "signal")) {
899e5ffb 3186do_signal:
cb4e1ebb
JO		 3187 s->signal = true;
3188 pr_debug("switch-output with SIGUSR2 signal\n");
dc0c6127
JO		 3189 goto enabled;
3190 }
3191
3192 val = parse_tag_value(s->str, tags_size);
3193 if (val != (unsigned long) -1) {
3194 s->size = val;
3195 pr_debug("switch-output with %s size threshold\n", s->str);
3196 goto enabled;
cb4e1ebb
JO		 3197 }
3198
bfacbe3b
JO		 3199 val = parse_tag_value(s->str, tags_time);
3200 if (val != (unsigned long) -1) {
3201 s->time = val;
3202 pr_debug("switch-output with %s time threshold (%lu seconds)\n",
3203 s->str, s->time);
3204 goto enabled;
3205 }
3206
cb4e1ebb 3207 return -1;
dc0c6127
JO		 3208
3209enabled:
3210 rec->timestamp_filename = true;
3211 s->enabled = true;
0c582449
JO		 3212
3213 if (s->size && !rec->opts.no_buffering)
3214 switch_output_size_warn(rec);
3215
dc0c6127 3216 return 0;
cb4e1ebb
JO		 3217}
3218
e5b2c207 3219static const char * const __record_usage[] = {
9e096753
MG		 3220 "perf record [<options>] [<command>]",
3221 "perf record [<options>] -- <command> [<options>]",
0e9b20b8
IM		 3222 NULL
3223};
e5b2c207 3224const char * const *record_usage = __record_usage;
0e9b20b8 3225
6e0a9b3d
ACM		 3226static int build_id__process_mmap(struct perf_tool *tool, union perf_event *event,
3227 struct perf_sample *sample, struct machine *machine)
3228{
3229 /*
3230 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
3231 * no need to add them twice.
3232 */
3233 if (!(event->header.misc & PERF_RECORD_MISC_USER))
3234 return 0;
3235 return perf_event__process_mmap(tool, event, sample, machine);
3236}
3237
3238static int build_id__process_mmap2(struct perf_tool *tool, union perf_event *event,
3239 struct perf_sample *sample, struct machine *machine)
3240{
3241 /*
3242 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
3243 * no need to add them twice.
3244 */
3245 if (!(event->header.misc & PERF_RECORD_MISC_USER))
3246 return 0;
3247
3248 return perf_event__process_mmap2(tool, event, sample, machine);
3249}
3250
66286ed3
AH		 3251static int process_timestamp_boundary(struct perf_tool *tool,
3252 union perf_event *event __maybe_unused,
3253 struct perf_sample *sample,
3254 struct machine *machine __maybe_unused)
3255{
3256 struct record *rec = container_of(tool, struct record, tool);
3257
3258 set_timestamp_boundary(rec, sample->time);
3259 return 0;
3260}
3261
41b740b6
NK		 3262static int parse_record_synth_option(const struct option *opt,
3263 const char *str,
3264 int unset __maybe_unused)
3265{
3266 struct record_opts *opts = opt->value;
3267 char *p = strdup(str);
3268
3269 if (p == NULL)
3270 return -1;
3271
3272 opts->synth = parse_synth_opt(p);
3273 free(p);
3274
3275 if (opts->synth < 0) {
3276 pr_err("Invalid synth option: %s\n", str);
3277 return -1;
3278 }
3279 return 0;
3280}
3281
d20deb64 3282/*
8c6f45a7
ACM		 3283 * XXX Ideally would be local to cmd_record() and passed to a record__new
3284 * because we need to have access to it in record__exit, that is called
d20deb64
ACM		 3285 * after cmd_record() exits, but since record_options need to be accessible to
3286 * builtin-script, leave it here.
3287 *
3288 * At least we don't ouch it in all the other functions here directly.
3289 *
3290 * Just say no to tons of global variables, sigh.
3291 */
8c6f45a7 3292static struct record record = {
d20deb64 3293 .opts = {
8affc2b8 3294 .sample_time = true,
d20deb64
ACM		 3295 .mmap_pages = UINT_MAX,
3296 .user_freq = UINT_MAX,
3297 .user_interval = ULLONG_MAX,
447a6013 3298 .freq = 4000,
d1cb9fce
NK		 3299 .target = {
3300 .uses_mmap = true,
3aa5939d 3301 .default_per_cpu = true,
d1cb9fce 3302 },
470530bb 3303 .mmap_flush = MMAP_FLUSH_DEFAULT,
d99c22ea 3304 .nr_threads_synthesize = 1,
1d078ccb
AB		 3305 .ctl_fd = -1,
3306 .ctl_fd_ack = -1,
41b740b6 3307 .synth = PERF_SYNTH_ALL,
d20deb64 3308 },
e3d59112
NK		 3309 .tool = {
3310 .sample = process_sample_event,
3311 .fork = perf_event__process_fork,
cca8482c 3312 .exit = perf_event__process_exit,
e3d59112 3313 .comm = perf_event__process_comm,
f3b3614a 3314 .namespaces = perf_event__process_namespaces,
6e0a9b3d
ACM		 3315 .mmap = build_id__process_mmap,
3316 .mmap2 = build_id__process_mmap2,
66286ed3
AH		 3317 .itrace_start = process_timestamp_boundary,
3318 .aux = process_timestamp_boundary,
cca8482c 3319 .ordered_events = true,
e3d59112 3320 },
d20deb64 3321};
7865e817 3322
76a26549
NK		 3323const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
3324 "\n\t\t\t\tDefault: fp";
61eaa3be 3325
0aab2136
WN		 3326static bool dry_run;
3327
d20deb64
ACM		 3328/*
3329 * XXX Will stay a global variable till we fix builtin-script.c to stop messing
3330 * with it and switch to use the library functions in perf_evlist that came
b4006796 3331 * from builtin-record.c, i.e. use record_opts,
7b392ef0 3332 * evlist__prepare_workload, etc instead of fork+exec'in 'perf record',
d20deb64
ACM		 3333 * using pipes, etc.
3334 */
efd21307 3335static struct option __record_options[] = {
d20deb64 3336 OPT_CALLBACK('e', "event", &record.evlist, "event",
86847b62 3337 "event selector. use 'perf list' to list available events",
f120f9d5 3338 parse_events_option),
d20deb64 3339 OPT_CALLBACK(0, "filter", &record.evlist, "filter",
c171b552 3340 "event filter", parse_filter),
4ba1faa1
WN		 3341 OPT_CALLBACK_NOOPT(0, "exclude-perf", &record.evlist,
3342 NULL, "don't record events from perf itself",
3343 exclude_perf),
bea03405 3344 OPT_STRING('p', "pid", &record.opts.target.pid, "pid",
d6d901c2 3345 "record events on existing process id"),
bea03405 3346 OPT_STRING('t', "tid", &record.opts.target.tid, "tid",
d6d901c2 3347 "record events on existing thread id"),
d20deb64 3348 OPT_INTEGER('r', "realtime", &record.realtime_prio,
0e9b20b8 3349 "collect data with this RT SCHED_FIFO priority"),
509051ea 3350 OPT_BOOLEAN(0, "no-buffering", &record.opts.no_buffering,
acac03fa 3351 "collect data without buffering"),
d20deb64 3352 OPT_BOOLEAN('R', "raw-samples", &record.opts.raw_samples,
daac07b2 3353 "collect raw sample records from all opened counters"),
bea03405 3354 OPT_BOOLEAN('a', "all-cpus", &record.opts.target.system_wide,
0e9b20b8 3355 "system-wide collection from all CPUs"),
bea03405 3356 OPT_STRING('C', "cpu", &record.opts.target.cpu_list, "cpu",
c45c6ea2 3357 "list of cpus to monitor"),
d20deb64 3358 OPT_U64('c', "count", &record.opts.user_interval, "event period to sample"),
2d4f2799 3359 OPT_STRING('o', "output", &record.data.path, "file",
abaff32a 3360 "output file name"),
69e7e5b0
AH		 3361 OPT_BOOLEAN_SET('i', "no-inherit", &record.opts.no_inherit,
3362 &record.opts.no_inherit_set,
3363 "child tasks do not inherit counters"),
4ea648ae
WN		 3364 OPT_BOOLEAN(0, "tail-synthesize", &record.opts.tail_synthesize,
3365 "synthesize non-sample events at the end of output"),
626a6b78 3366 OPT_BOOLEAN(0, "overwrite", &record.opts.overwrite, "use overwrite mode"),
a060c1f1 3367 OPT_BOOLEAN(0, "no-bpf-event", &record.opts.no_bpf_event, "do not record bpf events"),
b09c2364
ACM		 3368 OPT_BOOLEAN(0, "strict-freq", &record.opts.strict_freq,
3369 "Fail if the specified frequency can't be used"),
67230479
ACM		 3370 OPT_CALLBACK('F', "freq", &record.opts, "freq or 'max'",
3371 "profile at this frequency",
3372 record__parse_freq),
e9db1310
AH		 3373 OPT_CALLBACK('m', "mmap-pages", &record.opts, "pages[,pages]",
3374 "number of mmap data pages and AUX area tracing mmap pages",
3375 record__parse_mmap_pages),
470530bb
AB		 3376 OPT_CALLBACK(0, "mmap-flush", &record.opts, "number",
3377 "Minimal number of bytes that is extracted from mmap data pages (default: 1)",
3378 record__mmap_flush_parse),
2ddd5c04 3379 OPT_CALLBACK_NOOPT('g', NULL, &callchain_param,
09b0fd45
JO		 3380 NULL, "enables call-graph recording" ,
3381 &record_callchain_opt),
3382 OPT_CALLBACK(0, "call-graph", &record.opts,
76a26549 3383 "record_mode[,record_size]", record_callchain_help,
09b0fd45 3384 &record_parse_callchain_opt),
c0555642 3385 OPT_INCR('v', "verbose", &verbose,
3da297a6 3386 "be more verbose (show counter open errors, etc)"),
a527c2c1 3387 OPT_BOOLEAN('q', "quiet", &quiet, "don't print any warnings or messages"),
d20deb64 3388 OPT_BOOLEAN('s', "stat", &record.opts.inherit_stat,
649c48a9 3389 "per thread counts"),
56100321 3390 OPT_BOOLEAN('d', "data", &record.opts.sample_address, "Record the sample addresses"),
3b0a5daa
KL		 3391 OPT_BOOLEAN(0, "phys-data", &record.opts.sample_phys_addr,
3392 "Record the sample physical addresses"),
542b88fd
KL		 3393 OPT_BOOLEAN(0, "data-page-size", &record.opts.sample_data_page_size,
3394 "Record the sampled data address data page size"),
c1de7f3d
KL		 3395 OPT_BOOLEAN(0, "code-page-size", &record.opts.sample_code_page_size,
3396 "Record the sampled code address (ip) page size"),
b6f35ed7 3397 OPT_BOOLEAN(0, "sample-cpu", &record.opts.sample_cpu, "Record the sample cpu"),
61110883
AH		 3398 OPT_BOOLEAN(0, "sample-identifier", &record.opts.sample_identifier,
3399 "Record the sample identifier"),
3abebc55
AH		 3400 OPT_BOOLEAN_SET('T', "timestamp", &record.opts.sample_time,
3401 &record.opts.sample_time_set,
3402 "Record the sample timestamps"),
f290aa1f
JO		 3403 OPT_BOOLEAN_SET('P', "period", &record.opts.period, &record.opts.period_set,
3404 "Record the sample period"),
d20deb64 3405 OPT_BOOLEAN('n', "no-samples", &record.opts.no_samples,
649c48a9 3406 "don't sample"),
d2db9a98
WN		 3407 OPT_BOOLEAN_SET('N', "no-buildid-cache", &record.no_buildid_cache,
3408 &record.no_buildid_cache_set,
3409 "do not update the buildid cache"),
3410 OPT_BOOLEAN_SET('B', "no-buildid", &record.no_buildid,
3411 &record.no_buildid_set,
3412 "do not collect buildids in perf.data"),
d20deb64 3413 OPT_CALLBACK('G', "cgroup", &record.evlist, "name",
023695d9
SE		 3414 "monitor event in cgroup name only",
3415 parse_cgroups),
6657a099
AH		 3416 OPT_CALLBACK('D', "delay", &record, "ms",
3417 "ms to wait before starting measurement after program start (-1: start with events disabled), "
3418 "or ranges of time to enable events e.g. '-D 10-20,30-40'",
3419 record__parse_event_enable_time),
eeb399b5 3420 OPT_BOOLEAN(0, "kcore", &record.opts.kcore, "copy /proc/kcore"),
bea03405
NK		 3421 OPT_STRING('u', "uid", &record.opts.target.uid_str, "user",
3422 "user to profile"),
a5aabdac
SE		 3423
3424 OPT_CALLBACK_NOOPT('b', "branch-any", &record.opts.branch_stack,
3425 "branch any", "sample any taken branches",
3426 parse_branch_stack),
3427
3428 OPT_CALLBACK('j', "branch-filter", &record.opts.branch_stack,
3429 "branch filter mask", "branch stack filter modes",
bdfebd84 3430 parse_branch_stack),
05484298
AK		 3431 OPT_BOOLEAN('W', "weight", &record.opts.sample_weight,
3432 "sample by weight (on special events only)"),
475eeab9
AK		 3433 OPT_BOOLEAN(0, "transaction", &record.opts.sample_transaction,
3434 "sample transaction flags (special events only)"),
3aa5939d
AH		 3435 OPT_BOOLEAN(0, "per-thread", &record.opts.target.per_thread,
3436 "use per-thread mmaps"),
bcc84ec6
SE		 3437 OPT_CALLBACK_OPTARG('I', "intr-regs", &record.opts.sample_intr_regs, NULL, "any register",
3438 "sample selected machine registers on interrupt,"
aeea9062 3439 " use '-I?' to list register names", parse_intr_regs),
84c41742
AK		 3440 OPT_CALLBACK_OPTARG(0, "user-regs", &record.opts.sample_user_regs, NULL, "any register",
3441 "sample selected machine registers on interrupt,"
aeea9062 3442 " use '--user-regs=?' to list register names", parse_user_regs),
85c273d2
AK		 3443 OPT_BOOLEAN(0, "running-time", &record.opts.running_time,
3444 "Record running/enabled time of read (:S) events"),
814c8c38
PZ		 3445 OPT_CALLBACK('k', "clockid", &record.opts,
3446 "clockid", "clockid to use for events, see clock_gettime()",
3447 parse_clockid),
2dd6d8a1
AH		 3448 OPT_STRING_OPTARG('S', "snapshot", &record.opts.auxtrace_snapshot_opts,
3449 "opts", "AUX area tracing Snapshot Mode", ""),
c0a6de06
AH		 3450 OPT_STRING_OPTARG(0, "aux-sample", &record.opts.auxtrace_sample_opts,
3451 "opts", "sample AUX area", ""),
3fcb10e4 3452 OPT_UINTEGER(0, "proc-map-timeout", &proc_map_timeout,
9d9cad76 3453 "per thread proc mmap processing timeout in ms"),
f3b3614a
HB		 3454 OPT_BOOLEAN(0, "namespaces", &record.opts.record_namespaces,
3455 "Record namespaces events"),
8fb4b679
NK		 3456 OPT_BOOLEAN(0, "all-cgroups", &record.opts.record_cgroup,
3457 "Record cgroup events"),
16b4b4e1
AH		 3458 OPT_BOOLEAN_SET(0, "switch-events", &record.opts.record_switch_events,
3459 &record.opts.record_switch_events_set,
3460 "Record context switch events"),
85723885
JO		 3461 OPT_BOOLEAN_FLAG(0, "all-kernel", &record.opts.all_kernel,
3462 "Configure all used events to run in kernel space.",
3463 PARSE_OPT_EXCLUSIVE),
3464 OPT_BOOLEAN_FLAG(0, "all-user", &record.opts.all_user,
3465 "Configure all used events to run in user space.",
3466 PARSE_OPT_EXCLUSIVE),
53651b28 3467 OPT_BOOLEAN(0, "kernel-callchains", &record.opts.kernel_callchains,
3468 "collect kernel callchains"),
3469 OPT_BOOLEAN(0, "user-callchains", &record.opts.user_callchains,
3470 "collect user callchains"),
71dc2326
WN		 3471 OPT_STRING(0, "clang-path", &llvm_param.clang_path, "clang path",
3472 "clang binary to use for compiling BPF scriptlets"),
3473 OPT_STRING(0, "clang-opt", &llvm_param.clang_opt, "clang options",
3474 "options passed to clang when compiling BPF scriptlets"),
7efe0e03
HK		 3475 OPT_STRING(0, "vmlinux", &symbol_conf.vmlinux_name,
3476 "file", "vmlinux pathname"),
6156681b
NK		 3477 OPT_BOOLEAN(0, "buildid-all", &record.buildid_all,
3478 "Record build-id of all DSOs regardless of hits"),
e29386c8
JO		 3479 OPT_BOOLEAN(0, "buildid-mmap", &record.buildid_mmap,
3480 "Record build-id in map events"),
ecfd7a9c
WN		 3481 OPT_BOOLEAN(0, "timestamp-filename", &record.timestamp_filename,
3482 "append timestamp to output filename"),
68588baf
JY		 3483 OPT_BOOLEAN(0, "timestamp-boundary", &record.timestamp_boundary,
3484 "Record timestamp boundary (time of first/last samples)"),
cb4e1ebb 3485 OPT_STRING_OPTARG_SET(0, "switch-output", &record.switch_output.str,
c38dab7d
AK		 3486 &record.switch_output.set, "signal or size[BKMG] or time[smhd]",
3487 "Switch output when receiving SIGUSR2 (signal) or cross a size or time threshold",
dc0c6127 3488 "signal"),
899e5ffb
ACM		 3489 OPT_CALLBACK_SET(0, "switch-output-event", &record.sb_evlist, &record.switch_output_event_set, "switch output event",
3490 "switch output event selector. use 'perf list' to list available events",
3491 parse_events_option_new_evlist),
03724b2e
AK		 3492 OPT_INTEGER(0, "switch-max-files", &record.switch_output.num_files,
3493 "Limit number of switch output generated files"),
0aab2136
WN		 3494 OPT_BOOLEAN(0, "dry-run", &dry_run,
3495 "Parse options then exit"),
d3d1af6f 3496#ifdef HAVE_AIO_SUPPORT
93f20c0f
AB		 3497 OPT_CALLBACK_OPTARG(0, "aio", &record.opts,
3498 &nr_cblocks_default, "n", "Use <n> control blocks in asynchronous trace writing mode (default: 1, max: 4)",
d3d1af6f
AB		 3499 record__aio_parse),
3500#endif
f4fe11b7
AB		 3501 OPT_CALLBACK(0, "affinity", &record.opts, "node|cpu",
3502 "Set affinity mask of trace reading thread to NUMA node cpu mask or cpu of processed mmap buffer",
3503 record__parse_affinity),
504c1ad1 3504#ifdef HAVE_ZSTD_SUPPORT
b5f2511d
AB		 3505 OPT_CALLBACK_OPTARG('z', "compression-level", &record.opts, &comp_level_default, "n",
3506 "Compress records using specified level (default: 1 - fastest compression, 22 - greatest compression)",
504c1ad1
AB		 3507 record__parse_comp_level),
3508#endif
6d575816
JS		 3509 OPT_CALLBACK(0, "max-size", &record.output_max_size,
3510 "size", "Limit the maximum size of the output file", parse_output_max_size),
d99c22ea
SE		 3511 OPT_UINTEGER(0, "num-thread-synthesize",
3512 &record.opts.nr_threads_synthesize,
3513 "number of threads to run for event synthesis"),
70943490
SE		 3514#ifdef HAVE_LIBPFM
3515 OPT_CALLBACK(0, "pfm-events", &record.evlist, "event",
3516 "libpfm4 event selector. use 'perf list' to list available events",
3517 parse_libpfm_events_option),
3518#endif
a8fcbd26 3519 OPT_CALLBACK(0, "control", &record.opts, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]",
d20aff15
AH		 3520 "Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events,\n"
3521 "\t\t\t 'snapshot': AUX area tracing snapshot).\n"
a8fcbd26
AH		 3522 "\t\t\t Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n"
3523 "\t\t\t Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.",
1d078ccb 3524 parse_control_option),
41b740b6
NK		 3525 OPT_CALLBACK(0, "synth", &record.opts, "no|all|task|mmap|cgroup",
3526 "Fine-tune event synthesis: default=all", parse_record_synth_option),
9bce13ea
JO		 3527 OPT_STRING_OPTARG_SET(0, "debuginfod", &record.debuginfod.urls,
3528 &record.debuginfod.set, "debuginfod urls",
3529 "Enable debuginfod data retrieval from DEBUGINFOD_URLS or specified urls",
3530 "system"),
06380a84
AB		 3531 OPT_CALLBACK_OPTARG(0, "threads", &record.opts, NULL, "spec",
3532 "write collected trace data into several data files using parallel threads",
3533 record__parse_threads),
edc41a10 3534 OPT_BOOLEAN(0, "off-cpu", &record.off_cpu, "Enable off-cpu analysis"),
0e9b20b8
IM		 3535 OPT_END()
3536};
3537
e5b2c207
NK		 3538struct option *record_options = __record_options;
3539
cbd7bfc7 3540static int record__mmap_cpu_mask_init(struct mmap_cpu_mask *mask, struct perf_cpu_map *cpus)
7954f716 3541{
0255571a
IR		 3542 struct perf_cpu cpu;
3543 int idx;
7954f716 3544
23380e4d 3545 if (cpu_map__is_dummy(cpus))
cbd7bfc7 3546 return 0;
23380e4d 3547
cbd7bfc7 3548 perf_cpu_map__for_each_cpu(cpu, idx, cpus) {
ca76d7d2
AH		 3549 if (cpu.cpu == -1)
3550 continue;
cbd7bfc7
AR		 3551 /* Return ENODEV is input cpu is greater than max cpu */
3552 if ((unsigned long)cpu.cpu > mask->nbits)
3553 return -ENODEV;
49bd97c2 3554 __set_bit(cpu.cpu, mask->bits);
cbd7bfc7
AR		 3555 }
3556
3557 return 0;
7954f716
AB		 3558}
3559
f466e5ed
AB		 3560static int record__mmap_cpu_mask_init_spec(struct mmap_cpu_mask *mask, const char *mask_spec)
3561{
3562 struct perf_cpu_map *cpus;
3563
3564 cpus = perf_cpu_map__new(mask_spec);
3565 if (!cpus)
3566 return -ENOMEM;
3567
3568 bitmap_zero(mask->bits, mask->nbits);
cbd7bfc7
AR		 3569 if (record__mmap_cpu_mask_init(mask, cpus))
3570 return -ENODEV;
3571
f466e5ed
AB		 3572 perf_cpu_map__put(cpus);
3573
3574 return 0;
3575}
3576
7954f716
AB		 3577static void record__free_thread_masks(struct record *rec, int nr_threads)
3578{
3579 int t;
3580
3581 if (rec->thread_masks)
3582 for (t = 0; t < nr_threads; t++)
3583 record__thread_mask_free(&rec->thread_masks[t]);
3584
3585 zfree(&rec->thread_masks);
3586}
3587
3588static int record__alloc_thread_masks(struct record *rec, int nr_threads, int nr_bits)
3589{
3590 int t, ret;
3591
3592 rec->thread_masks = zalloc(nr_threads * sizeof(*(rec->thread_masks)));
3593 if (!rec->thread_masks) {
3594 pr_err("Failed to allocate thread masks\n");
3595 return -ENOMEM;
3596 }
3597
3598 for (t = 0; t < nr_threads; t++) {
3599 ret = record__thread_mask_alloc(&rec->thread_masks[t], nr_bits);
3600 if (ret) {
3601 pr_err("Failed to allocate thread masks[%d]\n", t);
3602 goto out_free;
3603 }
3604 }
3605
3606 return 0;
3607
3608out_free:
3609 record__free_thread_masks(rec, nr_threads);
3610
3611 return ret;
3612}
3613
06380a84
AB		 3614static int record__init_thread_cpu_masks(struct record *rec, struct perf_cpu_map *cpus)
3615{
3616 int t, ret, nr_cpus = perf_cpu_map__nr(cpus);
3617
3618 ret = record__alloc_thread_masks(rec, nr_cpus, cpu__max_cpu().cpu);
3619 if (ret)
3620 return ret;
3621
3622 rec->nr_threads = nr_cpus;
3623 pr_debug("nr_threads: %d\n", rec->nr_threads);
3624
3625 for (t = 0; t < rec->nr_threads; t++) {
49bd97c2
SC		 3626 __set_bit(perf_cpu_map__cpu(cpus, t).cpu, rec->thread_masks[t].maps.bits);
3627 __set_bit(perf_cpu_map__cpu(cpus, t).cpu, rec->thread_masks[t].affinity.bits);
7c0a6144 3628 if (verbose > 0) {
06380a84
AB		 3629 pr_debug("thread_masks[%d]: ", t);
3630 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].maps, "maps");
3631 pr_debug("thread_masks[%d]: ", t);
3632 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].affinity, "affinity");
3633 }
3634 }
3635
3636 return 0;
3637}
3638
f466e5ed
AB		 3639static int record__init_thread_masks_spec(struct record *rec, struct perf_cpu_map *cpus,
3640 const char **maps_spec, const char **affinity_spec,
3641 u32 nr_spec)
3642{
3643 u32 s;
3644 int ret = 0, t = 0;
3645 struct mmap_cpu_mask cpus_mask;
3646 struct thread_mask thread_mask, full_mask, *thread_masks;
3647
3648 ret = record__mmap_cpu_mask_alloc(&cpus_mask, cpu__max_cpu().cpu);
3649 if (ret) {
3650 pr_err("Failed to allocate CPUs mask\n");
3651 return ret;
3652 }
cbd7bfc7
AR		 3653
3654 ret = record__mmap_cpu_mask_init(&cpus_mask, cpus);
3655 if (ret) {
3656 pr_err("Failed to init cpu mask\n");
3657 goto out_free_cpu_mask;
3658 }
f466e5ed
AB		 3659
3660 ret = record__thread_mask_alloc(&full_mask, cpu__max_cpu().cpu);
3661 if (ret) {
3662 pr_err("Failed to allocate full mask\n");
3663 goto out_free_cpu_mask;
3664 }
3665
3666 ret = record__thread_mask_alloc(&thread_mask, cpu__max_cpu().cpu);
3667 if (ret) {
3668 pr_err("Failed to allocate thread mask\n");
3669 goto out_free_full_and_cpu_masks;
3670 }
3671
3672 for (s = 0; s < nr_spec; s++) {
3673 ret = record__mmap_cpu_mask_init_spec(&thread_mask.maps, maps_spec[s]);
3674 if (ret) {
3675 pr_err("Failed to initialize maps thread mask\n");
3676 goto out_free;
3677 }
3678 ret = record__mmap_cpu_mask_init_spec(&thread_mask.affinity, affinity_spec[s]);
3679 if (ret) {
3680 pr_err("Failed to initialize affinity thread mask\n");
3681 goto out_free;
3682 }
3683
3684 /* ignore invalid CPUs but do not allow empty masks */
3685 if (!bitmap_and(thread_mask.maps.bits, thread_mask.maps.bits,
3686 cpus_mask.bits, thread_mask.maps.nbits)) {
3687 pr_err("Empty maps mask: %s\n", maps_spec[s]);
3688 ret = -EINVAL;
3689 goto out_free;
3690 }
3691 if (!bitmap_and(thread_mask.affinity.bits, thread_mask.affinity.bits,
3692 cpus_mask.bits, thread_mask.affinity.nbits)) {
3693 pr_err("Empty affinity mask: %s\n", affinity_spec[s]);
3694 ret = -EINVAL;
3695 goto out_free;
3696 }
3697
3698 /* do not allow intersection with other masks (full_mask) */
3699 if (bitmap_intersects(thread_mask.maps.bits, full_mask.maps.bits,
3700 thread_mask.maps.nbits)) {
3701 pr_err("Intersecting maps mask: %s\n", maps_spec[s]);
3702 ret = -EINVAL;
3703 goto out_free;
3704 }
3705 if (bitmap_intersects(thread_mask.affinity.bits, full_mask.affinity.bits,
3706 thread_mask.affinity.nbits)) {
3707 pr_err("Intersecting affinity mask: %s\n", affinity_spec[s]);
3708 ret = -EINVAL;
3709 goto out_free;
3710 }
3711
3712 bitmap_or(full_mask.maps.bits, full_mask.maps.bits,
3713 thread_mask.maps.bits, full_mask.maps.nbits);
3714 bitmap_or(full_mask.affinity.bits, full_mask.affinity.bits,
3715 thread_mask.affinity.bits, full_mask.maps.nbits);
3716
3717 thread_masks = realloc(rec->thread_masks, (t + 1) * sizeof(struct thread_mask));
3718 if (!thread_masks) {
3719 pr_err("Failed to reallocate thread masks\n");
3720 ret = -ENOMEM;
3721 goto out_free;
3722 }
3723 rec->thread_masks = thread_masks;
3724 rec->thread_masks[t] = thread_mask;
7c0a6144 3725 if (verbose > 0) {
f466e5ed
AB		 3726 pr_debug("thread_masks[%d]: ", t);
3727 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].maps, "maps");
3728 pr_debug("thread_masks[%d]: ", t);
3729 mmap_cpu_mask__scnprintf(&rec->thread_masks[t].affinity, "affinity");
3730 }
3731 t++;
3732 ret = record__thread_mask_alloc(&thread_mask, cpu__max_cpu().cpu);
3733 if (ret) {
3734 pr_err("Failed to allocate thread mask\n");
3735 goto out_free_full_and_cpu_masks;
3736 }
3737 }
3738 rec->nr_threads = t;
3739 pr_debug("nr_threads: %d\n", rec->nr_threads);
3740 if (!rec->nr_threads)
3741 ret = -EINVAL;
3742
3743out_free:
3744 record__thread_mask_free(&thread_mask);
3745out_free_full_and_cpu_masks:
3746 record__thread_mask_free(&full_mask);
3747out_free_cpu_mask:
3748 record__mmap_cpu_mask_free(&cpus_mask);
3749
3750 return ret;
3751}
3752
3753static int record__init_thread_core_masks(struct record *rec, struct perf_cpu_map *cpus)
3754{
3755 int ret;
3756 struct cpu_topology *topo;
3757
3758 topo = cpu_topology__new();
3759 if (!topo) {
3760 pr_err("Failed to allocate CPU topology\n");
3761 return -ENOMEM;
3762 }
3763
3764 ret = record__init_thread_masks_spec(rec, cpus, topo->core_cpus_list,
3765 topo->core_cpus_list, topo->core_cpus_lists);
3766 cpu_topology__delete(topo);
3767
3768 return ret;
3769}
3770
3771static int record__init_thread_package_masks(struct record *rec, struct perf_cpu_map *cpus)
3772{
3773 int ret;
3774 struct cpu_topology *topo;
3775
3776 topo = cpu_topology__new();
3777 if (!topo) {
3778 pr_err("Failed to allocate CPU topology\n");
3779 return -ENOMEM;
3780 }
3781
3782 ret = record__init_thread_masks_spec(rec, cpus, topo->package_cpus_list,
3783 topo->package_cpus_list, topo->package_cpus_lists);
3784 cpu_topology__delete(topo);
3785
3786 return ret;
3787}
3788
3789static int record__init_thread_numa_masks(struct record *rec, struct perf_cpu_map *cpus)
3790{
3791 u32 s;
3792 int ret;
3793 const char **spec;
3794 struct numa_topology *topo;
3795
3796 topo = numa_topology__new();
3797 if (!topo) {
3798 pr_err("Failed to allocate NUMA topology\n");
3799 return -ENOMEM;
3800 }
3801
3802 spec = zalloc(topo->nr * sizeof(char *));
3803 if (!spec) {
3804 pr_err("Failed to allocate NUMA spec\n");
3805 ret = -ENOMEM;
3806 goto out_delete_topo;
3807 }
3808 for (s = 0; s < topo->nr; s++)
3809 spec[s] = topo->nodes[s].cpus;
3810
3811 ret = record__init_thread_masks_spec(rec, cpus, spec, spec, topo->nr);
3812
3813 zfree(&spec);
3814
3815out_delete_topo:
3816 numa_topology__delete(topo);
3817
3818 return ret;
3819}
3820
3821static int record__init_thread_user_masks(struct record *rec, struct perf_cpu_map *cpus)
3822{
3823 int t, ret;
3824 u32 s, nr_spec = 0;
3825 char **maps_spec = NULL, **affinity_spec = NULL, **tmp_spec;
3826 char *user_spec, *spec, *spec_ptr, *mask, *mask_ptr, *dup_mask = NULL;
3827
3828 for (t = 0, user_spec = (char *)rec->opts.threads_user_spec; ; t++, user_spec = NULL) {
3829 spec = strtok_r(user_spec, ":", &spec_ptr);
3830 if (spec == NULL)
3831 break;
3832 pr_debug2("threads_spec[%d]: %s\n", t, spec);
3833 mask = strtok_r(spec, "/", &mask_ptr);
3834 if (mask == NULL)
3835 break;
3836 pr_debug2(" maps mask: %s\n", mask);
3837 tmp_spec = realloc(maps_spec, (nr_spec + 1) * sizeof(char *));
3838 if (!tmp_spec) {
3839 pr_err("Failed to reallocate maps spec\n");
3840 ret = -ENOMEM;
3841 goto out_free;
3842 }
3843 maps_spec = tmp_spec;
3844 maps_spec[nr_spec] = dup_mask = strdup(mask);
3845 if (!maps_spec[nr_spec]) {
3846 pr_err("Failed to allocate maps spec[%d]\n", nr_spec);
3847 ret = -ENOMEM;
3848 goto out_free;
3849 }
3850 mask = strtok_r(NULL, "/", &mask_ptr);
3851 if (mask == NULL) {
3852 pr_err("Invalid thread maps or affinity specs\n");
3853 ret = -EINVAL;
3854 goto out_free;
3855 }
3856 pr_debug2(" affinity mask: %s\n", mask);
3857 tmp_spec = realloc(affinity_spec, (nr_spec + 1) * sizeof(char *));
3858 if (!tmp_spec) {
3859 pr_err("Failed to reallocate affinity spec\n");
3860 ret = -ENOMEM;
3861 goto out_free;
3862 }
3863 affinity_spec = tmp_spec;
3864 affinity_spec[nr_spec] = strdup(mask);
3865 if (!affinity_spec[nr_spec]) {
3866 pr_err("Failed to allocate affinity spec[%d]\n", nr_spec);
3867 ret = -ENOMEM;
3868 goto out_free;
3869 }
3870 dup_mask = NULL;
3871 nr_spec++;
3872 }
3873
3874 ret = record__init_thread_masks_spec(rec, cpus, (const char **)maps_spec,
3875 (const char **)affinity_spec, nr_spec);
3876
3877out_free:
3878 free(dup_mask);
3879 for (s = 0; s < nr_spec; s++) {
3880 if (maps_spec)
3881 free(maps_spec[s]);
3882 if (affinity_spec)
3883 free(affinity_spec[s]);
3884 }
3885 free(affinity_spec);
3886 free(maps_spec);
3887
3888 return ret;
3889}
3890
7954f716
AB		 3891static int record__init_thread_default_masks(struct record *rec, struct perf_cpu_map *cpus)
3892{
3893 int ret;
3894
3895 ret = record__alloc_thread_masks(rec, 1, cpu__max_cpu().cpu);
3896 if (ret)
3897 return ret;
3898
cbd7bfc7
AR		 3899 if (record__mmap_cpu_mask_init(&rec->thread_masks->maps, cpus))
3900 return -ENODEV;
7954f716
AB		 3901
3902 rec->nr_threads = 1;
3903
3904 return 0;
3905}
3906
3907static int record__init_thread_masks(struct record *rec)
3908{
f466e5ed 3909 int ret = 0;
7be1fedd 3910 struct perf_cpu_map *cpus = rec->evlist->core.all_cpus;
7954f716 3911
06380a84
AB		 3912 if (!record__threads_enabled(rec))
3913 return record__init_thread_default_masks(rec, cpus);
3914
7be1fedd 3915 if (evlist__per_thread(rec->evlist)) {
23380e4d
AB		 3916 pr_err("--per-thread option is mutually exclusive to parallel streaming mode.\n");
3917 return -EINVAL;
3918 }
3919
f466e5ed
AB		 3920 switch (rec->opts.threads_spec) {
3921 case THREAD_SPEC__CPU:
3922 ret = record__init_thread_cpu_masks(rec, cpus);
3923 break;
3924 case THREAD_SPEC__CORE:
3925 ret = record__init_thread_core_masks(rec, cpus);
3926 break;
3927 case THREAD_SPEC__PACKAGE:
3928 ret = record__init_thread_package_masks(rec, cpus);
3929 break;
3930 case THREAD_SPEC__NUMA:
3931 ret = record__init_thread_numa_masks(rec, cpus);
3932 break;
3933 case THREAD_SPEC__USER:
3934 ret = record__init_thread_user_masks(rec, cpus);
3935 break;
3936 default:
3937 break;
3938 }
3939
3940 return ret;
7954f716
AB		 3941}
3942
b0ad8ea6 3943int cmd_record(int argc, const char **argv)
0e9b20b8 3944{
ef149c25 3945 int err;
8c6f45a7 3946 struct record *rec = &record;
16ad2ffb 3947 char errbuf[BUFSIZ];
0e9b20b8 3948
67230479
ACM		 3949 setlocale(LC_ALL, "");
3950
48e1cab1
WN		 3951#ifndef HAVE_LIBBPF_SUPPORT
3952# define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, "NO_LIBBPF=1", c)
3953 set_nobuild('\0', "clang-path", true);
3954 set_nobuild('\0', "clang-opt", true);
3955# undef set_nobuild
7efe0e03
HK		 3956#endif
3957
3958#ifndef HAVE_BPF_PROLOGUE
3959# if !defined (HAVE_DWARF_SUPPORT)
3960# define REASON "NO_DWARF=1"
3961# elif !defined (HAVE_LIBBPF_SUPPORT)
3962# define REASON "NO_LIBBPF=1"
3963# else
3964# define REASON "this architecture doesn't support BPF prologue"
3965# endif
3966# define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, REASON, c)
3967 set_nobuild('\0', "vmlinux", true);
3968# undef set_nobuild
3969# undef REASON
edc41a10
NK		 3970#endif
3971
3972#ifndef HAVE_BPF_SKEL
3973# define set_nobuild(s, l, m, c) set_option_nobuild(record_options, s, l, m, c)
3974 set_nobuild('\0', "off-cpu", "no BUILD_BPF_SKEL=1", true);
3975# undef set_nobuild
48e1cab1
WN		 3976#endif
3977
9d2ed645
AB		 3978 rec->opts.affinity = PERF_AFFINITY_SYS;
3979
0f98b11c 3980 rec->evlist = evlist__new();
3e2be2da 3981 if (rec->evlist == NULL)
361c99a6
ACM		 3982 return -ENOMEM;
3983
ecc4c561
ACM		 3984 err = perf_config(perf_record_config, rec);
3985 if (err)
3986 return err;
eb853e80 3987
bca647aa 3988 argc = parse_options(argc, argv, record_options, record_usage,
655000e7 3989 PARSE_OPT_STOP_AT_NON_OPTION);
68ba3235
NK		 3990 if (quiet)
3991 perf_quiet_option();
483635a9 3992
7cc72553
JC		 3993 err = symbol__validate_sym_arguments();
3994 if (err)
3995 return err;
3996
9bce13ea
JO		 3997 perf_debuginfod_setup(&record.debuginfod);
3998
483635a9 3999 /* Make system wide (-a) the default target. */
602ad878 4000 if (!argc && target__none(&rec->opts.target))
483635a9 4001 rec->opts.target.system_wide = true;
0e9b20b8 4002
bea03405 4003 if (nr_cgroups && !rec->opts.target.system_wide) {
c7118369
NK		 4004 usage_with_options_msg(record_usage, record_options,
4005 "cgroup monitoring only available in system-wide mode");
4006
023695d9 4007 }
504c1ad1 4008
e29386c8
JO		 4009 if (rec->buildid_mmap) {
4010 if (!perf_can_record_build_id()) {
4011 pr_err("Failed: no support to record build id in mmap events, update your kernel.\n");
4012 err = -EINVAL;
4013 goto out_opts;
4014 }
4015 pr_debug("Enabling build id in mmap2 events.\n");
4016 /* Enable mmap build id synthesizing. */
4017 symbol_conf.buildid_mmap2 = true;
4018 /* Enable perf_event_attr::build_id bit. */
4019 rec->opts.build_id = true;
4020 /* Disable build id cache. */
4021 rec->no_buildid = true;
4022 }
4023
4f2abe91
NK		 4024 if (rec->opts.record_cgroup && !perf_can_record_cgroup()) {
4025 pr_err("Kernel has no cgroup sampling support.\n");
4026 err = -EINVAL;
4027 goto out_opts;
4028 }
4029
f42c0ce5
AH		 4030 if (rec->opts.kcore)
4031 rec->opts.text_poke = true;
4032
56f735ff 4033 if (rec->opts.kcore || record__threads_enabled(rec))
eeb399b5
AH		 4034 rec->data.is_dir = true;
4035
b5f2511d
AB		 4036 if (record__threads_enabled(rec)) {
4037 if (rec->opts.affinity != PERF_AFFINITY_SYS) {
4038 pr_err("--affinity option is mutually exclusive to parallel streaming mode.\n");
4039 goto out_opts;
4040 }
4041 if (record__aio_enabled(rec)) {
4042 pr_err("Asynchronous streaming mode (--aio) is mutually exclusive to parallel streaming mode.\n");
4043 goto out_opts;
4044 }
4045 }
4046
504c1ad1
AB		 4047 if (rec->opts.comp_level != 0) {
4048 pr_debug("Compression enabled, disabling build id collection at the end of the session.\n");
4049 rec->no_buildid = true;
4050 }
4051
b757bb09
AH		 4052 if (rec->opts.record_switch_events &&
4053 !perf_can_record_switch_events()) {
c7118369
NK		 4054 ui__error("kernel does not support recording context switch events\n");
4055 parse_options_usage(record_usage, record_options, "switch-events", 0);
a8fcbd26
AH		 4056 err = -EINVAL;
4057 goto out_opts;
b757bb09 4058 }
023695d9 4059
cb4e1ebb
JO		 4060 if (switch_output_setup(rec)) {
4061 parse_options_usage(record_usage, record_options, "switch-output", 0);
a8fcbd26
AH		 4062 err = -EINVAL;
4063 goto out_opts;
cb4e1ebb
JO		 4064 }
4065
bfacbe3b
JO		 4066 if (rec->switch_output.time) {
4067 signal(SIGALRM, alarm_sig_handler);
4068 alarm(rec->switch_output.time);
4069 }
4070
03724b2e
AK		 4071 if (rec->switch_output.num_files) {
4072 rec->switch_output.filenames = calloc(sizeof(char *),
4073 rec->switch_output.num_files);
a8fcbd26
AH		 4074 if (!rec->switch_output.filenames) {
4075 err = -EINVAL;
4076 goto out_opts;
4077 }
03724b2e
AK		 4078 }
4079
b5f2511d
AB		 4080 if (rec->timestamp_filename && record__threads_enabled(rec)) {
4081 rec->timestamp_filename = false;
4082 pr_warning("WARNING: --timestamp-filename option is not available in parallel streaming mode.\n");
4083 }
4084
1b36c03e
AH		 4085 /*
4086 * Allow aliases to facilitate the lookup of symbols for address
4087 * filters. Refer to auxtrace_parse_filters().
4088 */
4089 symbol_conf.allow_aliases = true;
4090
4091 symbol__init(NULL);
4092
4b5ea3bd 4093 err = record__auxtrace_init(rec);
1b36c03e
AH		 4094 if (err)
4095 goto out;
4096
0aab2136 4097 if (dry_run)
5c01ad60 4098 goto out;
0aab2136 4099
d7888573
WN		 4100 err = bpf__setup_stdout(rec->evlist);
4101 if (err) {
4102 bpf__strerror_setup_stdout(rec->evlist, err, errbuf, sizeof(errbuf));
4103 pr_err("ERROR: Setup BPF stdout failed: %s\n",
4104 errbuf);
5c01ad60 4105 goto out;
d7888573
WN		 4106 }
4107
ef149c25
AH		 4108 err = -ENOMEM;
4109
0c1d46a8 4110 if (rec->no_buildid_cache || rec->no_buildid) {
a1ac1d3c 4111 disable_buildid_cache();
dc0c6127 4112 } else if (rec->switch_output.enabled) {
0c1d46a8
WN		 4113 /*
4114 * In 'perf record --switch-output', disable buildid
4115 * generation by default to reduce data file switching
4116 * overhead. Still generate buildid if they are required
4117 * explicitly using
4118 *
60437ac0 4119 * perf record --switch-output --no-no-buildid \
0c1d46a8
WN		 4120 * --no-no-buildid-cache
4121 *
4122 * Following code equals to:
4123 *
4124 * if ((rec->no_buildid || !rec->no_buildid_set) &&
4125 * (rec->no_buildid_cache || !rec->no_buildid_cache_set))
4126 * disable_buildid_cache();
4127 */
4128 bool disable = true;
4129
4130 if (rec->no_buildid_set && !rec->no_buildid)
4131 disable = false;
4132 if (rec->no_buildid_cache_set && !rec->no_buildid_cache)
4133 disable = false;
4134 if (disable) {
4135 rec->no_buildid = true;
4136 rec->no_buildid_cache = true;
4137 disable_buildid_cache();
4138 }
4139 }
655000e7 4140
4ea648ae
WN		 4141 if (record.opts.overwrite)
4142 record.opts.tail_synthesize = true;
4143
b53a0755
JY		 4144 if (rec->evlist->core.nr_entries == 0) {
4145 if (perf_pmu__has_hybrid()) {
4146 err = evlist__add_default_hybrid(rec->evlist,
4147 !record.opts.no_samples);
4148 } else {
4149 err = __evlist__add_default(rec->evlist,
4150 !record.opts.no_samples);
4151 }
4152
4153 if (err < 0) {
4154 pr_err("Not enough memory for event selector list\n");
4155 goto out;
4156 }
bbd36e5e 4157 }
0e9b20b8 4158
69e7e5b0
AH		 4159 if (rec->opts.target.tid && !rec->opts.no_inherit_set)
4160 rec->opts.no_inherit = true;
4161
602ad878 4162 err = target__validate(&rec->opts.target);
16ad2ffb 4163 if (err) {
602ad878 4164 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
c3dec27b 4165 ui__warning("%s\n", errbuf);
16ad2ffb
NK		 4166 }
4167
602ad878 4168 err = target__parse_uid(&rec->opts.target);
16ad2ffb
NK		 4169 if (err) {
4170 int saved_errno = errno;
4bd0f2d2 4171
602ad878 4172 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
3780f488 4173 ui__error("%s", errbuf);
16ad2ffb
NK		 4174
4175 err = -saved_errno;
394c01ed 4176 goto out;
16ad2ffb 4177 }
0d37aa34 4178
ca800068
MZ		 4179 /* Enable ignoring missing threads when -u/-p option is defined. */
4180 rec->opts.ignore_missing_thread = rec->opts.target.uid != UINT_MAX || rec->opts.target.pid;
23dc4f15 4181
1d3351e6
JY		 4182 if (evlist__fix_hybrid_cpus(rec->evlist, rec->opts.target.cpu_list)) {
4183 pr_err("failed to use cpu list %s\n",
4184 rec->opts.target.cpu_list);
4185 goto out;
4186 }
4187
4188 rec->opts.target.hybrid = perf_pmu__has_hybrid();
7248e308
AT		 4189
4190 if (callchain_param.enabled && callchain_param.record_mode == CALLCHAIN_FP)
4191 arch__add_leaf_frame_record_opts(&rec->opts);
4192
16ad2ffb 4193 err = -ENOMEM;
1bf7d836
ML		 4194 if (evlist__create_maps(rec->evlist, &rec->opts.target) < 0) {
4195 if (rec->opts.target.pid != NULL) {
4196 pr_err("Couldn't create thread/CPU maps: %s\n",
4197 errno == ENOENT ? "No such process" : str_error_r(errno, errbuf, sizeof(errbuf)));
4198 goto out;
4199 }
4200 else
4201 usage_with_options(record_usage, record_options);
4202 }
69aad6f1 4203
ef149c25
AH		 4204 err = auxtrace_record__options(rec->itr, rec->evlist, &rec->opts);
4205 if (err)
394c01ed 4206 goto out;
ef149c25 4207
6156681b
NK		 4208 /*
4209 * We take all buildids when the file contains
4210 * AUX area tracing data because we do not decode the
4211 * trace because it would take too long.
4212 */
4213 if (rec->opts.full_auxtrace)
4214 rec->buildid_all = true;
4215
246eba8e
AH		 4216 if (rec->opts.text_poke) {
4217 err = record__config_text_poke(rec->evlist);
4218 if (err) {
4219 pr_err("record__config_text_poke failed, error %d\n", err);
4220 goto out;
4221 }
4222 }
4223
edc41a10
NK		 4224 if (rec->off_cpu) {
4225 err = record__config_off_cpu(rec);
4226 if (err) {
4227 pr_err("record__config_off_cpu failed, error %d\n", err);
4228 goto out;
4229 }
4230 }
4231
b4006796 4232 if (record_opts__config(&rec->opts)) {
39d17dac 4233 err = -EINVAL;
394c01ed 4234 goto out;
7e4ff9e3
MG		 4235 }
4236
7954f716
AB		 4237 err = record__init_thread_masks(rec);
4238 if (err) {
4239 pr_err("Failed to initialize parallel data streaming masks\n");
4240 goto out;
4241 }
4242
93f20c0f
AB		 4243 if (rec->opts.nr_cblocks > nr_cblocks_max)
4244 rec->opts.nr_cblocks = nr_cblocks_max;
5d7f4116 4245 pr_debug("nr_cblocks: %d\n", rec->opts.nr_cblocks);
d3d1af6f 4246
9d2ed645 4247 pr_debug("affinity: %s\n", affinity_tags[rec->opts.affinity]);
470530bb 4248 pr_debug("mmap flush: %d\n", rec->opts.mmap_flush);
9d2ed645 4249
51255a8a
AB		 4250 if (rec->opts.comp_level > comp_level_max)
4251 rec->opts.comp_level = comp_level_max;
4252 pr_debug("comp level: %d\n", rec->opts.comp_level);
4253
d20deb64 4254 err = __cmd_record(&record, argc, argv);
394c01ed 4255out:
c12995a5 4256 evlist__delete(rec->evlist);
d65a458b 4257 symbol__exit();
ef149c25 4258 auxtrace_record__free(rec->itr);
a8fcbd26 4259out_opts:
7954f716
AB		 4260 record__free_thread_masks(rec, rec->nr_threads);
4261 rec->nr_threads = 0;
ee7fe31e 4262 evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close);
39d17dac 4263 return err;
0e9b20b8 4264}
2dd6d8a1
AH		 4265
4266static void snapshot_sig_handler(int sig __maybe_unused)
4267{
dc0c6127
JO		 4268 struct record *rec = &record;
4269
d20aff15 4270 hit_auxtrace_snapshot_trigger(rec);
3c1cb7e3 4271
dc0c6127 4272 if (switch_output_signal(rec))
3c1cb7e3 4273 trigger_hit(&switch_output_trigger);
2dd6d8a1 4274}
bfacbe3b
JO		 4275
4276static void alarm_sig_handler(int sig __maybe_unused)
4277{
4278 struct record *rec = &record;
4279
4280 if (switch_output_time(rec))
4281 trigger_hit(&switch_output_trigger);
4282}
Linux 6.x block layer and io_uring tree(s)