projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
perf record: Introduce data transferred and compressed stats
[linux-2.6-block.git] / tools / perf / builtin-record.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
abaff32a 2/*
bf9e1876
IM		 3 * builtin-record.c
4 *
5 * Builtin record command: Record the profile of a workload
6 * (or a CPU, or a PID) into the perf.data output file - for
7 * later analysis via perf report.
abaff32a 8 */
16f762a2 9#include "builtin.h"
bf9e1876 10
6122e4e4 11#include "util/build-id.h"
4b6ab94e 12#include <subcmd/parse-options.h>
8ad8db37 13#include "util/parse-events.h"
41840d21 14#include "util/config.h"
6eda5838 15
8f651eae 16#include "util/callchain.h"
f14d5707 17#include "util/cgroup.h"
7c6a1c65 18#include "util/header.h"
66e274f3 19#include "util/event.h"
361c99a6 20#include "util/evlist.h"
69aad6f1 21#include "util/evsel.h"
8f28827a 22#include "util/debug.h"
e0fcfb08 23#include "util/mmap.h"
aeb00b1a 24#include "util/target.h"
94c744b6 25#include "util/session.h"
45694aa7 26#include "util/tool.h"
8d06367f 27#include "util/symbol.h"
aeb00b1a 28#include "util/record.h"
a12b51c4 29#include "util/cpumap.h"
fd78260b 30#include "util/thread_map.h"
f5fc1412 31#include "util/data.h"
bcc84ec6 32#include "util/perf_regs.h"
ef149c25 33#include "util/auxtrace.h"
46bc29b9 34#include "util/tsc.h"
f00898f4 35#include "util/parse-branch-options.h"
bcc84ec6 36#include "util/parse-regs-options.h"
40c7d246 37#include "util/perf_api_probe.h"
71dc2326 38#include "util/llvm-utils.h"
8690a2a7 39#include "util/bpf-loader.h"
5f9cf599 40#include "util/trigger.h"
a074865e 41#include "util/perf-hooks.h"
f13de660 42#include "util/cpu-set-sched.h"
ea49e01c 43#include "util/synthetic-events.h"
c5e4027e 44#include "util/time-utils.h"
58db1d6e 45#include "util/units.h"
7b612e29 46#include "util/bpf-event.h"
d99c22ea 47#include "util/util.h"
70943490 48#include "util/pfm.h"
6953beb4 49#include "util/clockid.h"
b53a0755
JY		 50#include "util/pmu-hybrid.h"
51#include "util/evlist-hybrid.h"
d8871ea7 52#include "asm/bug.h"
c1a604df 53#include "perf.h"
7c6a1c65 54
a43783ae 55#include <errno.h>
fd20e811 56#include <inttypes.h>
67230479 57#include <locale.h>
4208735d 58#include <poll.h>
d99c22ea 59#include <pthread.h>
97124d5e 60#include <unistd.h>
415ccb58
AB		 61#ifndef HAVE_GETTID
62#include <syscall.h>
63#endif
de9ac07b 64#include <sched.h>
9607ad3a 65#include <signal.h>
da231338
AM		 66#ifdef HAVE_EVENTFD_SUPPORT
67#include <sys/eventfd.h>
68#endif
a41794cd 69#include <sys/mman.h>
4208735d 70#include <sys/wait.h>
eeb399b5
AH		 71#include <sys/types.h>
72#include <sys/stat.h>
73#include <fcntl.h>
6ef81c55 74#include <linux/err.h>
8520a98d 75#include <linux/string.h>
0693e680 76#include <linux/time64.h>
d8f9da24 77#include <linux/zalloc.h>
8384a260 78#include <linux/bitmap.h>
d1e325cf 79#include <sys/time.h>
78da39fa 80
1b43b704 81struct switch_output {
dc0c6127 82 bool enabled;
1b43b704 83 bool signal;
dc0c6127 84 unsigned long size;
bfacbe3b 85 unsigned long time;
cb4e1ebb
JO		 86 const char *str;
87 bool set;
03724b2e
AK		 88 char **filenames;
89 int num_files;
90 int cur_file;
1b43b704
JO		 91};
92
7954f716
AB		 93struct thread_mask {
94 struct mmap_cpu_mask maps;
95 struct mmap_cpu_mask affinity;
96};
97
415ccb58
AB		 98struct record_thread {
99 pid_t tid;
100 struct thread_mask *mask;
101 struct {
102 int msg[2];
103 int ack[2];
104 } pipes;
105 struct fdarray pollfd;
106 int ctlfd_pos;
107 int nr_mmaps;
108 struct mmap **maps;
109 struct mmap **overwrite_maps;
110 struct record *rec;
396b626b
AB		 111 unsigned long long samples;
112 unsigned long waking;
ae9c7242 113 u64 bytes_written;
610fbc01
AB		 114 u64 bytes_transferred;
115 u64 bytes_compressed;
415ccb58
AB		 116};
117
396b626b
AB		 118static __thread struct record_thread *thread;
119
1e5de7d9
AB		 120enum thread_msg {
121 THREAD_MSG__UNDEFINED = 0,
122 THREAD_MSG__READY,
123 THREAD_MSG__MAX,
124};
125
126static const char *thread_msg_tags[THREAD_MSG__MAX] = {
127 "UNDEFINED", "READY"
128};
129
8c6f45a7 130struct record {
45694aa7 131 struct perf_tool tool;
b4006796 132 struct record_opts opts;
d20deb64 133 u64 bytes_written;
8ceb41d7 134 struct perf_data data;
ef149c25 135 struct auxtrace_record *itr;
63503dba 136 struct evlist *evlist;
d20deb64 137 struct perf_session *session;
bc477d79 138 struct evlist *sb_evlist;
899e5ffb 139 pthread_t thread_id;
d20deb64 140 int realtime_prio;
899e5ffb 141 bool switch_output_event_set;
d20deb64 142 bool no_buildid;
d2db9a98 143 bool no_buildid_set;
d20deb64 144 bool no_buildid_cache;
d2db9a98 145 bool no_buildid_cache_set;
6156681b 146 bool buildid_all;
e29386c8 147 bool buildid_mmap;
ecfd7a9c 148 bool timestamp_filename;
68588baf 149 bool timestamp_boundary;
1b43b704 150 struct switch_output switch_output;
9f065194 151 unsigned long long samples;
6d575816 152 unsigned long output_max_size; /* = 0: unlimited */
9bce13ea 153 struct perf_debuginfod debuginfod;
7954f716
AB		 154 int nr_threads;
155 struct thread_mask *thread_masks;
415ccb58 156 struct record_thread *thread_data;
0f82ebc4 157};
a21ca2ca 158
6d575816
JS		 159static volatile int done;
160
dc0c6127
JO		 161static volatile int auxtrace_record__snapshot_started;
162static DEFINE_TRIGGER(auxtrace_snapshot_trigger);
163static DEFINE_TRIGGER(switch_output_trigger);
164
9d2ed645
AB		 165static const char *affinity_tags[PERF_AFFINITY_MAX] = {
166 "SYS", "NODE", "CPU"
167};
168
415ccb58
AB		 169#ifndef HAVE_GETTID
170static inline pid_t gettid(void)
171{
172 return (pid_t)syscall(__NR_gettid);
173}
174#endif
175
3217e9fe
AB		 176static int record__threads_enabled(struct record *rec)
177{
178 return rec->opts.threads_spec;
179}
180
dc0c6127
JO		 181static bool switch_output_signal(struct record *rec)
182{
183 return rec->switch_output.signal &&
184 trigger_is_ready(&switch_output_trigger);
185}
186
187static bool switch_output_size(struct record *rec)
188{
189 return rec->switch_output.size &&
190 trigger_is_ready(&switch_output_trigger) &&
191 (rec->bytes_written >= rec->switch_output.size);
192}
193
bfacbe3b
JO		 194static bool switch_output_time(struct record *rec)
195{
196 return rec->switch_output.time &&
197 trigger_is_ready(&switch_output_trigger);
198}
199
ae9c7242
AB		 200static u64 record__bytes_written(struct record *rec)
201{
202 int t;
203 u64 bytes_written = rec->bytes_written;
204 struct record_thread *thread_data = rec->thread_data;
205
206 for (t = 0; t < rec->nr_threads; t++)
207 bytes_written += thread_data[t].bytes_written;
208
209 return bytes_written;
210}
211
6d575816
JS		 212static bool record__output_max_size_exceeded(struct record *rec)
213{
214 return rec->output_max_size &&
ae9c7242 215 (record__bytes_written(rec) >= rec->output_max_size);
6d575816
JS		 216}
217
a5830532 218static int record__write(struct record *rec, struct mmap *map __maybe_unused,
ded2b8fe 219 void *bf, size_t size)
f5970550 220{
ded2b8fe
JO		 221 struct perf_data_file *file = &rec->session->data->file;
222
56f735ff
AB		 223 if (map && map->file)
224 file = map->file;
225
ded2b8fe 226 if (perf_data_file__write(file, bf, size) < 0) {
50a9b868
JO		 227 pr_err("failed to write perf data, error: %m\n");
228 return -1;
f5970550 229 }
8d3eca20 230
ae9c7242
AB		 231 if (map && map->file)
232 thread->bytes_written += size;
233 else
56f735ff 234 rec->bytes_written += size;
dc0c6127 235
6d575816
JS		 236 if (record__output_max_size_exceeded(rec) && !done) {
237 fprintf(stderr, "[ perf record: perf size limit reached (%" PRIu64 " KB),"
238 " stopping session ]\n",
ae9c7242 239 record__bytes_written(rec) >> 10);
6d575816
JS		 240 done = 1;
241 }
242
dc0c6127
JO		 243 if (switch_output_size(rec))
244 trigger_hit(&switch_output_trigger);
245
8d3eca20 246 return 0;
f5970550
PZ		 247}
248
ef781128
AB		 249static int record__aio_enabled(struct record *rec);
250static int record__comp_enabled(struct record *rec);
75f5f1fc
AB		 251static size_t zstd_compress(struct perf_session *session, struct mmap *map,
252 void *dst, size_t dst_size, void *src, size_t src_size);
5d7f4116 253
d3d1af6f
AB		 254#ifdef HAVE_AIO_SUPPORT
255static int record__aio_write(struct aiocb *cblock, int trace_fd,
256 void *buf, size_t size, off_t off)
257{
258 int rc;
259
260 cblock->aio_fildes = trace_fd;
261 cblock->aio_buf = buf;
262 cblock->aio_nbytes = size;
263 cblock->aio_offset = off;
264 cblock->aio_sigevent.sigev_notify = SIGEV_NONE;
265
266 do {
267 rc = aio_write(cblock);
268 if (rc == 0) {
269 break;
270 } else if (errno != EAGAIN) {
271 cblock->aio_fildes = -1;
272 pr_err("failed to queue perf data, error: %m\n");
273 break;
274 }
275 } while (1);
276
277 return rc;
278}
279
a5830532 280static int record__aio_complete(struct mmap *md, struct aiocb *cblock)
d3d1af6f
AB		 281{
282 void *rem_buf;
283 off_t rem_off;
284 size_t rem_size;
285 int rc, aio_errno;
286 ssize_t aio_ret, written;
287
288 aio_errno = aio_error(cblock);
289 if (aio_errno == EINPROGRESS)
290 return 0;
291
292 written = aio_ret = aio_return(cblock);
293 if (aio_ret < 0) {
294 if (aio_errno != EINTR)
295 pr_err("failed to write perf data, error: %m\n");
296 written = 0;
297 }
298
299 rem_size = cblock->aio_nbytes - written;
300
301 if (rem_size == 0) {
302 cblock->aio_fildes = -1;
303 /*
ef781128
AB		 304 * md->refcount is incremented in record__aio_pushfn() for
305 * every aio write request started in record__aio_push() so
306 * decrement it because the request is now complete.
d3d1af6f 307 */
80e53d11 308 perf_mmap__put(&md->core);
d3d1af6f
AB		 309 rc = 1;
310 } else {
311 /*
312 * aio write request may require restart with the
313 * reminder if the kernel didn't write whole
314 * chunk at once.
315 */
316 rem_off = cblock->aio_offset + written;
317 rem_buf = (void *)(cblock->aio_buf + written);
318 record__aio_write(cblock, cblock->aio_fildes,
319 rem_buf, rem_size, rem_off);
320 rc = 0;
321 }
322
323 return rc;
324}
325
a5830532 326static int record__aio_sync(struct mmap *md, bool sync_all)
d3d1af6f 327{
93f20c0f
AB		 328 struct aiocb **aiocb = md->aio.aiocb;
329 struct aiocb *cblocks = md->aio.cblocks;
d3d1af6f 330 struct timespec timeout = { 0, 1000 * 1000 * 1 }; /* 1ms */
93f20c0f 331 int i, do_suspend;
d3d1af6f
AB		 332
333 do {
93f20c0f
AB		 334 do_suspend = 0;
335 for (i = 0; i < md->aio.nr_cblocks; ++i) {
336 if (cblocks[i].aio_fildes == -1 || record__aio_complete(md, &cblocks[i])) {
337 if (sync_all)
338 aiocb[i] = NULL;
339 else
340 return i;
341 } else {
342 /*
343 * Started aio write is not complete yet
344 * so it has to be waited before the
345 * next allocation.
346 */
347 aiocb[i] = &cblocks[i];
348 do_suspend = 1;
349 }
350 }
351 if (!do_suspend)
352 return -1;
d3d1af6f 353
93f20c0f 354 while (aio_suspend((const struct aiocb **)aiocb, md->aio.nr_cblocks, &timeout)) {
d3d1af6f
AB		 355 if (!(errno == EAGAIN || errno == EINTR))
356 pr_err("failed to sync perf data, error: %m\n");
357 }
358 } while (1);
359}
360
ef781128
AB		 361struct record_aio {
362 struct record *rec;
363 void *data;
364 size_t size;
365};
366
a5830532 367static int record__aio_pushfn(struct mmap *map, void *to, void *buf, size_t size)
d3d1af6f 368{
ef781128 369 struct record_aio *aio = to;
d3d1af6f 370
ef781128 371 /*
547740f7 372 * map->core.base data pointed by buf is copied into free map->aio.data[] buffer
ef781128
AB		 373 * to release space in the kernel buffer as fast as possible, calling
374 * perf_mmap__consume() from perf_mmap__push() function.
375 *
376 * That lets the kernel to proceed with storing more profiling data into
377 * the kernel buffer earlier than other per-cpu kernel buffers are handled.
378 *
379 * Coping can be done in two steps in case the chunk of profiling data
380 * crosses the upper bound of the kernel buffer. In this case we first move
381 * part of data from map->start till the upper bound and then the reminder
382 * from the beginning of the kernel buffer till the end of the data chunk.
383 */
384
385 if (record__comp_enabled(aio->rec)) {
75f5f1fc 386 size = zstd_compress(aio->rec->session, NULL, aio->data + aio->size,
bf59b305 387 mmap__mmap_len(map) - aio->size,
ef781128
AB		 388 buf, size);
389 } else {
390 memcpy(aio->data + aio->size, buf, size);
391 }
392
393 if (!aio->size) {
394 /*
395 * Increment map->refcount to guard map->aio.data[] buffer
396 * from premature deallocation because map object can be
397 * released earlier than aio write request started on
398 * map->aio.data[] buffer is complete.
399 *
400 * perf_mmap__put() is done at record__aio_complete()
401 * after started aio request completion or at record__aio_push()
402 * if the request failed to start.
403 */
e75710f0 404 perf_mmap__get(&map->core);
ef781128
AB		 405 }
406
407 aio->size += size;
408
409 return size;
410}
411
a5830532 412static int record__aio_push(struct record *rec, struct mmap *map, off_t *off)
ef781128
AB		 413{
414 int ret, idx;
415 int trace_fd = rec->session->data->file.fd;
416 struct record_aio aio = { .rec = rec, .size = 0 };
d3d1af6f 417
ef781128
AB		 418 /*
419 * Call record__aio_sync() to wait till map->aio.data[] buffer
420 * becomes available after previous aio write operation.
421 */
422
423 idx = record__aio_sync(map, false);
424 aio.data = map->aio.data[idx];
425 ret = perf_mmap__push(map, &aio, record__aio_pushfn);
426 if (ret != 0) /* ret > 0 - no data, ret < 0 - error */
427 return ret;
428
429 rec->samples++;
430 ret = record__aio_write(&(map->aio.cblocks[idx]), trace_fd, aio.data, aio.size, *off);
d3d1af6f 431 if (!ret) {
ef781128
AB		 432 *off += aio.size;
433 rec->bytes_written += aio.size;
d3d1af6f
AB		 434 if (switch_output_size(rec))
435 trigger_hit(&switch_output_trigger);
ef781128
AB		 436 } else {
437 /*
438 * Decrement map->refcount incremented in record__aio_pushfn()
439 * back if record__aio_write() operation failed to start, otherwise
440 * map->refcount is decremented in record__aio_complete() after
441 * aio write operation finishes successfully.
442 */
80e53d11 443 perf_mmap__put(&map->core);
d3d1af6f
AB		 444 }
445
446 return ret;
447}
448
449static off_t record__aio_get_pos(int trace_fd)
450{
451 return lseek(trace_fd, 0, SEEK_CUR);
452}
453
454static void record__aio_set_pos(int trace_fd, off_t pos)
455{
456 lseek(trace_fd, pos, SEEK_SET);
457}
458
459static void record__aio_mmap_read_sync(struct record *rec)
460{
461 int i;
63503dba 462 struct evlist *evlist = rec->evlist;
a5830532 463 struct mmap *maps = evlist->mmap;
d3d1af6f 464
ef781128 465 if (!record__aio_enabled(rec))
d3d1af6f
AB		 466 return;
467
c976ee11 468 for (i = 0; i < evlist->core.nr_mmaps; i++) {
a5830532 469 struct mmap *map = &maps[i];
d3d1af6f 470
547740f7 471 if (map->core.base)
93f20c0f 472 record__aio_sync(map, true);
d3d1af6f
AB		 473 }
474}
475
476static int nr_cblocks_default = 1;
93f20c0f 477static int nr_cblocks_max = 4;
d3d1af6f
AB		 478
479static int record__aio_parse(const struct option *opt,
93f20c0f 480 const char *str,
d3d1af6f
AB		 481 int unset)
482{
483 struct record_opts *opts = (struct record_opts *)opt->value;
484
93f20c0f 485 if (unset) {
d3d1af6f 486 opts->nr_cblocks = 0;
93f20c0f
AB		 487 } else {
488 if (str)
489 opts->nr_cblocks = strtol(str, NULL, 0);
490 if (!opts->nr_cblocks)
491 opts->nr_cblocks = nr_cblocks_default;
492 }
d3d1af6f
AB		 493
494 return 0;
495}
496#else /* HAVE_AIO_SUPPORT */
93f20c0f
AB		 497static int nr_cblocks_max = 0;
498
a5830532 499static int record__aio_push(struct record *rec __maybe_unused, struct mmap *map __maybe_unused,
ef781128 500 off_t *off __maybe_unused)
d3d1af6f
AB		 501{
502 return -1;
503}
504
505static off_t record__aio_get_pos(int trace_fd __maybe_unused)
506{
507 return -1;
508}
509
510static void record__aio_set_pos(int trace_fd __maybe_unused, off_t pos __maybe_unused)
511{
512}
513
514static void record__aio_mmap_read_sync(struct record *rec __maybe_unused)
515{
516}
517#endif
518
519static int record__aio_enabled(struct record *rec)
520{
521 return rec->opts.nr_cblocks > 0;
522}
523
470530bb
AB		 524#define MMAP_FLUSH_DEFAULT 1
525static int record__mmap_flush_parse(const struct option *opt,
526 const char *str,
527 int unset)
528{
529 int flush_max;
530 struct record_opts *opts = (struct record_opts *)opt->value;
531 static struct parse_tag tags[] = {
532 { .tag = 'B', .mult = 1 },
533 { .tag = 'K', .mult = 1 << 10 },
534 { .tag = 'M', .mult = 1 << 20 },
535 { .tag = 'G', .mult = 1 << 30 },
536 { .tag = 0 },
537 };
538
539 if (unset)
540 return 0;
541
542 if (str) {
543 opts->mmap_flush = parse_tag_value(str, tags);
544 if (opts->mmap_flush == (int)-1)
545 opts->mmap_flush = strtol(str, NULL, 0);
546 }
547
548 if (!opts->mmap_flush)
549 opts->mmap_flush = MMAP_FLUSH_DEFAULT;
550
9521b5f2 551 flush_max = evlist__mmap_size(opts->mmap_pages);
470530bb
AB		 552 flush_max /= 4;
553 if (opts->mmap_flush > flush_max)
554 opts->mmap_flush = flush_max;
555
556 return 0;
557}
558
504c1ad1
AB		 559#ifdef HAVE_ZSTD_SUPPORT
560static unsigned int comp_level_default = 1;
561
562static int record__parse_comp_level(const struct option *opt, const char *str, int unset)
563{
564 struct record_opts *opts = opt->value;
565
566 if (unset) {
567 opts->comp_level = 0;
568 } else {
569 if (str)
570 opts->comp_level = strtol(str, NULL, 0);
571 if (!opts->comp_level)
572 opts->comp_level = comp_level_default;
573 }
574
575 return 0;
576}
577#endif
51255a8a
AB		 578static unsigned int comp_level_max = 22;
579
42e1fd80
AB		 580static int record__comp_enabled(struct record *rec)
581{
582 return rec->opts.comp_level > 0;
583}
584
45694aa7 585static int process_synthesized_event(struct perf_tool *tool,
d20deb64 586 union perf_event *event,
1d037ca1
IT		 587 struct perf_sample *sample __maybe_unused,
588 struct machine *machine __maybe_unused)
234fbbf5 589{
8c6f45a7 590 struct record *rec = container_of(tool, struct record, tool);
ded2b8fe 591 return record__write(rec, NULL, event, event->header.size);
234fbbf5
ACM		 592}
593
d99c22ea
SE		 594static int process_locked_synthesized_event(struct perf_tool *tool,
595 union perf_event *event,
596 struct perf_sample *sample __maybe_unused,
597 struct machine *machine __maybe_unused)
598{
599 static pthread_mutex_t synth_lock = PTHREAD_MUTEX_INITIALIZER;
600 int ret;
601
602 pthread_mutex_lock(&synth_lock);
603 ret = process_synthesized_event(tool, event, sample, machine);
604 pthread_mutex_unlock(&synth_lock);
605 return ret;
606}
607
a5830532 608static int record__pushfn(struct mmap *map, void *to, void *bf, size_t size)
d37f1586
ACM		 609{
610 struct record *rec = to;
611
5d7f4116 612 if (record__comp_enabled(rec)) {
75f5f1fc 613 size = zstd_compress(rec->session, map, map->data, mmap__mmap_len(map), bf, size);
5d7f4116
AB		 614 bf = map->data;
615 }
616
396b626b 617 thread->samples++;
ded2b8fe 618 return record__write(rec, map, bf, size);
d37f1586
ACM		 619}
620
2dd6d8a1
AH		 621static volatile int signr = -1;
622static volatile int child_finished;
da231338
AM		 623#ifdef HAVE_EVENTFD_SUPPORT
624static int done_fd = -1;
625#endif
c0bdc1c4 626
2dd6d8a1
AH		 627static void sig_handler(int sig)
628{
629 if (sig == SIGCHLD)
630 child_finished = 1;
631 else
632 signr = sig;
633
634 done = 1;
da231338
AM		 635#ifdef HAVE_EVENTFD_SUPPORT
636{
637 u64 tmp = 1;
638 /*
639 * It is possible for this signal handler to run after done is checked
640 * in the main loop, but before the perf counter fds are polled. If this
641 * happens, the poll() will continue to wait even though done is set,
642 * and will only break out if either another signal is received, or the
643 * counters are ready for read. To ensure the poll() doesn't sleep when
644 * done is set, use an eventfd (done_fd) to wake up the poll().
645 */
646 if (write(done_fd, &tmp, sizeof(tmp)) < 0)
647 pr_err("failed to signal wakeup fd, error: %m\n");
648}
649#endif // HAVE_EVENTFD_SUPPORT
2dd6d8a1
AH		 650}
651
a074865e
WN		 652static void sigsegv_handler(int sig)
653{
654 perf_hooks__recover();
655 sighandler_dump_stack(sig);
656}
657
2dd6d8a1
AH		 658static void record__sig_exit(void)
659{
660 if (signr == -1)
661 return;
662
663 signal(signr, SIG_DFL);
664 raise(signr);
665}
666
e31f0d01
AH		 667#ifdef HAVE_AUXTRACE_SUPPORT
668
ef149c25 669static int record__process_auxtrace(struct perf_tool *tool,
a5830532 670 struct mmap *map,
ef149c25
AH		 671 union perf_event *event, void *data1,
672 size_t len1, void *data2, size_t len2)
673{
674 struct record *rec = container_of(tool, struct record, tool);
8ceb41d7 675 struct perf_data *data = &rec->data;
ef149c25
AH		 676 size_t padding;
677 u8 pad[8] = {0};
678
46e201ef 679 if (!perf_data__is_pipe(data) && perf_data__is_single_file(data)) {
99fa2984 680 off_t file_offset;
8ceb41d7 681 int fd = perf_data__fd(data);
99fa2984
AH		 682 int err;
683
684 file_offset = lseek(fd, 0, SEEK_CUR);
685 if (file_offset == -1)
686 return -1;
687 err = auxtrace_index__auxtrace_event(&rec->session->auxtrace_index,
688 event, file_offset);
689 if (err)
690 return err;
691 }
692
ef149c25
AH		 693 /* event.auxtrace.size includes padding, see __auxtrace_mmap__read() */
694 padding = (len1 + len2) & 7;
695 if (padding)
696 padding = 8 - padding;
697
ded2b8fe
JO		 698 record__write(rec, map, event, event->header.size);
699 record__write(rec, map, data1, len1);
ef149c25 700 if (len2)
ded2b8fe
JO		 701 record__write(rec, map, data2, len2);
702 record__write(rec, map, &pad, padding);
ef149c25
AH		 703
704 return 0;
705}
706
707static int record__auxtrace_mmap_read(struct record *rec,
a5830532 708 struct mmap *map)
ef149c25
AH		 709{
710 int ret;
711
e035f4ca 712 ret = auxtrace_mmap__read(map, rec->itr, &rec->tool,
ef149c25
AH		 713 record__process_auxtrace);
714 if (ret < 0)
715 return ret;
716
717 if (ret)
718 rec->samples++;
719
720 return 0;
721}
722
2dd6d8a1 723static int record__auxtrace_mmap_read_snapshot(struct record *rec,
a5830532 724 struct mmap *map)
2dd6d8a1
AH		 725{
726 int ret;
727
e035f4ca 728 ret = auxtrace_mmap__read_snapshot(map, rec->itr, &rec->tool,
2dd6d8a1
AH		 729 record__process_auxtrace,
730 rec->opts.auxtrace_snapshot_size);
731 if (ret < 0)
732 return ret;
733
734 if (ret)
735 rec->samples++;
736
737 return 0;
738}
739
740static int record__auxtrace_read_snapshot_all(struct record *rec)
741{
742 int i;
743 int rc = 0;
744
c976ee11 745 for (i = 0; i < rec->evlist->core.nr_mmaps; i++) {
a5830532 746 struct mmap *map = &rec->evlist->mmap[i];
2dd6d8a1 747
e035f4ca 748 if (!map->auxtrace_mmap.base)
2dd6d8a1
AH		 749 continue;
750
e035f4ca 751 if (record__auxtrace_mmap_read_snapshot(rec, map) != 0) {
2dd6d8a1
AH		 752 rc = -1;
753 goto out;
754 }
755 }
756out:
757 return rc;
758}
759
ce7b0e42 760static void record__read_auxtrace_snapshot(struct record *rec, bool on_exit)
2dd6d8a1
AH		 761{
762 pr_debug("Recording AUX area tracing snapshot\n");
763 if (record__auxtrace_read_snapshot_all(rec) < 0) {
5f9cf599 764 trigger_error(&auxtrace_snapshot_trigger);
2dd6d8a1 765 } else {
ce7b0e42 766 if (auxtrace_record__snapshot_finish(rec->itr, on_exit))
5f9cf599
WN		 767 trigger_error(&auxtrace_snapshot_trigger);
768 else
769 trigger_ready(&auxtrace_snapshot_trigger);
2dd6d8a1
AH		 770 }
771}
772
ce7b0e42
AS		 773static int record__auxtrace_snapshot_exit(struct record *rec)
774{
775 if (trigger_is_error(&auxtrace_snapshot_trigger))
776 return 0;
777
778 if (!auxtrace_record__snapshot_started &&
779 auxtrace_record__snapshot_start(rec->itr))
780 return -1;
781
782 record__read_auxtrace_snapshot(rec, true);
783 if (trigger_is_error(&auxtrace_snapshot_trigger))
784 return -1;
785
786 return 0;
787}
788
4b5ea3bd
AH		 789static int record__auxtrace_init(struct record *rec)
790{
791 int err;
792
793 if (!rec->itr) {
794 rec->itr = auxtrace_record__init(rec->evlist, &err);
795 if (err)
796 return err;
797 }
798
799 err = auxtrace_parse_snapshot_options(rec->itr, &rec->opts,
800 rec->opts.auxtrace_snapshot_opts);
801 if (err)
802 return err;
803
c0a6de06
AH		 804 err = auxtrace_parse_sample_options(rec->itr, rec->evlist, &rec->opts,
805 rec->opts.auxtrace_sample_opts);
806 if (err)
807 return err;
808
d58b3f7e
AH		 809 auxtrace_regroup_aux_output(rec->evlist);
810
4b5ea3bd
AH		 811 return auxtrace_parse_filters(rec->evlist);
812}
813
e31f0d01
AH		 814#else
815
816static inline
817int record__auxtrace_mmap_read(struct record *rec __maybe_unused,
a5830532 818 struct mmap *map __maybe_unused)
e31f0d01
AH		 819{
820 return 0;
821}
822
2dd6d8a1 823static inline
ce7b0e42
AS		 824void record__read_auxtrace_snapshot(struct record *rec __maybe_unused,
825 bool on_exit __maybe_unused)
de9ac07b 826{
f7b7c26e
PZ		 827}
828
2dd6d8a1
AH		 829static inline
830int auxtrace_record__snapshot_start(struct auxtrace_record *itr __maybe_unused)
f7b7c26e 831{
2dd6d8a1 832 return 0;
de9ac07b
PZ		 833}
834
ce7b0e42
AS		 835static inline
836int record__auxtrace_snapshot_exit(struct record *rec __maybe_unused)
837{
838 return 0;
839}
840
4b5ea3bd
AH		 841static int record__auxtrace_init(struct record *rec __maybe_unused)
842{
843 return 0;
844}
845
2dd6d8a1
AH		 846#endif
847
246eba8e
AH		 848static int record__config_text_poke(struct evlist *evlist)
849{
850 struct evsel *evsel;
851 int err;
852
853 /* Nothing to do if text poke is already configured */
854 evlist__for_each_entry(evlist, evsel) {
855 if (evsel->core.attr.text_poke)
856 return 0;
857 }
858
859 err = parse_events(evlist, "dummy:u", NULL);
860 if (err)
861 return err;
862
863 evsel = evlist__last(evlist);
864
865 evsel->core.attr.freq = 0;
866 evsel->core.attr.sample_period = 1;
867 evsel->core.attr.text_poke = 1;
868 evsel->core.attr.ksymbol = 1;
869
870 evsel->core.system_wide = true;
871 evsel->no_aux_samples = true;
872 evsel->immediate = true;
873
874 /* Text poke must be collected on all CPUs */
875 perf_cpu_map__put(evsel->core.own_cpus);
876 evsel->core.own_cpus = perf_cpu_map__new(NULL);
877 perf_cpu_map__put(evsel->core.cpus);
878 evsel->core.cpus = perf_cpu_map__get(evsel->core.own_cpus);
879
880 evsel__set_sample_bit(evsel, TIME);
881
882 return 0;
883}
884
eeb399b5
AH		 885static bool record__kcore_readable(struct machine *machine)
886{
887 char kcore[PATH_MAX];
888 int fd;
889
890 scnprintf(kcore, sizeof(kcore), "%s/proc/kcore", machine->root_dir);
891
892 fd = open(kcore, O_RDONLY);
893 if (fd < 0)
894 return false;
895
896 close(fd);
897
898 return true;
899}
900
901static int record__kcore_copy(struct machine *machine, struct perf_data *data)
902{
903 char from_dir[PATH_MAX];
904 char kcore_dir[PATH_MAX];
905 int ret;
906
907 snprintf(from_dir, sizeof(from_dir), "%s/proc", machine->root_dir);
908
909 ret = perf_data__make_kcore_dir(data, kcore_dir, sizeof(kcore_dir));
910 if (ret)
911 return ret;
912
913 return kcore_copy(from_dir, kcore_dir);
914}
915
415ccb58
AB		 916static void record__thread_data_init_pipes(struct record_thread *thread_data)
917{
918 thread_data->pipes.msg[0] = -1;
919 thread_data->pipes.msg[1] = -1;
920 thread_data->pipes.ack[0] = -1;
921 thread_data->pipes.ack[1] = -1;
922}
923
924static int record__thread_data_open_pipes(struct record_thread *thread_data)
925{
926 if (pipe(thread_data->pipes.msg))
927 return -EINVAL;
928
929 if (pipe(thread_data->pipes.ack)) {
930 close(thread_data->pipes.msg[0]);
931 thread_data->pipes.msg[0] = -1;
932 close(thread_data->pipes.msg[1]);
933 thread_data->pipes.msg[1] = -1;
934 return -EINVAL;
935 }
936
937 pr_debug2("thread_data[%p]: msg=[%d,%d], ack=[%d,%d]\n", thread_data,
938 thread_data->pipes.msg[0], thread_data->pipes.msg[1],
939 thread_data->pipes.ack[0], thread_data->pipes.ack[1]);
940
941 return 0;
942}
943
944static void record__thread_data_close_pipes(struct record_thread *thread_data)
945{
946 if (thread_data->pipes.msg[0] != -1) {
947 close(thread_data->pipes.msg[0]);
948 thread_data->pipes.msg[0] = -1;
949 }
950 if (thread_data->pipes.msg[1] != -1) {
951 close(thread_data->pipes.msg[1]);
952 thread_data->pipes.msg[1] = -1;
953 }
954 if (thread_data->pipes.ack[0] != -1) {
955 close(thread_data->pipes.ack[0]);
956 thread_data->pipes.ack[0] = -1;
957 }
958 if (thread_data->pipes.ack[1] != -1) {
959 close(thread_data->pipes.ack[1]);
960 thread_data->pipes.ack[1] = -1;
961 }
962}
963
964static int record__thread_data_init_maps(struct record_thread *thread_data, struct evlist *evlist)
965{
966 int m, tm, nr_mmaps = evlist->core.nr_mmaps;
967 struct mmap *mmap = evlist->mmap;
968 struct mmap *overwrite_mmap = evlist->overwrite_mmap;
969 struct perf_cpu_map *cpus = evlist->core.cpus;
970
971 thread_data->nr_mmaps = bitmap_weight(thread_data->mask->maps.bits,
972 thread_data->mask->maps.nbits);
973 if (mmap) {
974 thread_data->maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
975 if (!thread_data->maps)
976 return -ENOMEM;
977 }
978 if (overwrite_mmap) {
979 thread_data->overwrite_maps = zalloc(thread_data->nr_mmaps * sizeof(struct mmap *));
980 if (!thread_data->overwrite_maps) {
981 zfree(&thread_data->maps);
982 return -ENOMEM;
983 }
984 }
985 pr_debug2("thread_data[%p]: nr_mmaps=%d, maps=%p, ow_maps=%p\n", thread_data,
986 thread_data->nr_mmaps, thread_data->maps, thread_data->overwrite_maps);
987
988 for (m = 0, tm = 0; m < nr_mmaps && tm < thread_data->nr_mmaps; m++) {
989 if (test_bit(cpus->map[m].cpu, thread_data->mask->maps.bits)) {
990 if (thread_data->maps) {
991 thread_data->maps[tm] = &mmap[m];
992 pr_debug2("thread_data[%p]: cpu%d: maps[%d] -> mmap[%d]\n",
993 thread_data, cpus->map[m].cpu, tm, m);
994 }
995 if (thread_data->overwrite_maps) {
996 thread_data->overwrite_maps[tm] = &overwrite_mmap[m];
997 pr_debug2("thread_data[%p]: cpu%d: ow_maps[%d] -> ow_mmap[%d]\n",
998 thread_data, cpus->map[m].cpu, tm, m);
999 }
1000 tm++;
1001 }
1002 }
1003
1004 return 0;
1005}
1006
1007static int record__thread_data_init_pollfd(struct record_thread *thread_data, struct evlist *evlist)
1008{
1009 int f, tm, pos;
1010 struct mmap *map, *overwrite_map;
1011
1012 fdarray__init(&thread_data->pollfd, 64);
1013
1014 for (tm = 0; tm < thread_data->nr_mmaps; tm++) {
1015 map = thread_data->maps ? thread_data->maps[tm] : NULL;
1016 overwrite_map = thread_data->overwrite_maps ?
1017 thread_data->overwrite_maps[tm] : NULL;
1018
1019 for (f = 0; f < evlist->core.pollfd.nr; f++) {
1020 void *ptr = evlist->core.pollfd.priv[f].ptr;
1021
1022 if ((map && ptr == map) || (overwrite_map && ptr == overwrite_map)) {
1023 pos = fdarray__dup_entry_from(&thread_data->pollfd, f,
1024 &evlist->core.pollfd);
1025 if (pos < 0)
1026 return pos;
1027 pr_debug2("thread_data[%p]: pollfd[%d] <- event_fd=%d\n",
1028 thread_data, pos, evlist->core.pollfd.entries[f].fd);
1029 }
1030 }
1031 }
1032
1033 return 0;
1034}
1035
1036static void record__free_thread_data(struct record *rec)
1037{
1038 int t;
1039 struct record_thread *thread_data = rec->thread_data;
1040
1041 if (thread_data == NULL)
1042 return;
1043
1044 for (t = 0; t < rec->nr_threads; t++) {
1045 record__thread_data_close_pipes(&thread_data[t]);
1046 zfree(&thread_data[t].maps);
1047 zfree(&thread_data[t].overwrite_maps);
1048 fdarray__exit(&thread_data[t].pollfd);
1049 }
1050
1051 zfree(&rec->thread_data);
1052}
1053
1054static int record__alloc_thread_data(struct record *rec, struct evlist *evlist)
1055{
1056 int t, ret;
1057 struct record_thread *thread_data;
1058
1059 rec->thread_data = zalloc(rec->nr_threads * sizeof(*(rec->thread_data)));
1060 if (!rec->thread_data) {
1061 pr_err("Failed to allocate thread data\n");
1062 return -ENOMEM;
1063 }
1064 thread_data = rec->thread_data;
1065
1066 for (t = 0; t < rec->nr_threads; t++)
1067 record__thread_data_init_pipes(&thread_data[t]);
1068
1069 for (t = 0; t < rec->nr_threads; t++) {
1070 thread_data[t].rec = rec;
1071 thread_data[t].mask = &rec->thread_masks[t];
1072 ret = record__thread_data_init_maps(&thread_data[t], evlist);
1073 if (ret) {
1074 pr_err("Failed to initialize thread[%d] maps\n", t);
1075 goto out_free;
1076 }
1077 ret = record__thread_data_init_pollfd(&thread_data[t], evlist);
1078 if (ret) {
1079 pr_err("Failed to initialize thread[%d] pollfd\n", t);
1080 goto out_free;
1081 }
1082 if (t) {
1083 thread_data[t].tid = -1;
1084 ret = record__thread_data_open_pipes(&thread_data[t]);
1085 if (ret) {
1086 pr_err("Failed to open thread[%d] communication pipes\n", t);
1087 goto out_free;
1088 }
1089 ret = fdarray__add(&thread_data[t].pollfd, thread_data[t].pipes.msg[0],
1090 POLLIN | POLLERR | POLLHUP, fdarray_flag__nonfilterable);
1091 if (ret < 0) {
1092 pr_err("Failed to add descriptor to thread[%d] pollfd\n", t);
1093 goto out_free;
1094 }
1095 thread_data[t].ctlfd_pos = ret;
1096 pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
1097 thread_data, thread_data[t].ctlfd_pos,
1098 thread_data[t].pipes.msg[0]);
1099 } else {
1100 thread_data[t].tid = gettid();
1101 if (evlist->ctl_fd.pos == -1)
1102 continue;
1103 ret = fdarray__dup_entry_from(&thread_data[t].pollfd, evlist->ctl_fd.pos,
1104 &evlist->core.pollfd);
1105 if (ret < 0) {
1106 pr_err("Failed to duplicate descriptor in main thread pollfd\n");
1107 goto out_free;
1108 }
1109 thread_data[t].ctlfd_pos = ret;
1110 pr_debug2("thread_data[%p]: pollfd[%d] <- ctl_fd=%d\n",
1111 thread_data, thread_data[t].ctlfd_pos,
1112 evlist->core.pollfd.entries[evlist->ctl_fd.pos].fd);
1113 }
1114 }
1115
1116 return 0;
1117
1118out_free:
1119 record__free_thread_data(rec);
1120
1121 return ret;
1122}
1123
cda57a8c 1124static int record__mmap_evlist(struct record *rec,
63503dba 1125 struct evlist *evlist)
cda57a8c 1126{
56f735ff 1127 int i, ret;
cda57a8c 1128 struct record_opts *opts = &rec->opts;
c0a6de06
AH		 1129 bool auxtrace_overwrite = opts->auxtrace_snapshot_mode ||
1130 opts->auxtrace_sample_mode;
cda57a8c
WN		 1131 char msg[512];
1132
f13de660
AB		 1133 if (opts->affinity != PERF_AFFINITY_SYS)
1134 cpu__setup_cpunode_map();
1135
9521b5f2 1136 if (evlist__mmap_ex(evlist, opts->mmap_pages,
cda57a8c 1137 opts->auxtrace_mmap_pages,
c0a6de06 1138 auxtrace_overwrite,
470530bb 1139 opts->nr_cblocks, opts->affinity,
51255a8a 1140 opts->mmap_flush, opts->comp_level) < 0) {
cda57a8c
WN		 1141 if (errno == EPERM) {
1142 pr_err("Permission error mapping pages.\n"
1143 "Consider increasing "
1144 "/proc/sys/kernel/perf_event_mlock_kb,\n"
1145 "or try again with a smaller value of -m/--mmap_pages.\n"
1146 "(current value: %u,%u)\n",
1147 opts->mmap_pages, opts->auxtrace_mmap_pages);
1148 return -errno;
1149 } else {
1150 pr_err("failed to mmap with %d (%s)\n", errno,
c8b5f2c9 1151 str_error_r(errno, msg, sizeof(msg)));
cda57a8c
WN		 1152 if (errno)
1153 return -errno;
1154 else
1155 return -EINVAL;
1156 }
1157 }
415ccb58
AB		 1158
1159 if (evlist__initialize_ctlfd(evlist, opts->ctl_fd, opts->ctl_fd_ack))
1160 return -1;
1161
1162 ret = record__alloc_thread_data(rec, evlist);
1163 if (ret)
1164 return ret;
1165
56f735ff
AB		 1166 if (record__threads_enabled(rec)) {
1167 ret = perf_data__create_dir(&rec->data, evlist->core.nr_mmaps);
1168 if (ret)
1169 return ret;
1170 for (i = 0; i < evlist->core.nr_mmaps; i++) {
1171 if (evlist->mmap)
1172 evlist->mmap[i].file = &rec->data.dir.files[i];
1173 if (evlist->overwrite_mmap)
1174 evlist->overwrite_mmap[i].file = &rec->data.dir.files[i];
1175 }
1176 }
1177
cda57a8c
WN		 1178 return 0;
1179}
1180
1181static int record__mmap(struct record *rec)
1182{
1183 return record__mmap_evlist(rec, rec->evlist);
1184}
1185
8c6f45a7 1186static int record__open(struct record *rec)
dd7927f4 1187{
d6195a6a 1188 char msg[BUFSIZ];
32dcd021 1189 struct evsel *pos;
63503dba 1190 struct evlist *evlist = rec->evlist;
d20deb64 1191 struct perf_session *session = rec->session;
b4006796 1192 struct record_opts *opts = &rec->opts;
8d3eca20 1193 int rc = 0;
dd7927f4 1194
d3dbf43c 1195 /*
b91e5492
KL		 1196 * For initial_delay, system wide or a hybrid system, we need to add a
1197 * dummy event so that we can track PERF_RECORD_MMAP to cover the delay
1198 * of waiting or event synthesis.
d3dbf43c 1199 */
b91e5492
KL		 1200 if (opts->initial_delay || target__has_cpu(&opts->target) ||
1201 perf_pmu__has_hybrid()) {
e80db255 1202 pos = evlist__get_tracking_event(evlist);
442ad225
AH		 1203 if (!evsel__is_dummy_event(pos)) {
1204 /* Set up dummy event. */
facbf0b9 1205 if (evlist__add_dummy(evlist))
442ad225
AH		 1206 return -ENOMEM;
1207 pos = evlist__last(evlist);
e80db255 1208 evlist__set_tracking_event(evlist, pos);
442ad225
AH		 1209 }
1210
0a892c1c
IR		 1211 /*
1212 * Enable the dummy event when the process is forked for
1213 * initial_delay, immediately for system wide.
1214 */
bb07d62e
NK		 1215 if (opts->initial_delay && !pos->immediate &&
1216 !target__has_cpu(&opts->target))
0a892c1c
IR		 1217 pos->core.attr.enable_on_exec = 1;
1218 else
1219 pos->immediate = 1;
d3dbf43c
ACM		 1220 }
1221
78e1bc25 1222 evlist__config(evlist, opts, &callchain_param);
cac21425 1223
e5cadb93 1224 evlist__for_each_entry(evlist, pos) {
dd7927f4 1225try_again:
af663bd0 1226 if (evsel__open(pos, pos->core.cpus, pos->core.threads) < 0) {
ae430892 1227 if (evsel__fallback(pos, errno, msg, sizeof(msg))) {
bb963e16 1228 if (verbose > 0)
c0a54341 1229 ui__warning("%s\n", msg);
d6d901c2
ZY		 1230 goto try_again;
1231 }
cf99ad14 1232 if ((errno == EINVAL || errno == EBADF) &&
fba7c866 1233 pos->core.leader != &pos->core &&
cf99ad14 1234 pos->weak_group) {
64b4778b 1235 pos = evlist__reset_weak_group(evlist, pos, true);
cf99ad14
AK		 1236 goto try_again;
1237 }
56e52e85 1238 rc = -errno;
2bb72dbb 1239 evsel__open_strerror(pos, &opts->target, errno, msg, sizeof(msg));
56e52e85 1240 ui__error("%s\n", msg);
8d3eca20 1241 goto out;
c171b552 1242 }
bfd8f72c
AK		 1243
1244 pos->supported = true;
c171b552 1245 }
a43d3f08 1246
78e1bc25 1247 if (symbol_conf.kptr_restrict && !evlist__exclude_kernel(evlist)) {
c8b567c8
ACM		 1248 pr_warning(
1249"WARNING: Kernel address maps (/proc/{kallsyms,modules}) are restricted,\n"
1250"check /proc/sys/kernel/kptr_restrict and /proc/sys/kernel/perf_event_paranoid.\n\n"
1251"Samples in kernel functions may not be resolved if a suitable vmlinux\n"
1252"file is not found in the buildid cache or in the vmlinux path.\n\n"
1253"Samples in kernel modules won't be resolved at all.\n\n"
1254"If some relocation was applied (e.g. kexec) symbols may be misresolved\n"
1255"even with a suitable vmlinux or kallsyms file.\n\n");
1256 }
1257
24bf91a7 1258 if (evlist__apply_filters(evlist, &pos)) {
62d94b00 1259 pr_err("failed to set filter \"%s\" on event %s with %d (%s)\n",
8ab2e96d 1260 pos->filter, evsel__name(pos), errno,
c8b5f2c9 1261 str_error_r(errno, msg, sizeof(msg)));
8d3eca20 1262 rc = -1;
5d8bb1ec
MP		 1263 goto out;
1264 }
1265
cda57a8c
WN		 1266 rc = record__mmap(rec);
1267 if (rc)
8d3eca20 1268 goto out;
0a27d7f9 1269
563aecb2 1270 session->evlist = evlist;
7b56cce2 1271 perf_session__set_id_hdr_size(session);
8d3eca20
DA		 1272out:
1273 return rc;
16c8a109
PZ		 1274}
1275
66286ed3
AH		 1276static void set_timestamp_boundary(struct record *rec, u64 sample_time)
1277{
1278 if (rec->evlist->first_sample_time == 0)
1279 rec->evlist->first_sample_time = sample_time;
1280
1281 if (sample_time)
1282 rec->evlist->last_sample_time = sample_time;
1283}
1284
e3d59112
NK		 1285static int process_sample_event(struct perf_tool *tool,
1286 union perf_event *event,
1287 struct perf_sample *sample,
32dcd021 1288 struct evsel *evsel,
e3d59112
NK		 1289 struct machine *machine)
1290{
1291 struct record *rec = container_of(tool, struct record, tool);
1292
66286ed3 1293 set_timestamp_boundary(rec, sample->time);
e3d59112 1294
68588baf
JY		 1295 if (rec->buildid_all)
1296 return 0;
1297
1298 rec->samples++;
e3d59112
NK		 1299 return build_id__mark_dso_hit(tool, event, sample, evsel, machine);
1300}
1301
8c6f45a7 1302static int process_buildids(struct record *rec)
6122e4e4 1303{
f5fc1412 1304 struct perf_session *session = rec->session;
6122e4e4 1305
45112e89 1306 if (perf_data__size(&rec->data) == 0)
9f591fd7
ACM		 1307 return 0;
1308
00dc8657
NK		 1309 /*
1310 * During this process, it'll load kernel map and replace the
1311 * dso->long_name to a real pathname it found. In this case
1312 * we prefer the vmlinux path like
1313 * /lib/modules/3.16.4/build/vmlinux
1314 *
1315 * rather than build-id path (in debug directory).
1316 * $HOME/.debug/.build-id/f0/6e17aa50adf4d00b88925e03775de107611551
1317 */
1318 symbol_conf.ignore_vmlinux_buildid = true;
1319
6156681b
NK		 1320 /*
1321 * If --buildid-all is given, it marks all DSO regardless of hits,
68588baf
JY		 1322 * so no need to process samples. But if timestamp_boundary is enabled,
1323 * it still needs to walk on all samples to get the timestamps of
1324 * first/last samples.
6156681b 1325 */
68588baf 1326 if (rec->buildid_all && !rec->timestamp_boundary)
6156681b
NK		 1327 rec->tool.sample = NULL;
1328
b7b61cbe 1329 return perf_session__process_events(session);
6122e4e4
ACM		 1330}
1331
8115d60c 1332static void perf_event__synthesize_guest_os(struct machine *machine, void *data)
a1645ce1
ZY		 1333{
1334 int err;
45694aa7 1335 struct perf_tool *tool = data;
a1645ce1
ZY		 1336 /*
1337 *As for guest kernel when processing subcommand record&report,
1338 *we arrange module mmap prior to guest kernel mmap and trigger
1339 *a preload dso because default guest module symbols are loaded
1340 *from guest kallsyms instead of /lib/modules/XXX/XXX. This
1341 *method is used to avoid symbol missing when the first addr is
1342 *in module instead of in guest kernel.
1343 */
45694aa7 1344 err = perf_event__synthesize_modules(tool, process_synthesized_event,
743eb868 1345 machine);
a1645ce1
ZY		 1346 if (err < 0)
1347 pr_err("Couldn't record guest kernel [%d]'s reference"
23346f21 1348 " relocation symbol.\n", machine->pid);
a1645ce1 1349
a1645ce1
ZY		 1350 /*
1351 * We use _stext for guest kernel because guest kernel's /proc/kallsyms
1352 * have no _text sometimes.
1353 */
45694aa7 1354 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
0ae617be 1355 machine);
a1645ce1
ZY		 1356 if (err < 0)
1357 pr_err("Couldn't record guest kernel [%d]'s reference"
23346f21 1358 " relocation symbol.\n", machine->pid);
a1645ce1
ZY		 1359}
1360
98402807
FW		 1361static struct perf_event_header finished_round_event = {
1362 .size = sizeof(struct perf_event_header),
1363 .type = PERF_RECORD_FINISHED_ROUND,
1364};
1365
a5830532 1366static void record__adjust_affinity(struct record *rec, struct mmap *map)
f13de660
AB		 1367{
1368 if (rec->opts.affinity != PERF_AFFINITY_SYS &&
396b626b
AB		 1369 !bitmap_equal(thread->mask->affinity.bits, map->affinity_mask.bits,
1370 thread->mask->affinity.nbits)) {
1371 bitmap_zero(thread->mask->affinity.bits, thread->mask->affinity.nbits);
1372 bitmap_or(thread->mask->affinity.bits, thread->mask->affinity.bits,
1373 map->affinity_mask.bits, thread->mask->affinity.nbits);
1374 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
1375 (cpu_set_t *)thread->mask->affinity.bits);
1376 if (verbose == 2) {
1377 pr_debug("threads[%d]: running on cpu%d: ", thread->tid, sched_getcpu());
1378 mmap_cpu_mask__scnprintf(&thread->mask->affinity, "affinity");
1379 }
f13de660
AB		 1380 }
1381}
1382
5d7f4116
AB		 1383static size_t process_comp_header(void *record, size_t increment)
1384{
72932371 1385 struct perf_record_compressed *event = record;
5d7f4116
AB		 1386 size_t size = sizeof(*event);
1387
1388 if (increment) {
1389 event->header.size += increment;
1390 return increment;
1391 }
1392
1393 event->header.type = PERF_RECORD_COMPRESSED;
1394 event->header.size = size;
1395
1396 return size;
1397}
1398
75f5f1fc
AB		 1399static size_t zstd_compress(struct perf_session *session, struct mmap *map,
1400 void *dst, size_t dst_size, void *src, size_t src_size)
5d7f4116
AB		 1401{
1402 size_t compressed;
72932371 1403 size_t max_record_size = PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_record_compressed) - 1;
75f5f1fc 1404 struct zstd_data *zstd_data = &session->zstd_data;
5d7f4116 1405
75f5f1fc
AB		 1406 if (map && map->file)
1407 zstd_data = &map->zstd_data;
1408
1409 compressed = zstd_compress_stream_to_records(zstd_data, dst, dst_size, src, src_size,
5d7f4116
AB		 1410 max_record_size, process_comp_header);
1411
610fbc01
AB		 1412 if (map && map->file) {
1413 thread->bytes_transferred += src_size;
1414 thread->bytes_compressed += compressed;
1415 } else {
1416 session->bytes_transferred += src_size;
1417 session->bytes_compressed += compressed;
1418 }
5d7f4116
AB		 1419
1420 return compressed;
1421}
1422
63503dba 1423static int record__mmap_read_evlist(struct record *rec, struct evlist *evlist,
470530bb 1424 bool overwrite, bool synch)
98402807 1425{
dcabb507 1426 u64 bytes_written = rec->bytes_written;
0e2e63dd 1427 int i;
8d3eca20 1428 int rc = 0;
396b626b
AB		 1429 int nr_mmaps;
1430 struct mmap **maps;
d3d1af6f 1431 int trace_fd = rec->data.file.fd;
ef781128 1432 off_t off = 0;
98402807 1433
cb21686b
WN		 1434 if (!evlist)
1435 return 0;
ef149c25 1436
396b626b
AB		 1437 nr_mmaps = thread->nr_mmaps;
1438 maps = overwrite ? thread->overwrite_maps : thread->maps;
1439
a4ea0ec4
WN		 1440 if (!maps)
1441 return 0;
1442
0b72d69a 1443 if (overwrite && evlist->bkw_mmap_state != BKW_MMAP_DATA_PENDING)
54cc54de
WN		 1444 return 0;
1445
d3d1af6f
AB		 1446 if (record__aio_enabled(rec))
1447 off = record__aio_get_pos(trace_fd);
1448
396b626b 1449 for (i = 0; i < nr_mmaps; i++) {
470530bb 1450 u64 flush = 0;
396b626b 1451 struct mmap *map = maps[i];
cb21686b 1452
547740f7 1453 if (map->core.base) {
f13de660 1454 record__adjust_affinity(rec, map);
470530bb 1455 if (synch) {
65aa2e6b
JO		 1456 flush = map->core.flush;
1457 map->core.flush = 1;
470530bb 1458 }
d3d1af6f 1459 if (!record__aio_enabled(rec)) {
ef781128 1460 if (perf_mmap__push(map, rec, record__pushfn) < 0) {
470530bb 1461 if (synch)
65aa2e6b 1462 map->core.flush = flush;
d3d1af6f
AB		 1463 rc = -1;
1464 goto out;
1465 }
1466 } else {
ef781128 1467 if (record__aio_push(rec, map, &off) < 0) {
d3d1af6f 1468 record__aio_set_pos(trace_fd, off);
470530bb 1469 if (synch)
65aa2e6b 1470 map->core.flush = flush;
d3d1af6f
AB		 1471 rc = -1;
1472 goto out;
1473 }
8d3eca20 1474 }
470530bb 1475 if (synch)
65aa2e6b 1476 map->core.flush = flush;
8d3eca20 1477 }
ef149c25 1478
e035f4ca 1479 if (map->auxtrace_mmap.base && !rec->opts.auxtrace_snapshot_mode &&
c0a6de06 1480 !rec->opts.auxtrace_sample_mode &&
e035f4ca 1481 record__auxtrace_mmap_read(rec, map) != 0) {
ef149c25
AH		 1482 rc = -1;
1483 goto out;
1484 }
98402807
FW		 1485 }
1486
d3d1af6f
AB		 1487 if (record__aio_enabled(rec))
1488 record__aio_set_pos(trace_fd, off);
1489
dcabb507
JO		 1490 /*
1491 * Mark the round finished in case we wrote
1492 * at least one event.
56f735ff
AB		 1493 *
1494 * No need for round events in directory mode,
1495 * because per-cpu maps and files have data
1496 * sorted by kernel.
dcabb507 1497 */
56f735ff 1498 if (!record__threads_enabled(rec) && bytes_written != rec->bytes_written)
ded2b8fe 1499 rc = record__write(rec, NULL, &finished_round_event, sizeof(finished_round_event));
8d3eca20 1500
0b72d69a 1501 if (overwrite)
ade9d208 1502 evlist__toggle_bkw_mmap(evlist, BKW_MMAP_EMPTY);
8d3eca20
DA		 1503out:
1504 return rc;
98402807
FW		 1505}
1506
470530bb 1507static int record__mmap_read_all(struct record *rec, bool synch)
cb21686b
WN		 1508{
1509 int err;
1510
470530bb 1511 err = record__mmap_read_evlist(rec, rec->evlist, false, synch);
cb21686b
WN		 1512 if (err)
1513 return err;
1514
470530bb 1515 return record__mmap_read_evlist(rec, rec->evlist, true, synch);
cb21686b
WN		 1516}
1517
396b626b
AB		 1518static void record__thread_munmap_filtered(struct fdarray *fda, int fd,
1519 void *arg __maybe_unused)
1520{
1521 struct perf_mmap *map = fda->priv[fd].ptr;
1522
1523 if (map)
1524 perf_mmap__put(map);
1525}
1526
3217e9fe
AB		 1527static void *record__thread(void *arg)
1528{
1529 enum thread_msg msg = THREAD_MSG__READY;
1530 bool terminate = false;
1531 struct fdarray *pollfd;
1532 int err, ctlfd_pos;
1533
1534 thread = arg;
1535 thread->tid = gettid();
1536
1537 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1538 if (err == -1)
1539 pr_warning("threads[%d]: failed to notify on start: %s\n",
1540 thread->tid, strerror(errno));
1541
1542 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
1543
1544 pollfd = &thread->pollfd;
1545 ctlfd_pos = thread->ctlfd_pos;
1546
1547 for (;;) {
1548 unsigned long long hits = thread->samples;
1549
1550 if (record__mmap_read_all(thread->rec, false) < 0 || terminate)
1551 break;
1552
1553 if (hits == thread->samples) {
1554
1555 err = fdarray__poll(pollfd, -1);
1556 /*
1557 * Propagate error, only if there's any. Ignore positive
1558 * number of returned events and interrupt error.
1559 */
1560 if (err > 0 || (err < 0 && errno == EINTR))
1561 err = 0;
1562 thread->waking++;
1563
1564 if (fdarray__filter(pollfd, POLLERR | POLLHUP,
1565 record__thread_munmap_filtered, NULL) == 0)
1566 break;
1567 }
1568
1569 if (pollfd->entries[ctlfd_pos].revents & POLLHUP) {
1570 terminate = true;
1571 close(thread->pipes.msg[0]);
1572 thread->pipes.msg[0] = -1;
1573 pollfd->entries[ctlfd_pos].fd = -1;
1574 pollfd->entries[ctlfd_pos].events = 0;
1575 }
1576
1577 pollfd->entries[ctlfd_pos].revents = 0;
1578 }
1579 record__mmap_read_all(thread->rec, true);
1580
1581 err = write(thread->pipes.ack[1], &msg, sizeof(msg));
1582 if (err == -1)
1583 pr_warning("threads[%d]: failed to notify on termination: %s\n",
1584 thread->tid, strerror(errno));
1585
1586 return NULL;
1587}
1588
8c6f45a7 1589static void record__init_features(struct record *rec)
57706abc 1590{
57706abc
DA		 1591 struct perf_session *session = rec->session;
1592 int feat;
1593
1594 for (feat = HEADER_FIRST_FEATURE; feat < HEADER_LAST_FEATURE; feat++)
1595 perf_header__set_feat(&session->header, feat);
1596
1597 if (rec->no_buildid)
1598 perf_header__clear_feat(&session->header, HEADER_BUILD_ID);
1599
ce9036a6 1600 if (!have_tracepoints(&rec->evlist->core.entries))
57706abc
DA		 1601 perf_header__clear_feat(&session->header, HEADER_TRACING_DATA);
1602
1603 if (!rec->opts.branch_stack)
1604 perf_header__clear_feat(&session->header, HEADER_BRANCH_STACK);
ef149c25
AH		 1605
1606 if (!rec->opts.full_auxtrace)
1607 perf_header__clear_feat(&session->header, HEADER_AUXTRACE);
ffa517ad 1608
cf790516
AB		 1609 if (!(rec->opts.use_clockid && rec->opts.clockid_res_ns))
1610 perf_header__clear_feat(&session->header, HEADER_CLOCKID);
1611
d1e325cf
JO		 1612 if (!rec->opts.use_clockid)
1613 perf_header__clear_feat(&session->header, HEADER_CLOCK_DATA);
1614
56f735ff
AB		 1615 if (!record__threads_enabled(rec))
1616 perf_header__clear_feat(&session->header, HEADER_DIR_FORMAT);
1617
42e1fd80
AB		 1618 if (!record__comp_enabled(rec))
1619 perf_header__clear_feat(&session->header, HEADER_COMPRESSED);
258031c0 1620
ffa517ad 1621 perf_header__clear_feat(&session->header, HEADER_STAT);
57706abc
DA		 1622}
1623
e1ab48ba
WN		 1624static void
1625record__finish_output(struct record *rec)
1626{
56f735ff 1627 int i;
8ceb41d7
JO		 1628 struct perf_data *data = &rec->data;
1629 int fd = perf_data__fd(data);
e1ab48ba 1630
8ceb41d7 1631 if (data->is_pipe)
e1ab48ba
WN		 1632 return;
1633
1634 rec->session->header.data_size += rec->bytes_written;
45112e89 1635 data->file.size = lseek(perf_data__fd(data), 0, SEEK_CUR);
56f735ff
AB		 1636 if (record__threads_enabled(rec)) {
1637 for (i = 0; i < data->dir.nr; i++)
1638 data->dir.files[i].size = lseek(data->dir.files[i].fd, 0, SEEK_CUR);
1639 }
e1ab48ba
WN		 1640
1641 if (!rec->no_buildid) {
1642 process_buildids(rec);
1643
1644 if (rec->buildid_all)
1645 dsos__hit_all(rec->session);
1646 }
1647 perf_session__write_header(rec->session, rec->evlist, fd, true);
1648
1649 return;
1650}
1651
4ea648ae 1652static int record__synthesize_workload(struct record *rec, bool tail)
be7b0c9e 1653{
9d6aae72 1654 int err;
9749b90e 1655 struct perf_thread_map *thread_map;
41b740b6 1656 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
be7b0c9e 1657
4ea648ae
WN		 1658 if (rec->opts.tail_synthesize != tail)
1659 return 0;
1660
9d6aae72
ACM		 1661 thread_map = thread_map__new_by_tid(rec->evlist->workload.pid);
1662 if (thread_map == NULL)
1663 return -1;
1664
1665 err = perf_event__synthesize_thread_map(&rec->tool, thread_map,
be7b0c9e
WN		 1666 process_synthesized_event,
1667 &rec->session->machines.host,
41b740b6 1668 needs_mmap,
3fcb10e4 1669 rec->opts.sample_address);
7836e52e 1670 perf_thread_map__put(thread_map);
9d6aae72 1671 return err;
be7b0c9e
WN		 1672}
1673
4ea648ae 1674static int record__synthesize(struct record *rec, bool tail);
3c1cb7e3 1675
ecfd7a9c
WN		 1676static int
1677record__switch_output(struct record *rec, bool at_exit)
1678{
8ceb41d7 1679 struct perf_data *data = &rec->data;
ecfd7a9c 1680 int fd, err;
03724b2e 1681 char *new_filename;
ecfd7a9c
WN		 1682
1683 /* Same Size: "2015122520103046"*/
1684 char timestamp[] = "InvalidTimestamp";
1685
d3d1af6f
AB		 1686 record__aio_mmap_read_sync(rec);
1687
4ea648ae
WN		 1688 record__synthesize(rec, true);
1689 if (target__none(&rec->opts.target))
1690 record__synthesize_workload(rec, true);
1691
ecfd7a9c
WN		 1692 rec->samples = 0;
1693 record__finish_output(rec);
1694 err = fetch_current_timestamp(timestamp, sizeof(timestamp));
1695 if (err) {
1696 pr_err("Failed to get current timestamp\n");
1697 return -EINVAL;
1698 }
1699
8ceb41d7 1700 fd = perf_data__switch(data, timestamp,
ecfd7a9c 1701 rec->session->header.data_offset,
03724b2e 1702 at_exit, &new_filename);
ecfd7a9c
WN		 1703 if (fd >= 0 && !at_exit) {
1704 rec->bytes_written = 0;
1705 rec->session->header.data_size = 0;
1706 }
1707
1708 if (!quiet)
1709 fprintf(stderr, "[ perf record: Dump %s.%s ]\n",
2d4f2799 1710 data->path, timestamp);
3c1cb7e3 1711
03724b2e
AK		 1712 if (rec->switch_output.num_files) {
1713 int n = rec->switch_output.cur_file + 1;
1714
1715 if (n >= rec->switch_output.num_files)
1716 n = 0;
1717 rec->switch_output.cur_file = n;
1718 if (rec->switch_output.filenames[n]) {
1719 remove(rec->switch_output.filenames[n]);
d8f9da24 1720 zfree(&rec->switch_output.filenames[n]);
03724b2e
AK		 1721 }
1722 rec->switch_output.filenames[n] = new_filename;
1723 } else {
1724 free(new_filename);
1725 }
1726
3c1cb7e3 1727 /* Output tracking events */
be7b0c9e 1728 if (!at_exit) {
4ea648ae 1729 record__synthesize(rec, false);
3c1cb7e3 1730
be7b0c9e
WN		 1731 /*
1732 * In 'perf record --switch-output' without -a,
1733 * record__synthesize() in record__switch_output() won't
1734 * generate tracking events because there's no thread_map
1735 * in evlist. Which causes newly created perf.data doesn't
1736 * contain map and comm information.
1737 * Create a fake thread_map and directly call
1738 * perf_event__synthesize_thread_map() for those events.
1739 */
1740 if (target__none(&rec->opts.target))
4ea648ae 1741 record__synthesize_workload(rec, false);
be7b0c9e 1742 }
ecfd7a9c
WN		 1743 return fd;
1744}
1745
f33cbe72
ACM		 1746static volatile int workload_exec_errno;
1747
1748/*
7b392ef0 1749 * evlist__prepare_workload will send a SIGUSR1
f33cbe72
ACM		 1750 * if the fork fails, since we asked by setting its
1751 * want_signal to true.
1752 */
45604710
NK		 1753static void workload_exec_failed_signal(int signo __maybe_unused,
1754 siginfo_t *info,
f33cbe72
ACM		 1755 void *ucontext __maybe_unused)
1756{
1757 workload_exec_errno = info->si_value.sival_int;
1758 done = 1;
f33cbe72
ACM		 1759 child_finished = 1;
1760}
1761
2dd6d8a1 1762static void snapshot_sig_handler(int sig);
bfacbe3b 1763static void alarm_sig_handler(int sig);
2dd6d8a1 1764
db0ea13c 1765static const struct perf_event_mmap_page *evlist__pick_pc(struct evlist *evlist)
ee667f94 1766{
b2cb615d 1767 if (evlist) {
547740f7
JO		 1768 if (evlist->mmap && evlist->mmap[0].core.base)
1769 return evlist->mmap[0].core.base;
1770 if (evlist->overwrite_mmap && evlist->overwrite_mmap[0].core.base)
1771 return evlist->overwrite_mmap[0].core.base;
b2cb615d 1772 }
ee667f94
WN		 1773 return NULL;
1774}
1775
c45628b0
WN		 1776static const struct perf_event_mmap_page *record__pick_pc(struct record *rec)
1777{
db0ea13c 1778 const struct perf_event_mmap_page *pc = evlist__pick_pc(rec->evlist);
ee667f94
WN		 1779 if (pc)
1780 return pc;
c45628b0
WN		 1781 return NULL;
1782}
1783
4ea648ae 1784static int record__synthesize(struct record *rec, bool tail)
c45c86eb
WN		 1785{
1786 struct perf_session *session = rec->session;
1787 struct machine *machine = &session->machines.host;
8ceb41d7 1788 struct perf_data *data = &rec->data;
c45c86eb
WN		 1789 struct record_opts *opts = &rec->opts;
1790 struct perf_tool *tool = &rec->tool;
c45c86eb 1791 int err = 0;
d99c22ea 1792 event_op f = process_synthesized_event;
c45c86eb 1793
4ea648ae
WN		 1794 if (rec->opts.tail_synthesize != tail)
1795 return 0;
1796
8ceb41d7 1797 if (data->is_pipe) {
c3a057dc 1798 err = perf_event__synthesize_for_pipe(tool, session, data,
a2015516 1799 process_synthesized_event);
c3a057dc
NK		 1800 if (err < 0)
1801 goto out;
a2015516 1802
c3a057dc 1803 rec->bytes_written += err;
c45c86eb
WN		 1804 }
1805
c45628b0 1806 err = perf_event__synth_time_conv(record__pick_pc(rec), tool,
46bc29b9
AH		 1807 process_synthesized_event, machine);
1808 if (err)
1809 goto out;
1810
c0a6de06 1811 /* Synthesize id_index before auxtrace_info */
61750473 1812 if (rec->opts.auxtrace_sample_mode || rec->opts.full_auxtrace) {
c0a6de06
AH		 1813 err = perf_event__synthesize_id_index(tool,
1814 process_synthesized_event,
1815 session->evlist, machine);
1816 if (err)
1817 goto out;
1818 }
1819
c45c86eb
WN		 1820 if (rec->opts.full_auxtrace) {
1821 err = perf_event__synthesize_auxtrace_info(rec->itr, tool,
1822 session, process_synthesized_event);
1823 if (err)
1824 goto out;
1825 }
1826
78e1bc25 1827 if (!evlist__exclude_kernel(rec->evlist)) {
6c443954
ACM		 1828 err = perf_event__synthesize_kernel_mmap(tool, process_synthesized_event,
1829 machine);
1830 WARN_ONCE(err < 0, "Couldn't record kernel reference relocation symbol\n"
1831 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
1832 "Check /proc/kallsyms permission or run as root.\n");
1833
1834 err = perf_event__synthesize_modules(tool, process_synthesized_event,
1835 machine);
1836 WARN_ONCE(err < 0, "Couldn't record kernel module information.\n"
1837 "Symbol resolution may be skewed if relocation was used (e.g. kexec).\n"
1838 "Check /proc/modules permission or run as root.\n");
1839 }
c45c86eb
WN		 1840
1841 if (perf_guest) {
1842 machines__process_guests(&session->machines,
1843 perf_event__synthesize_guest_os, tool);
1844 }
1845
bfd8f72c
AK		 1846 err = perf_event__synthesize_extra_attr(&rec->tool,
1847 rec->evlist,
1848 process_synthesized_event,
1849 data->is_pipe);
1850 if (err)
1851 goto out;
1852
03617c22 1853 err = perf_event__synthesize_thread_map2(&rec->tool, rec->evlist->core.threads,
373565d2
AK		 1854 process_synthesized_event,
1855 NULL);
1856 if (err < 0) {
1857 pr_err("Couldn't synthesize thread map.\n");
1858 return err;
1859 }
1860
f72f901d 1861 err = perf_event__synthesize_cpu_map(&rec->tool, rec->evlist->core.cpus,
373565d2
AK		 1862 process_synthesized_event, NULL);
1863 if (err < 0) {
1864 pr_err("Couldn't synthesize cpu map.\n");
1865 return err;
1866 }
1867
e5416950 1868 err = perf_event__synthesize_bpf_events(session, process_synthesized_event,
7b612e29
SL		 1869 machine, opts);
1870 if (err < 0)
1871 pr_warning("Couldn't synthesize bpf events.\n");
1872
41b740b6
NK		 1873 if (rec->opts.synth & PERF_SYNTH_CGROUP) {
1874 err = perf_event__synthesize_cgroups(tool, process_synthesized_event,
1875 machine);
1876 if (err < 0)
1877 pr_warning("Couldn't synthesize cgroup events.\n");
1878 }
ab64069f 1879
d99c22ea
SE		 1880 if (rec->opts.nr_threads_synthesize > 1) {
1881 perf_set_multithreaded();
1882 f = process_locked_synthesized_event;
1883 }
1884
41b740b6
NK		 1885 if (rec->opts.synth & PERF_SYNTH_TASK) {
1886 bool needs_mmap = rec->opts.synth & PERF_SYNTH_MMAP;
1887
1888 err = __machine__synthesize_threads(machine, tool, &opts->target,
1889 rec->evlist->core.threads,
1890 f, needs_mmap, opts->sample_address,
1891 rec->opts.nr_threads_synthesize);
1892 }
d99c22ea
SE		 1893
1894 if (rec->opts.nr_threads_synthesize > 1)
1895 perf_set_singlethreaded();
1896
c45c86eb
WN		 1897out:
1898 return err;
1899}
1900
899e5ffb
ACM		 1901static int record__process_signal_event(union perf_event *event __maybe_unused, void *data)
1902{
1903 struct record *rec = data;
1904 pthread_kill(rec->thread_id, SIGUSR2);
1905 return 0;
1906}
1907
23cbb41c
ACM		 1908static int record__setup_sb_evlist(struct record *rec)
1909{
1910 struct record_opts *opts = &rec->opts;
1911
1912 if (rec->sb_evlist != NULL) {
1913 /*
1914 * We get here if --switch-output-event populated the
1915 * sb_evlist, so associate a callback that will send a SIGUSR2
1916 * to the main thread.
1917 */
1918 evlist__set_cb(rec->sb_evlist, record__process_signal_event, rec);
1919 rec->thread_id = pthread_self();
1920 }
1101c872 1921#ifdef HAVE_LIBBPF_SUPPORT
23cbb41c
ACM		 1922 if (!opts->no_bpf_event) {
1923 if (rec->sb_evlist == NULL) {
1924 rec->sb_evlist = evlist__new();
1925
1926 if (rec->sb_evlist == NULL) {
1927 pr_err("Couldn't create side band evlist.\n.");
1928 return -1;
1929 }
1930 }
1931
1932 if (evlist__add_bpf_sb_event(rec->sb_evlist, &rec->session->header.env)) {
1933 pr_err("Couldn't ask for PERF_RECORD_BPF_EVENT side band events.\n.");
1934 return -1;
1935 }
1936 }
1101c872 1937#endif
08c83997 1938 if (evlist__start_sb_thread(rec->sb_evlist, &rec->opts.target)) {
23cbb41c
ACM		 1939 pr_debug("Couldn't start the BPF side band thread:\nBPF programs starting from now on won't be annotatable\n");
1940 opts->no_bpf_event = true;
1941 }
1942
1943 return 0;
1944}
1945
d1e325cf
JO		 1946static int record__init_clock(struct record *rec)
1947{
1948 struct perf_session *session = rec->session;
1949 struct timespec ref_clockid;
1950 struct timeval ref_tod;
1951 u64 ref;
1952
1953 if (!rec->opts.use_clockid)
1954 return 0;
1955
9d88a1a1
JO		 1956 if (rec->opts.use_clockid && rec->opts.clockid_res_ns)
1957 session->header.env.clock.clockid_res_ns = rec->opts.clockid_res_ns;
1958
d1e325cf
JO		 1959 session->header.env.clock.clockid = rec->opts.clockid;
1960
1961 if (gettimeofday(&ref_tod, NULL) != 0) {
1962 pr_err("gettimeofday failed, cannot set reference time.\n");
1963 return -1;
1964 }
1965
1966 if (clock_gettime(rec->opts.clockid, &ref_clockid)) {
1967 pr_err("clock_gettime failed, cannot set reference time.\n");
1968 return -1;
1969 }
1970
1971 ref = (u64) ref_tod.tv_sec * NSEC_PER_SEC +
1972 (u64) ref_tod.tv_usec * NSEC_PER_USEC;
1973
1974 session->header.env.clock.tod_ns = ref;
1975
1976 ref = (u64) ref_clockid.tv_sec * NSEC_PER_SEC +
1977 (u64) ref_clockid.tv_nsec;
1978
1979 session->header.env.clock.clockid_ns = ref;
1980 return 0;
1981}
1982
d20aff15
AH		 1983static void hit_auxtrace_snapshot_trigger(struct record *rec)
1984{
1985 if (trigger_is_ready(&auxtrace_snapshot_trigger)) {
1986 trigger_hit(&auxtrace_snapshot_trigger);
1987 auxtrace_record__snapshot_started = 1;
1988 if (auxtrace_record__snapshot_start(rec->itr))
1989 trigger_error(&auxtrace_snapshot_trigger);
1990 }
1991}
1992
91c0f5ec
JY		 1993static void record__uniquify_name(struct record *rec)
1994{
1995 struct evsel *pos;
1996 struct evlist *evlist = rec->evlist;
1997 char *new_name;
1998 int ret;
1999
2000 if (!perf_pmu__has_hybrid())
2001 return;
2002
2003 evlist__for_each_entry(evlist, pos) {
2004 if (!evsel__is_hybrid(pos))
2005 continue;
2006
2007 if (strchr(pos->name, '/'))
2008 continue;
2009
2010 ret = asprintf(&new_name, "%s/%s/",
2011 pos->pmu_name, pos->name);
2012 if (ret) {
2013 free(pos->name);
2014 pos->name = new_name;
2015 }
2016 }
2017}
2018
1e5de7d9
AB		 2019static int record__terminate_thread(struct record_thread *thread_data)
2020{
2021 int err;
2022 enum thread_msg ack = THREAD_MSG__UNDEFINED;
2023 pid_t tid = thread_data->tid;
2024
2025 close(thread_data->pipes.msg[1]);
2026 thread_data->pipes.msg[1] = -1;
2027 err = read(thread_data->pipes.ack[0], &ack, sizeof(ack));
2028 if (err > 0)
2029 pr_debug2("threads[%d]: sent %s\n", tid, thread_msg_tags[ack]);
2030 else
2031 pr_warning("threads[%d]: failed to receive termination notification from %d\n",
2032 thread->tid, tid);
2033
2034 return 0;
2035}
2036
396b626b
AB		 2037static int record__start_threads(struct record *rec)
2038{
3217e9fe 2039 int t, tt, err, ret = 0, nr_threads = rec->nr_threads;
396b626b 2040 struct record_thread *thread_data = rec->thread_data;
3217e9fe
AB		 2041 sigset_t full, mask;
2042 pthread_t handle;
2043 pthread_attr_t attrs;
396b626b
AB		 2044
2045 thread = &thread_data[0];
2046
3217e9fe
AB		 2047 if (!record__threads_enabled(rec))
2048 return 0;
2049
2050 sigfillset(&full);
2051 if (sigprocmask(SIG_SETMASK, &full, &mask)) {
2052 pr_err("Failed to block signals on threads start: %s\n", strerror(errno));
2053 return -1;
2054 }
2055
2056 pthread_attr_init(&attrs);
2057 pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED);
2058
2059 for (t = 1; t < nr_threads; t++) {
2060 enum thread_msg msg = THREAD_MSG__UNDEFINED;
2061
2062#ifdef HAVE_PTHREAD_ATTR_SETAFFINITY_NP
2063 pthread_attr_setaffinity_np(&attrs,
2064 MMAP_CPU_MASK_BYTES(&(thread_data[t].mask->affinity)),
2065 (cpu_set_t *)(thread_data[t].mask->affinity.bits));
2066#endif
2067 if (pthread_create(&handle, &attrs, record__thread, &thread_data[t])) {
2068 for (tt = 1; tt < t; tt++)
2069 record__terminate_thread(&thread_data[t]);
2070 pr_err("Failed to start threads: %s\n", strerror(errno));
2071 ret = -1;
2072 goto out_err;
2073 }
2074
2075 err = read(thread_data[t].pipes.ack[0], &msg, sizeof(msg));
2076 if (err > 0)
2077 pr_debug2("threads[%d]: sent %s\n", rec->thread_data[t].tid,
2078 thread_msg_tags[msg]);
2079 else
2080 pr_warning("threads[%d]: failed to receive start notification from %d\n",
2081 thread->tid, rec->thread_data[t].tid);
2082 }
2083
2084 sched_setaffinity(0, MMAP_CPU_MASK_BYTES(&thread->mask->affinity),
2085 (cpu_set_t *)thread->mask->affinity.bits);
2086
396b626b
AB		 2087 pr_debug("threads[%d]: started on cpu%d\n", thread->tid, sched_getcpu());
2088
3217e9fe
AB		 2089out_err:
2090 pthread_attr_destroy(&attrs);
2091
2092 if (sigprocmask(SIG_SETMASK, &mask, NULL)) {
2093 pr_err("Failed to unblock signals on threads start: %s\n", strerror(errno));
2094 ret = -1;
2095 }
2096
2097 return ret;
396b626b
AB		 2098}
2099
2100static int record__stop_threads(struct record *rec)
2101{
2102 int t;
2103 struct record_thread *thread_data = rec->thread_data;
2104
1e5de7d9
AB		 2105 for (t = 1; t < rec->nr_threads; t++)
2106 record__terminate_thread(&thread_data[t]);
2107
610fbc01 2108 for (t = 0; t < rec->nr_threads; t++) {
396b626b 2109 rec->samples += thread_data[t].samples;
610fbc01
AB		 2110 if (!record__threads_enabled(rec))
2111 continue;
2112 rec->session->bytes_transferred += thread_data[t].bytes_transferred;
2113 rec->session->bytes_compressed += thread_data[t].bytes_compressed;
2114 pr_debug("threads[%d]: samples=%lld, wakes=%ld, ", thread_data[t].tid,
2115 thread_data[t].samples, thread_data[t].waking);
2116 if (thread_data[t].bytes_transferred && thread_data[t].bytes_compressed)
2117 pr_debug("transferred=%" PRIu64 ", compressed=%" PRIu64 "\n",
2118 thread_data[t].bytes_transferred, thread_data[t].bytes_compressed);
2119 else
2120 pr_debug("written=%" PRIu64 "\n", thread_data[t].bytes_written);
2121 }
396b626b
AB		 2122
2123 return 0;
2124}
2125
2126static unsigned long record__waking(struct record *rec)
2127{
2128 int t;
2129 unsigned long waking = 0;
2130 struct record_thread *thread_data = rec->thread_data;
2131
2132 for (t = 0; t < rec->nr_threads; t++)
2133 waking += thread_data[t].waking;
2134
2135 return waking;
2136}
2137
8c6f45a7 2138static int __cmd_record(struct record *rec, int argc, const char **argv)
16c8a109 2139{
57706abc 2140 int err;
45604710 2141 int status = 0;
46be604b 2142 const bool forks = argc > 0;
45694aa7 2143 struct perf_tool *tool = &rec->tool;
b4006796 2144 struct record_opts *opts = &rec->opts;
8ceb41d7 2145 struct perf_data *data = &rec->data;
d20deb64 2146 struct perf_session *session;
6dcf45ef 2147 bool disabled = false, draining = false;
42aa276f 2148 int fd;
d3c8c08e 2149 float ratio = 0;
acce0223 2150 enum evlist_ctl_cmd cmd = EVLIST_CTL_CMD_UNSUPPORTED;
de9ac07b 2151
45604710 2152 atexit(record__sig_exit);
f5970550
PZ		 2153 signal(SIGCHLD, sig_handler);
2154 signal(SIGINT, sig_handler);
804f7ac7 2155 signal(SIGTERM, sig_handler);
a074865e 2156 signal(SIGSEGV, sigsegv_handler);
c0bdc1c4 2157
f3b3614a
HB		 2158 if (rec->opts.record_namespaces)
2159 tool->namespace_events = true;
2160
8fb4b679
NK		 2161 if (rec->opts.record_cgroup) {
2162#ifdef HAVE_FILE_HANDLE
2163 tool->cgroup_events = true;
2164#else
2165 pr_err("cgroup tracking is not supported\n");
2166 return -1;
2167#endif
2168 }
2169
dc0c6127 2170 if (rec->opts.auxtrace_snapshot_mode || rec->switch_output.enabled) {
2dd6d8a1 2171 signal(SIGUSR2, snapshot_sig_handler);
3c1cb7e3
WN		 2172 if (rec->opts.auxtrace_snapshot_mode)
2173 trigger_on(&auxtrace_snapshot_trigger);
dc0c6127 2174 if (rec->switch_output.enabled)
3c1cb7e3 2175 trigger_on(&switch_output_trigger);
c0bdc1c4 2176 } else {
2dd6d8a1 2177 signal(SIGUSR2, SIG_IGN);
c0bdc1c4 2178 }
f5970550 2179
2681bd85 2180 session = perf_session__new(data, tool);
6ef81c55 2181 if (IS_ERR(session)) {
ffa91880 2182 pr_err("Perf session creation failed.\n");
6ef81c55 2183 return PTR_ERR(session);
a9a70bbc
ACM		 2184 }
2185
8ceb41d7 2186 fd = perf_data__fd(data);
d20deb64
ACM		 2187 rec->session = session;
2188
5d7f4116
AB		 2189 if (zstd_init(&session->zstd_data, rec->opts.comp_level) < 0) {
2190 pr_err("Compression initialization failed.\n");
2191 return -1;
2192 }
da231338
AM		 2193#ifdef HAVE_EVENTFD_SUPPORT
2194 done_fd = eventfd(0, EFD_NONBLOCK);
2195 if (done_fd < 0) {
2196 pr_err("Failed to create wakeup eventfd, error: %m\n");
2197 status = -1;
2198 goto out_delete_session;
2199 }
e16c2ce7 2200 err = evlist__add_wakeup_eventfd(rec->evlist, done_fd);
da231338
AM		 2201 if (err < 0) {
2202 pr_err("Failed to add wakeup eventfd to poll list\n");
2203 status = err;
2204 goto out_delete_session;
2205 }
2206#endif // HAVE_EVENTFD_SUPPORT
5d7f4116
AB		 2207
2208 session->header.env.comp_type = PERF_COMP_ZSTD;
2209 session->header.env.comp_level = rec->opts.comp_level;
2210
eeb399b5
AH		 2211 if (rec->opts.kcore &&
2212 !record__kcore_readable(&session->machines.host)) {
2213 pr_err("ERROR: kcore is not readable.\n");
2214 return -1;
2215 }
2216
d1e325cf
JO		 2217 if (record__init_clock(rec))
2218 return -1;
2219
8c6f45a7 2220 record__init_features(rec);
330aa675 2221
d4db3f16 2222 if (forks) {
7b392ef0
ACM		 2223 err = evlist__prepare_workload(rec->evlist, &opts->target, argv, data->is_pipe,
2224 workload_exec_failed_signal);
35b9d88e
ACM		 2225 if (err < 0) {
2226 pr_err("Couldn't run the workload!\n");
45604710 2227 status = err;
35b9d88e 2228 goto out_delete_session;
856e9660 2229 }
856e9660
PZ		 2230 }
2231
ad46e48c
JO		 2232 /*
2233 * If we have just single event and are sending data
2234 * through pipe, we need to force the ids allocation,
2235 * because we synthesize event name through the pipe
2236 * and need the id for that.
2237 */
6484d2f9 2238 if (data->is_pipe && rec->evlist->core.nr_entries == 1)
ad46e48c
JO		 2239 rec->opts.sample_id = true;
2240
91c0f5ec
JY		 2241 record__uniquify_name(rec);
2242
8c6f45a7 2243 if (record__open(rec) != 0) {
8d3eca20 2244 err = -1;
396b626b 2245 goto out_free_threads;
8d3eca20 2246 }
f6fa4375 2247 session->header.env.comp_mmap_len = session->evlist->core.mmap_len;
de9ac07b 2248
eeb399b5
AH		 2249 if (rec->opts.kcore) {
2250 err = record__kcore_copy(&session->machines.host, data);
2251 if (err) {
2252 pr_err("ERROR: Failed to copy kcore\n");
396b626b 2253 goto out_free_threads;
eeb399b5
AH		 2254 }
2255 }
2256
8690a2a7
WN		 2257 err = bpf__apply_obj_config();
2258 if (err) {
2259 char errbuf[BUFSIZ];
2260
2261 bpf__strerror_apply_obj_config(err, errbuf, sizeof(errbuf));
2262 pr_err("ERROR: Apply config to BPF failed: %s\n",
2263 errbuf);
396b626b 2264 goto out_free_threads;
8690a2a7
WN		 2265 }
2266
cca8482c
AH		 2267 /*
2268 * Normally perf_session__new would do this, but it doesn't have the
2269 * evlist.
2270 */
8cedf3a5 2271 if (rec->tool.ordered_events && !evlist__sample_id_all(rec->evlist)) {
cca8482c
AH		 2272 pr_warning("WARNING: No sample_id_all support, falling back to unordered processing\n");
2273 rec->tool.ordered_events = false;
2274 }
2275
3a683120 2276 if (!rec->evlist->core.nr_groups)
a8bb559b
NK		 2277 perf_header__clear_feat(&session->header, HEADER_GROUP_DESC);
2278
8ceb41d7 2279 if (data->is_pipe) {
42aa276f 2280 err = perf_header__write_pipe(fd);
529870e3 2281 if (err < 0)
396b626b 2282 goto out_free_threads;
563aecb2 2283 } else {
42aa276f 2284 err = perf_session__write_header(session, rec->evlist, fd, false);
d5eed904 2285 if (err < 0)
396b626b 2286 goto out_free_threads;
56b03f3c
ACM		 2287 }
2288
b38d85ef 2289 err = -1;
d3665498 2290 if (!rec->no_buildid
e20960c0 2291 && !perf_header__has_feat(&session->header, HEADER_BUILD_ID)) {
d3665498 2292 pr_err("Couldn't generate buildids. "
e20960c0 2293 "Use --no-buildid to profile anyway.\n");
396b626b 2294 goto out_free_threads;
e20960c0
RR		 2295 }
2296
23cbb41c
ACM		 2297 err = record__setup_sb_evlist(rec);
2298 if (err)
396b626b 2299 goto out_free_threads;
657ee553 2300
4ea648ae 2301 err = record__synthesize(rec, false);
c45c86eb 2302 if (err < 0)
396b626b 2303 goto out_free_threads;
8d3eca20 2304
d20deb64 2305 if (rec->realtime_prio) {
de9ac07b
PZ		 2306 struct sched_param param;
2307
d20deb64 2308 param.sched_priority = rec->realtime_prio;
de9ac07b 2309 if (sched_setscheduler(0, SCHED_FIFO, &param)) {
6beba7ad 2310 pr_err("Could not set realtime priority.\n");
8d3eca20 2311 err = -1;
396b626b 2312 goto out_free_threads;
de9ac07b
PZ		 2313 }
2314 }
2315
396b626b
AB		 2316 if (record__start_threads(rec))
2317 goto out_free_threads;
2318
774cb499
JO		 2319 /*
2320 * When perf is starting the traced process, all the events
2321 * (apart from group members) have enable_on_exec=1 set,
2322 * so don't spoil it by prematurely enabling them.
2323 */
6619a53e 2324 if (!target__none(&opts->target) && !opts->initial_delay)
1c87f165 2325 evlist__enable(rec->evlist);
764e16a3 2326
856e9660
PZ		 2327 /*
2328 * Let the child rip
2329 */
e803cf97 2330 if (forks) {
20a8a3cf 2331 struct machine *machine = &session->machines.host;
e5bed564 2332 union perf_event *event;
e907caf3 2333 pid_t tgid;
e5bed564
NK		 2334
2335 event = malloc(sizeof(event->comm) + machine->id_hdr_size);
2336 if (event == NULL) {
2337 err = -ENOMEM;
2338 goto out_child;
2339 }
2340
e803cf97
NK		 2341 /*
2342 * Some H/W events are generated before COMM event
2343 * which is emitted during exec(), so perf script
2344 * cannot see a correct process name for those events.
2345 * Synthesize COMM event to prevent it.
2346 */
e907caf3
HB		 2347 tgid = perf_event__synthesize_comm(tool, event,
2348 rec->evlist->workload.pid,
2349 process_synthesized_event,
2350 machine);
2351 free(event);
2352
2353 if (tgid == -1)
2354 goto out_child;
2355
2356 event = malloc(sizeof(event->namespaces) +
2357 (NR_NAMESPACES * sizeof(struct perf_ns_link_info)) +
2358 machine->id_hdr_size);
2359 if (event == NULL) {
2360 err = -ENOMEM;
2361 goto out_child;
2362 }
2363
2364 /*
2365 * Synthesize NAMESPACES event for the command specified.
2366 */
2367 perf_event__synthesize_namespaces(tool, event,
2368 rec->evlist->workload.pid,
2369 tgid, process_synthesized_event,
2370 machine);
e5bed564 2371 free(event);
e803cf97 2372
7b392ef0 2373 evlist__start_workload(rec->evlist);
e803cf97 2374 }
856e9660 2375
6619a53e 2376 if (opts->initial_delay) {
68cd3b45
AB		 2377 pr_info(EVLIST_DISABLED_MSG);
2378 if (opts->initial_delay > 0) {
2379 usleep(opts->initial_delay * USEC_PER_MSEC);
2380 evlist__enable(rec->evlist);
2381 pr_info(EVLIST_ENABLED_MSG);
2382 }
6619a53e
AK		 2383 }
2384
5f9cf599 2385 trigger_ready(&auxtrace_snapshot_trigger);
3c1cb7e3 2386 trigger_ready(&switch_output_trigger);
a074865e 2387 perf_hooks__invoke_record_start();
649c48a9 2388 for (;;) {
396b626b 2389 unsigned long long hits = thread->samples;
de9ac07b 2390
05737464
WN		 2391 /*
2392 * rec->evlist->bkw_mmap_state is possible to be
2393 * BKW_MMAP_EMPTY here: when done == true and
2394 * hits != rec->samples in previous round.
2395 *
ade9d208 2396 * evlist__toggle_bkw_mmap ensure we never
05737464
WN		 2397 * convert BKW_MMAP_EMPTY to BKW_MMAP_DATA_PENDING.
2398 */
2399 if (trigger_is_hit(&switch_output_trigger) || done || draining)
ade9d208 2400 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_DATA_PENDING);
05737464 2401
470530bb 2402 if (record__mmap_read_all(rec, false) < 0) {
5f9cf599 2403 trigger_error(&auxtrace_snapshot_trigger);
3c1cb7e3 2404 trigger_error(&switch_output_trigger);
8d3eca20 2405 err = -1;
45604710 2406 goto out_child;
8d3eca20 2407 }
de9ac07b 2408
2dd6d8a1
AH		 2409 if (auxtrace_record__snapshot_started) {
2410 auxtrace_record__snapshot_started = 0;
5f9cf599 2411 if (!trigger_is_error(&auxtrace_snapshot_trigger))
ce7b0e42 2412 record__read_auxtrace_snapshot(rec, false);
5f9cf599 2413 if (trigger_is_error(&auxtrace_snapshot_trigger)) {
2dd6d8a1
AH		 2414 pr_err("AUX area tracing snapshot failed\n");
2415 err = -1;
2416 goto out_child;
2417 }
2418 }
2419
3c1cb7e3 2420 if (trigger_is_hit(&switch_output_trigger)) {
05737464
WN		 2421 /*
2422 * If switch_output_trigger is hit, the data in
2423 * overwritable ring buffer should have been collected,
2424 * so bkw_mmap_state should be set to BKW_MMAP_EMPTY.
2425 *
2426 * If SIGUSR2 raise after or during record__mmap_read_all(),
2427 * record__mmap_read_all() didn't collect data from
2428 * overwritable ring buffer. Read again.
2429 */
2430 if (rec->evlist->bkw_mmap_state == BKW_MMAP_RUNNING)
2431 continue;
3c1cb7e3
WN		 2432 trigger_ready(&switch_output_trigger);
2433
05737464
WN		 2434 /*
2435 * Reenable events in overwrite ring buffer after
2436 * record__mmap_read_all(): we should have collected
2437 * data from it.
2438 */
ade9d208 2439 evlist__toggle_bkw_mmap(rec->evlist, BKW_MMAP_RUNNING);
05737464 2440
3c1cb7e3
WN		 2441 if (!quiet)
2442 fprintf(stderr, "[ perf record: dump data: Woken up %ld times ]\n",
396b626b
AB		 2443 record__waking(rec));
2444 thread->waking = 0;
3c1cb7e3
WN		 2445 fd = record__switch_output(rec, false);
2446 if (fd < 0) {
2447 pr_err("Failed to switch to new file\n");
2448 trigger_error(&switch_output_trigger);
2449 err = fd;
2450 goto out_child;
2451 }
bfacbe3b
JO		 2452
2453 /* re-arm the alarm */
2454 if (rec->switch_output.time)
2455 alarm(rec->switch_output.time);
3c1cb7e3
WN		 2456 }
2457
396b626b 2458 if (hits == thread->samples) {
6dcf45ef 2459 if (done || draining)
649c48a9 2460 break;
396b626b 2461 err = fdarray__poll(&thread->pollfd, -1);
a515114f
JO		 2462 /*
2463 * Propagate error, only if there's any. Ignore positive
2464 * number of returned events and interrupt error.
2465 */
2466 if (err > 0 || (err < 0 && errno == EINTR))
45604710 2467 err = 0;
396b626b 2468 thread->waking++;
6dcf45ef 2469
396b626b
AB		 2470 if (fdarray__filter(&thread->pollfd, POLLERR | POLLHUP,
2471 record__thread_munmap_filtered, NULL) == 0)
6dcf45ef 2472 draining = true;
396b626b
AB		 2473
2474 evlist__ctlfd_update(rec->evlist,
2475 &thread->pollfd.entries[thread->ctlfd_pos]);
8b412664
PZ		 2476 }
2477
acce0223
AB		 2478 if (evlist__ctlfd_process(rec->evlist, &cmd) > 0) {
2479 switch (cmd) {
d20aff15
AH		 2480 case EVLIST_CTL_CMD_SNAPSHOT:
2481 hit_auxtrace_snapshot_trigger(rec);
2482 evlist__ctlfd_ack(rec->evlist);
2483 break;
f186cd61
JO		 2484 case EVLIST_CTL_CMD_STOP:
2485 done = 1;
2486 break;
acce0223
AB		 2487 case EVLIST_CTL_CMD_ACK:
2488 case EVLIST_CTL_CMD_UNSUPPORTED:
991ae4eb
JO		 2489 case EVLIST_CTL_CMD_ENABLE:
2490 case EVLIST_CTL_CMD_DISABLE:
142544a9 2491 case EVLIST_CTL_CMD_EVLIST:
47fddcb4 2492 case EVLIST_CTL_CMD_PING:
acce0223
AB		 2493 default:
2494 break;
2495 }
2496 }
2497
774cb499
JO		 2498 /*
2499 * When perf is starting the traced process, at the end events
2500 * die with the process and we wait for that. Thus no need to
2501 * disable events in this case.
2502 */
602ad878 2503 if (done && !disabled && !target__none(&opts->target)) {
5f9cf599 2504 trigger_off(&auxtrace_snapshot_trigger);
e74676de 2505 evlist__disable(rec->evlist);
2711926a
JO		 2506 disabled = true;
2507 }
de9ac07b 2508 }
ce7b0e42 2509
5f9cf599 2510 trigger_off(&auxtrace_snapshot_trigger);
3c1cb7e3 2511 trigger_off(&switch_output_trigger);
de9ac07b 2512
ce7b0e42
AS		 2513 if (opts->auxtrace_snapshot_on_exit)
2514 record__auxtrace_snapshot_exit(rec);
2515
f33cbe72 2516 if (forks && workload_exec_errno) {
3535a696 2517 char msg[STRERR_BUFSIZE], strevsels[2048];
c8b5f2c9 2518 const char *emsg = str_error_r(workload_exec_errno, msg, sizeof(msg));
3535a696
ACM		 2519
2520 evlist__scnprintf_evsels(rec->evlist, sizeof(strevsels), strevsels);
2521
2522 pr_err("Failed to collect '%s' for the '%s' workload: %s\n",
2523 strevsels, argv[0], emsg);
f33cbe72 2524 err = -1;
45604710 2525 goto out_child;
f33cbe72
ACM		 2526 }
2527
e3d59112 2528 if (!quiet)
396b626b
AB		 2529 fprintf(stderr, "[ perf record: Woken up %ld times to write data ]\n",
2530 record__waking(rec));
b44308f5 2531
4ea648ae
WN		 2532 if (target__none(&rec->opts.target))
2533 record__synthesize_workload(rec, true);
2534
45604710 2535out_child:
396b626b 2536 record__stop_threads(rec);
470530bb 2537 record__mmap_read_all(rec, true);
396b626b 2538out_free_threads:
415ccb58 2539 record__free_thread_data(rec);
396b626b 2540 evlist__finalize_ctlfd(rec->evlist);
d3d1af6f
AB		 2541 record__aio_mmap_read_sync(rec);
2542
d3c8c08e
AB		 2543 if (rec->session->bytes_transferred && rec->session->bytes_compressed) {
2544 ratio = (float)rec->session->bytes_transferred/(float)rec->session->bytes_compressed;
2545 session->header.env.comp_ratio = ratio + 0.5;
2546 }
2547
45604710
NK		 2548 if (forks) {
2549 int exit_status;
addc2785 2550
45604710
NK		 2551 if (!child_finished)
2552 kill(rec->evlist->workload.pid, SIGTERM);
2553
2554 wait(&exit_status);
2555
2556 if (err < 0)
2557 status = err;
2558 else if (WIFEXITED(exit_status))
2559 status = WEXITSTATUS(exit_status);
2560 else if (WIFSIGNALED(exit_status))
2561 signr = WTERMSIG(exit_status);
2562 } else
2563 status = err;
2564
4ea648ae 2565 record__synthesize(rec, true);
e3d59112
NK		 2566 /* this will be recalculated during process_buildids() */
2567 rec->samples = 0;
2568
ecfd7a9c
WN		 2569 if (!err) {
2570 if (!rec->timestamp_filename) {
2571 record__finish_output(rec);
2572 } else {
2573 fd = record__switch_output(rec, true);
2574 if (fd < 0) {
2575 status = fd;
2576 goto out_delete_session;
2577 }
2578 }
2579 }
39d17dac 2580
a074865e
WN		 2581 perf_hooks__invoke_record_end();
2582
e3d59112
NK		 2583 if (!err && !quiet) {
2584 char samples[128];
ecfd7a9c
WN		 2585 const char *postfix = rec->timestamp_filename ?
2586 ".<timestamp>" : "";
e3d59112 2587
ef149c25 2588 if (rec->samples && !rec->opts.full_auxtrace)
e3d59112
NK		 2589 scnprintf(samples, sizeof(samples),
2590 " (%" PRIu64 " samples)", rec->samples);
2591 else
2592 samples[0] = '\0';
2593
d3c8c08e 2594 fprintf(stderr, "[ perf record: Captured and wrote %.3f MB %s%s%s",
8ceb41d7 2595 perf_data__size(data) / 1024.0 / 1024.0,
2d4f2799 2596 data->path, postfix, samples);
d3c8c08e
AB		 2597 if (ratio) {
2598 fprintf(stderr, ", compressed (original %.3f MB, ratio is %.3f)",
2599 rec->session->bytes_transferred / 1024.0 / 1024.0,
2600 ratio);
2601 }
2602 fprintf(stderr, " ]\n");
e3d59112
NK		 2603 }
2604
39d17dac 2605out_delete_session:
da231338
AM		 2606#ifdef HAVE_EVENTFD_SUPPORT
2607 if (done_fd >= 0)
2608 close(done_fd);
2609#endif
5d7f4116 2610 zstd_fini(&session->zstd_data);
39d17dac 2611 perf_session__delete(session);
657ee553
SL		 2612
2613 if (!opts->no_bpf_event)
08c83997 2614 evlist__stop_sb_thread(rec->sb_evlist);
45604710 2615 return status;
de9ac07b 2616}
0e9b20b8 2617
0883e820 2618static void callchain_debug(struct callchain_param *callchain)
09b0fd45 2619{
aad2b21c 2620 static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF", "LBR" };
a601fdff 2621
0883e820 2622 pr_debug("callchain: type %s\n", str[callchain->record_mode]);
26d33022 2623
0883e820 2624 if (callchain->record_mode == CALLCHAIN_DWARF)
09b0fd45 2625 pr_debug("callchain: stack dump size %d\n",
0883e820 2626 callchain->dump_size);
09b0fd45
JO		 2627}
2628
0883e820
ACM		 2629int record_opts__parse_callchain(struct record_opts *record,
2630 struct callchain_param *callchain,
2631 const char *arg, bool unset)
09b0fd45 2632{
09b0fd45 2633 int ret;
0883e820 2634 callchain->enabled = !unset;
eb853e80 2635
09b0fd45
JO		 2636 /* --no-call-graph */
2637 if (unset) {
0883e820 2638 callchain->record_mode = CALLCHAIN_NONE;
09b0fd45
JO		 2639 pr_debug("callchain: disabled\n");
2640 return 0;
2641 }
2642
0883e820 2643 ret = parse_callchain_record_opt(arg, callchain);
5c0cf224
JO		 2644 if (!ret) {
2645 /* Enable data address sampling for DWARF unwind. */
0883e820 2646 if (callchain->record_mode == CALLCHAIN_DWARF)
5c0cf224 2647 record->sample_address = true;
0883e820 2648 callchain_debug(callchain);
5c0cf224 2649 }
26d33022
JO		 2650
2651 return ret;
2652}
2653
0883e820
ACM		 2654int record_parse_callchain_opt(const struct option *opt,
2655 const char *arg,
2656 int unset)
2657{
2658 return record_opts__parse_callchain(opt->value, &callchain_param, arg, unset);
2659}
2660
c421e80b 2661int record_callchain_opt(const struct option *opt,
09b0fd45
JO		 2662 const char *arg __maybe_unused,
2663 int unset __maybe_unused)
2664{
2ddd5c04 2665 struct callchain_param *callchain = opt->value;
c421e80b 2666
2ddd5c04 2667 callchain->enabled = true;
09b0fd45 2668
2ddd5c04
ACM		 2669 if (callchain->record_mode == CALLCHAIN_NONE)
2670 callchain->record_mode = CALLCHAIN_FP;
eb853e80 2671
2ddd5c04 2672 callchain_debug(callchain);
09b0fd45
JO		 2673 return 0;
2674}
2675
eb853e80
JO		 2676static int perf_record_config(const char *var, const char *value, void *cb)
2677{
7a29c087
NK		 2678 struct record *rec = cb;
2679
2680 if (!strcmp(var, "record.build-id")) {
2681 if (!strcmp(value, "cache"))
2682 rec->no_buildid_cache = false;
2683 else if (!strcmp(value, "no-cache"))
2684 rec->no_buildid_cache = true;
2685 else if (!strcmp(value, "skip"))
2686 rec->no_buildid = true;
e29386c8
JO		 2687 else if (!strcmp(value, "mmap"))
2688 rec->buildid_mmap = true;
7a29c087
NK		 2689 else
2690 return -1;
2691 return 0;
2692 }
cff17205
YX		 2693 if (!strcmp(var, "record.call-graph")) {
2694 var = "call-graph.record-mode";
2695 return perf_default_config(var, value, cb);
2696 }
93f20c0f
AB		 2697#ifdef HAVE_AIO_SUPPORT
2698 if (!strcmp(var, "record.aio")) {
2699 rec->opts.nr_cblocks = strtol(value, NULL, 0);
2700 if (!rec->opts.nr_cblocks)
2701 rec->opts.nr_cblocks = nr_cblocks_default;
2702 }
2703#endif
9bce13ea
JO		 2704 if (!strcmp(var, "record.debuginfod")) {
2705 rec->debuginfod.urls = strdup(value);
2706 if (!rec->debuginfod.urls)
2707 return -ENOMEM;
2708 rec->debuginfod.set = true;
2709 }
eb853e80 2710
cff17205 2711 return 0;
eb853e80
JO		 2712}
2713
814c8c38 2714
f4fe11b7
AB		 2715static int record__parse_affinity(const struct option *opt, const char *str, int unset)
2716{
2717 struct record_opts *opts = (struct record_opts *)opt->value;
2718
2719 if (unset || !str)
2720 return 0;
2721
2722 if (!strcasecmp(str, "node"))
2723 opts->affinity = PERF_AFFINITY_NODE;
2724 else if (!strcasecmp(str, "cpu"))
2725 opts->affinity = PERF_AFFINITY_CPU;
2726
2727 return 0;
2728}
2729
7954f716
AB		 2730static int record__mmap_cpu_mask_alloc(struct mmap_cpu_mask *mask, int nr_bits)
2731{
2732 mask->nbits = nr_bits;
2733 mask->bits = bitmap_zalloc(mask->nbits);
2734 if (!mask->bits)
2735 return -ENOMEM;
2736
2737 return 0;
2738}
2739
2740static void record__mmap_cpu_mask_free(struct mmap_cpu_mask *mask)
2741{
2742 bitmap_free(mask->bits);
2743 mask->nbits = 0;
2744}
2745
2746static int record__thread_mask_alloc(struct thread_mask *mask, int nr_bits)
2747{
2748 int ret;
2749
2750 ret = record__mmap_cpu_mask_alloc(&mask->maps, nr_bits);
2751 if (ret) {
2752 mask->affinity.bits = NULL;
2753 return ret;
2754 }
2755
2756 ret = record__mmap_cpu_mask_alloc(&mask->affinity, nr_bits);
2757 if (ret) {
2758 record__mmap_cpu_mask_free(&mask->maps);
2759 mask->maps.bits = NULL;
2760 }
2761
2762 return ret;
2763}
2764
2765static void record__thread_mask_free(struct thread_mask *mask)
2766{
2767 record__mmap_cpu_mask_free(&mask->maps);
2768 record__mmap_cpu_mask_free(&mask->affinity);
2769}
2770
6d575816
JS		 2771static int parse_output_max_size(const struct option *opt,
2772 const char *str, int unset)
2773{
2774 unsigned long *s = (unsigned long *)opt->value;
2775 static struct parse_tag tags_size[] = {
2776 { .tag = 'B', .mult = 1 },
2777 { .tag = 'K', .mult = 1 << 10 },
2778 { .tag = 'M', .mult = 1 << 20 },
2779 { .tag = 'G', .mult = 1 << 30 },
2780 { .tag = 0 },
2781 };
2782 unsigned long val;
2783
2784 if (unset) {
2785 *s = 0;
2786 return 0;
2787 }
2788
2789 val = parse_tag_value(str, tags_size);
2790 if (val != (unsigned long) -1) {
2791 *s = val;
2792 return 0;
2793 }
2794
2795 return -1;
2796}
2797
e9db1310
AH		 2798static int record__parse_mmap_pages(const struct option *opt,
2799 const char *str,
2800 int unset __maybe_unused)
2801{
2802 struct record_opts *opts = opt->value;
2803 char *s, *p;
2804 unsigned int mmap_pages;
2805 int ret;
2806
2807 if (!str)
2808 return -EINVAL;
2809
2810 s = strdup(str);
2811 if (!s)
2812 return -ENOMEM;
2813
2814 p = strchr(s, ',');
2815 if (p)
2816 *p = '\0';
2817
2818 if (*s) {
25f84702 2819 ret = __evlist__parse_mmap_pages(&mmap_pages, s);
e9db1310
AH		 2820 if (ret)
2821 goto out_free;
2822 opts->mmap_pages = mmap_pages;
2823 }
2824
2825 if (!p) {
2826 ret = 0;
2827 goto out_free;
2828 }
2829
25f84702 2830 ret = __evlist__parse_mmap_pages(&mmap_pages, p + 1);
e9db1310
AH		 2831 if (ret)
2832 goto out_free;
2833
2834 opts->auxtrace_mmap_pages = mmap_pages;
2835
2836out_free:
2837 free(s);
2838 return ret;
2839}
2840
7248e308
AT		 2841void __weak arch__add_leaf_frame_record_opts(struct record_opts *opts __maybe_unused)
2842{
2843}
2844
1d078ccb
AB		 2845static int parse_control_option(const struct option *opt,
2846 const char *str,
2847 int unset __maybe_unused)
2848{
9864a66d 2849 struct record_opts *opts = opt->value;
1d078ccb 2850
a8fcbd26
AH		 2851 return evlist__parse_control(str, &opts->ctl_fd, &opts->ctl_fd_ack, &opts->ctl_fd_close);
2852}
2853
0c582449
JO		 2854static void switch_output_size_warn(struct record *rec)
2855{
9521b5f2 2856 u64 wakeup_size = evlist__mmap_size(rec->opts.mmap_pages);
0c582449
JO		 2857 struct switch_output *s = &rec->switch_output;
2858
2859 wakeup_size /= 2;
2860
2861 if (s->size < wakeup_size) {
2862 char buf[100];
2863
2864 unit_number__scnprintf(buf, sizeof(buf), wakeup_size);
2865 pr_warning("WARNING: switch-output data size lower than "
2866 "wakeup kernel buffer size (%s) "
2867 "expect bigger perf.data sizes\n", buf);
2868 }
2869}
2870
cb4e1ebb
JO		 2871static int switch_output_setup(struct record *rec)
2872{
2873 struct switch_output *s = &rec->switch_output;
dc0c6127
JO		 2874 static struct parse_tag tags_size[] = {
2875 { .tag = 'B', .mult = 1 },
2876 { .tag = 'K', .mult = 1 << 10 },
2877 { .tag = 'M', .mult = 1 << 20 },
2878 { .tag = 'G', .mult = 1 << 30 },
2879 { .tag = 0 },
2880 };
bfacbe3b
JO		 2881 static struct parse_tag tags_time[] = {
2882 { .tag = 's', .mult = 1 },
2883 { .tag = 'm', .mult = 60 },
2884 { .tag = 'h', .mult = 60*60 },
2885 { .tag = 'd', .mult = 60*60*24 },
2886 { .tag = 0 },
2887 };
dc0c6127 2888 unsigned long val;
cb4e1ebb 2889
899e5ffb
ACM		 2890 /*
2891 * If we're using --switch-output-events, then we imply its
2892 * --switch-output=signal, as we'll send a SIGUSR2 from the side band
2893 * thread to its parent.
2894 */
2895 if (rec->switch_output_event_set)
2896 goto do_signal;
2897
cb4e1ebb
JO		 2898 if (!s->set)
2899 return 0;
2900
2901 if (!strcmp(s->str, "signal")) {
899e5ffb 2902do_signal:
cb4e1ebb
JO		 2903 s->signal = true;
2904 pr_debug("switch-output with SIGUSR2 signal\n");
dc0c6127
JO		 2905 goto enabled;
2906 }
2907
2908 val = parse_tag_value(s->str, tags_size);
2909 if (val != (unsigned long) -1) {
2910 s->size = val;
2911 pr_debug("switch-output with %s size threshold\n", s->str);
2912 goto enabled;
cb4e1ebb
JO		 2913 }
2914
bfacbe3b
JO		 2915 val = parse_tag_value(s->str, tags_time);
2916 if (val != (unsigned long) -1) {
2917 s->time = val;
2918 pr_debug("switch-output with %s time threshold (%lu seconds)\n",
2919 s->str, s->time);
2920 goto enabled;
2921 }
2922
cb4e1ebb 2923 return -1;
dc0c6127
JO		 2924
2925enabled:
2926 rec->timestamp_filename = true;
2927 s->enabled = true;
0c582449
JO		 2928
2929 if (s->size && !rec->opts.no_buffering)
2930 switch_output_size_warn(rec);
2931
dc0c6127 2932 return 0;
cb4e1ebb
JO		 2933}
2934
e5b2c207 2935static const char * const __record_usage[] = {
9e096753
MG		 2936 "perf record [<options>] [<command>]",
2937 "perf record [<options>] -- <command> [<options>]",
0e9b20b8
IM		 2938 NULL
2939};
e5b2c207 2940const char * const *record_usage = __record_usage;
0e9b20b8 2941
6e0a9b3d
ACM		 2942static int build_id__process_mmap(struct perf_tool *tool, union perf_event *event,
2943 struct perf_sample *sample, struct machine *machine)
2944{
2945 /*
2946 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
2947 * no need to add them twice.
2948 */
2949 if (!(event->header.misc & PERF_RECORD_MISC_USER))
2950 return 0;
2951 return perf_event__process_mmap(tool, event, sample, machine);
2952}
2953
2954static int build_id__process_mmap2(struct perf_tool *tool, union perf_event *event,
2955 struct perf_sample *sample, struct machine *machine)
2956{
2957 /*
2958 * We already have the kernel maps, put in place via perf_session__create_kernel_maps()
2959 * no need to add them twice.
2960 */
2961 if (!(event->header.misc & PERF_RECORD_MISC_USER))
2962 return 0;
2963
2964 return perf_event__process_mmap2(tool, event, sample, machine);
2965}
2966
66286ed3
AH		 2967static int process_timestamp_boundary(struct perf_tool *tool,
2968 union perf_event *event __maybe_unused,
2969 struct perf_sample *sample,
2970 struct machine *machine __maybe_unused)
2971{
2972 struct record *rec = container_of(tool, struct record, tool);
2973
2974 set_timestamp_boundary(rec, sample->time);
2975 return 0;
2976}
2977
41b740b6
NK		 2978static int parse_record_synth_option(const struct option *opt,
2979 const char *str,
2980 int unset __maybe_unused)
2981{
2982 struct record_opts *opts = opt->value;
2983 char *p = strdup(str);
2984
2985 if (p == NULL)
2986 return -1;
2987
2988 opts->synth = parse_synth_opt(p);
2989 free(p);
2990
2991 if (opts->synth < 0) {
2992 pr_err("Invalid synth option: %s\n", str);
2993 return -1;
2994 }
2995 return 0;
2996}
2997
d20deb64 2998/*
8c6f45a7
ACM		 2999 * XXX Ideally would be local to cmd_record() and passed to a record__new
3000 * because we need to have access to it in record__exit, that is called
d20deb64
ACM		 3001 * after cmd_record() exits, but since record_options need to be accessible to
3002 * builtin-script, leave it here.
3003 *
3004 * At least we don't ouch it in all the other functions here directly.
3005 *
3006 * Just say no to tons of global variables, sigh.
3007 */
8c6f45a7 3008static struct record record = {
d20deb64 3009 .opts = {
8affc2b8 3010 .sample_time = true,
d20deb64
ACM		 3011 .mmap_pages = UINT_MAX,
3012 .user_freq = UINT_MAX,
3013 .user_interval = ULLONG_MAX,
447a6013 3014 .freq = 4000,
d1cb9fce
NK		 3015 .target = {
3016 .uses_mmap = true,
3aa5939d 3017 .default_per_cpu = true,
d1cb9fce 3018 },
470530bb 3019 .mmap_flush = MMAP_FLUSH_DEFAULT,
d99c22ea 3020 .nr_threads_synthesize = 1,
1d078ccb
AB		 3021 .ctl_fd = -1,
3022 .ctl_fd_ack = -1,
41b740b6 3023 .synth = PERF_SYNTH_ALL,
d20deb64 3024 },
e3d59112
NK		 3025 .tool = {
3026 .sample = process_sample_event,
3027 .fork = perf_event__process_fork,
cca8482c 3028 .exit = perf_event__process_exit,
e3d59112 3029 .comm = perf_event__process_comm,
f3b3614a 3030 .namespaces = perf_event__process_namespaces,
6e0a9b3d
ACM		 3031 .mmap = build_id__process_mmap,
3032 .mmap2 = build_id__process_mmap2,
66286ed3
AH		 3033 .itrace_start = process_timestamp_boundary,
3034 .aux = process_timestamp_boundary,
cca8482c 3035 .ordered_events = true,
e3d59112 3036 },
d20deb64 3037};
7865e817 3038
76a26549
NK		 3039const char record_callchain_help[] = CALLCHAIN_RECORD_HELP
3040 "\n\t\t\t\tDefault: fp";
61eaa3be 3041
0aab2136
WN		 3042static bool dry_run;
3043
d20deb64
ACM		 3044/*
3045 * XXX Will stay a global variable till we fix builtin-script.c to stop messing
3046 * with it and switch to use the library functions in perf_evlist that came
b4006796 3047 * from builtin-record.c, i.e. use record_opts,
7b392ef0 3048 * evlist__prepare_workload, etc instead of fork+exec'in 'perf record',
d20deb64
ACM		 3049 * using pipes, etc.
3050 */
efd21307 3051static struct option __record_options[] = {
d20deb64 3052 OPT_CALLBACK('e', "event", &record.evlist, "event",
86847b62 3053 "event selector. use 'perf list' to list available events",
f120f9d5 3054 parse_events_option),
d20deb64 3055 OPT_CALLBACK(0, "filter", &record.evlist, "filter",
c171b552 3056 "event filter", parse_filter),
4ba1faa1
WN		 3057 OPT_CALLBACK_NOOPT(0, "exclude-perf", &record.evlist,
3058 NULL, "don't record events from perf itself",
3059 exclude_perf),
bea03405 3060 OPT_STRING('p', "pid", &record.opts.target.pid, "pid",
d6d901c2 3061 "record events on existing process id"),
bea03405 3062 OPT_STRING('t', "tid", &record.opts.target.tid, "tid",
d6d901c2 3063 "record events on existing thread id"),
d20deb64 3064 OPT_INTEGER('r', "realtime", &record.realtime_prio,
0e9b20b8 3065 "collect data with this RT SCHED_FIFO priority"),
509051ea 3066 OPT_BOOLEAN(0, "no-buffering", &record.opts.no_buffering,
acac03fa 3067 "collect data without buffering"),
d20deb64 3068 OPT_BOOLEAN('R', "raw-samples", &record.opts.raw_samples,
daac07b2 3069 "collect raw sample records from all opened counters"),
bea03405 3070 OPT_BOOLEAN('a', "all-cpus", &record.opts.target.system_wide,
0e9b20b8 3071 "system-wide collection from all CPUs"),
bea03405 3072 OPT_STRING('C', "cpu", &record.opts.target.cpu_list, "cpu",
c45c6ea2 3073 "list of cpus to monitor"),
d20deb64 3074 OPT_U64('c', "count", &record.opts.user_interval, "event period to sample"),
2d4f2799 3075 OPT_STRING('o', "output", &record.data.path, "file",
abaff32a 3076 "output file name"),
69e7e5b0
AH		 3077 OPT_BOOLEAN_SET('i', "no-inherit", &record.opts.no_inherit,
3078 &record.opts.no_inherit_set,
3079 "child tasks do not inherit counters"),
4ea648ae
WN		 3080 OPT_BOOLEAN(0, "tail-synthesize", &record.opts.tail_synthesize,
3081 "synthesize non-sample events at the end of output"),
626a6b78 3082 OPT_BOOLEAN(0, "overwrite", &record.opts.overwrite, "use overwrite mode"),
a060c1f1 3083 OPT_BOOLEAN(0, "no-bpf-event", &record.opts.no_bpf_event, "do not record bpf events"),
b09c2364
ACM		 3084 OPT_BOOLEAN(0, "strict-freq", &record.opts.strict_freq,
3085 "Fail if the specified frequency can't be used"),
67230479
ACM		 3086 OPT_CALLBACK('F', "freq", &record.opts, "freq or 'max'",
3087 "profile at this frequency",
3088 record__parse_freq),
e9db1310
AH		 3089 OPT_CALLBACK('m', "mmap-pages", &record.opts, "pages[,pages]",
3090 "number of mmap data pages and AUX area tracing mmap pages",
3091 record__parse_mmap_pages),
470530bb
AB		 3092 OPT_CALLBACK(0, "mmap-flush", &record.opts, "number",
3093 "Minimal number of bytes that is extracted from mmap data pages (default: 1)",
3094 record__mmap_flush_parse),
d20deb64 3095 OPT_BOOLEAN(0, "group", &record.opts.group,
43bece79 3096 "put the counters into a counter group"),
2ddd5c04 3097 OPT_CALLBACK_NOOPT('g', NULL, &callchain_param,
09b0fd45
JO		 3098 NULL, "enables call-graph recording" ,
3099 &record_callchain_opt),
3100 OPT_CALLBACK(0, "call-graph", &record.opts,
76a26549 3101 "record_mode[,record_size]", record_callchain_help,
09b0fd45 3102 &record_parse_callchain_opt),
c0555642 3103 OPT_INCR('v', "verbose", &verbose,
3da297a6 3104 "be more verbose (show counter open errors, etc)"),
b44308f5 3105 OPT_BOOLEAN('q', "quiet", &quiet, "don't print any message"),
d20deb64 3106 OPT_BOOLEAN('s', "stat", &record.opts.inherit_stat,
649c48a9 3107 "per thread counts"),
56100321 3108 OPT_BOOLEAN('d', "data", &record.opts.sample_address, "Record the sample addresses"),
3b0a5daa
KL		 3109 OPT_BOOLEAN(0, "phys-data", &record.opts.sample_phys_addr,
3110 "Record the sample physical addresses"),
542b88fd
KL		 3111 OPT_BOOLEAN(0, "data-page-size", &record.opts.sample_data_page_size,
3112 "Record the sampled data address data page size"),
c1de7f3d
KL		 3113 OPT_BOOLEAN(0, "code-page-size", &record.opts.sample_code_page_size,
3114 "Record the sampled code address (ip) page size"),
b6f35ed7 3115 OPT_BOOLEAN(0, "sample-cpu", &record.opts.sample_cpu, "Record the sample cpu"),
3abebc55
AH		 3116 OPT_BOOLEAN_SET('T', "timestamp", &record.opts.sample_time,
3117 &record.opts.sample_time_set,
3118 "Record the sample timestamps"),
f290aa1f
JO		 3119 OPT_BOOLEAN_SET('P', "period", &record.opts.period, &record.opts.period_set,
3120 "Record the sample period"),
d20deb64 3121 OPT_BOOLEAN('n', "no-samples", &record.opts.no_samples,
649c48a9 3122 "don't sample"),
d2db9a98
WN		 3123 OPT_BOOLEAN_SET('N', "no-buildid-cache", &record.no_buildid_cache,
3124 &record.no_buildid_cache_set,
3125 "do not update the buildid cache"),
3126 OPT_BOOLEAN_SET('B', "no-buildid", &record.no_buildid,
3127 &record.no_buildid_set,
3128 "do not collect buildids in perf.data"),
d20deb64 3129 OPT_CALLBACK('G', "cgroup", &record.evlist, "name",
023695d9
SE		 3130 "monitor event in cgroup name only",
3131 parse_cgroups),
68cd3b45
AB		 3132 OPT_INTEGER('D', "delay", &record.opts.initial_delay,
3133 "ms to wait before starting measurement after program start (-1: start with events disabled)"),
eeb399b5 3134 OPT_BOOLEAN(0, "kcore", &record.opts.kcore, "copy /proc/kcore"),
bea03405
NK		 3135 OPT_STRING('u', "uid", &record.opts.target.uid_str, "user",
3136 "user to profile"),
a5aabdac
SE		 3137
3138 OPT_CALLBACK_NOOPT('b', "branch-any", &record.opts.branch_stack,
3139 "branch any", "sample any taken branches",
3140 parse_branch_stack),
3141
3142 OPT_CALLBACK('j', "branch-filter", &record.opts.branch_stack,
3143 "branch filter mask", "branch stack filter modes",
bdfebd84 3144 parse_branch_stack),
05484298
AK		 3145 OPT_BOOLEAN('W', "weight", &record.opts.sample_weight,
3146 "sample by weight (on special events only)"),
475eeab9
AK		 3147 OPT_BOOLEAN(0, "transaction", &record.opts.sample_transaction,
3148 "sample transaction flags (special events only)"),
3aa5939d
AH		 3149 OPT_BOOLEAN(0, "per-thread", &record.opts.target.per_thread,
3150 "use per-thread mmaps"),
bcc84ec6
SE		 3151 OPT_CALLBACK_OPTARG('I', "intr-regs", &record.opts.sample_intr_regs, NULL, "any register",
3152 "sample selected machine registers on interrupt,"
aeea9062 3153 " use '-I?' to list register names", parse_intr_regs),
84c41742
AK		 3154 OPT_CALLBACK_OPTARG(0, "user-regs", &record.opts.sample_user_regs, NULL, "any register",
3155 "sample selected machine registers on interrupt,"
aeea9062 3156 " use '--user-regs=?' to list register names", parse_user_regs),
85c273d2
AK		 3157 OPT_BOOLEAN(0, "running-time", &record.opts.running_time,
3158 "Record running/enabled time of read (:S) events"),
814c8c38
PZ		 3159 OPT_CALLBACK('k', "clockid", &record.opts,
3160 "clockid", "clockid to use for events, see clock_gettime()",
3161 parse_clockid),
2dd6d8a1
AH		 3162 OPT_STRING_OPTARG('S', "snapshot", &record.opts.auxtrace_snapshot_opts,
3163 "opts", "AUX area tracing Snapshot Mode", ""),
c0a6de06
AH		 3164 OPT_STRING_OPTARG(0, "aux-sample", &record.opts.auxtrace_sample_opts,
3165 "opts", "sample AUX area", ""),
3fcb10e4 3166 OPT_UINTEGER(0, "proc-map-timeout", &proc_map_timeout,
9d9cad76 3167 "per thread proc mmap processing timeout in ms"),
f3b3614a
HB		 3168 OPT_BOOLEAN(0, "namespaces", &record.opts.record_namespaces,
3169 "Record namespaces events"),
8fb4b679
NK		 3170 OPT_BOOLEAN(0, "all-cgroups", &record.opts.record_cgroup,
3171 "Record cgroup events"),
16b4b4e1
AH		 3172 OPT_BOOLEAN_SET(0, "switch-events", &record.opts.record_switch_events,
3173 &record.opts.record_switch_events_set,
3174 "Record context switch events"),
85723885
JO		 3175 OPT_BOOLEAN_FLAG(0, "all-kernel", &record.opts.all_kernel,
3176 "Configure all used events to run in kernel space.",
3177 PARSE_OPT_EXCLUSIVE),
3178 OPT_BOOLEAN_FLAG(0, "all-user", &record.opts.all_user,
3179 "Configure all used events to run in user space.",
3180 PARSE_OPT_EXCLUSIVE),
53651b28 3181 OPT_BOOLEAN(0, "kernel-callchains", &record.opts.kernel_callchains,
3182 "collect kernel callchains"),
3183 OPT_BOOLEAN(0, "user-callchains", &record.opts.user_callchains,
3184 "collect user callchains"),
71dc2326
WN		 3185 OPT_STRING(0, "clang-path", &llvm_param.clang_path, "clang path",
3186 "clang binary to use for compiling BPF scriptlets"),
3187 OPT_STRING(0, "clang-opt", &llvm_param.clang_opt, "clang options",
3188 "options passed to clang when compiling BPF scriptlets"),
7efe0e03
HK		 3189 OPT_STRING(0, "vmlinux", &symbol_conf.vmlinux_name,
3190 "file", "vmlinux pathname"),
6156681b
NK		 3191 OPT_BOOLEAN(0, "buildid-all", &record.buildid_all,
3192 "Record build-id of all DSOs regardless of hits"),
e29386c8
JO		 3193 OPT_BOOLEAN(0, "buildid-mmap", &record.buildid_mmap,
3194 "Record build-id in map events"),
ecfd7a9c
WN		 3195 OPT_BOOLEAN(0, "timestamp-filename", &record.timestamp_filename,
3196 "append timestamp to output filename"),
68588baf
JY		 3197 OPT_BOOLEAN(0, "timestamp-boundary", &record.timestamp_boundary,
3198 "Record timestamp boundary (time of first/last samples)"),
cb4e1ebb 3199 OPT_STRING_OPTARG_SET(0, "switch-output", &record.switch_output.str,
c38dab7d
AK		 3200 &record.switch_output.set, "signal or size[BKMG] or time[smhd]",
3201 "Switch output when receiving SIGUSR2 (signal) or cross a size or time threshold",
dc0c6127 3202 "signal"),
899e5ffb
ACM		 3203 OPT_CALLBACK_SET(0, "switch-output-event", &record.sb_evlist, &record.switch_output_event_set, "switch output event",
3204 "switch output event selector. use 'perf list' to list available events",
3205 parse_events_option_new_evlist),
03724b2e
AK		 3206 OPT_INTEGER(0, "switch-max-files", &record.switch_output.num_files,
3207 "Limit number of switch output generated files"),
0aab2136
WN		 3208 OPT_BOOLEAN(0, "dry-run", &dry_run,
3209 "Parse options then exit"),
d3d1af6f 3210#ifdef HAVE_AIO_SUPPORT
93f20c0f
AB		 3211 OPT_CALLBACK_OPTARG(0, "aio", &record.opts,
3212 &nr_cblocks_default, "n", "Use <n> control blocks in asynchronous trace writing mode (default: 1, max: 4)",
d3d1af6f
AB		 3213 record__aio_parse),
3214#endif
f4fe11b7
AB		 3215 OPT_CALLBACK(0, "affinity", &record.opts, "node|cpu",
3216 "Set affinity mask of trace reading thread to NUMA node cpu mask or cpu of processed mmap buffer",
3217 record__parse_affinity),
504c1ad1
AB		 3218#ifdef HAVE_ZSTD_SUPPORT
3219 OPT_CALLBACK_OPTARG('z', "compression-level", &record.opts, &comp_level_default,
3220 "n", "Compressed records using specified level (default: 1 - fastest compression, 22 - greatest compression)",
3221 record__parse_comp_level),
3222#endif
6d575816
JS		 3223 OPT_CALLBACK(0, "max-size", &record.output_max_size,
3224 "size", "Limit the maximum size of the output file", parse_output_max_size),
d99c22ea
SE		 3225 OPT_UINTEGER(0, "num-thread-synthesize",
3226 &record.opts.nr_threads_synthesize,
3227 "number of threads to run for event synthesis"),
70943490
SE		 3228#ifdef HAVE_LIBPFM
3229 OPT_CALLBACK(0, "pfm-events", &record.evlist, "event",
3230 "libpfm4 event selector. use 'perf list' to list available events",
3231 parse_libpfm_events_option),
3232#endif
a8fcbd26 3233 OPT_CALLBACK(0, "control", &record.opts, "fd:ctl-fd[,ack-fd] or fifo:ctl-fifo[,ack-fifo]",
d20aff15
AH		 3234 "Listen on ctl-fd descriptor for command to control measurement ('enable': enable events, 'disable': disable events,\n"
3235 "\t\t\t 'snapshot': AUX area tracing snapshot).\n"
a8fcbd26
AH		 3236 "\t\t\t Optionally send control command completion ('ack\\n') to ack-fd descriptor.\n"
3237 "\t\t\t Alternatively, ctl-fifo / ack-fifo will be opened and used as ctl-fd / ack-fd.",
1d078ccb 3238 parse_control_option),
41b740b6
NK		 3239 OPT_CALLBACK(0, "synth", &record.opts, "no|all|task|mmap|cgroup",
3240 "Fine-tune event synthesis: default=all", parse_record_synth_option),
9bce13ea
JO		 3241 OPT_STRING_OPTARG_SET(0, "debuginfod", &record.debuginfod.urls,
3242 &record.debuginfod.set, "debuginfod urls",
3243 "Enable debuginfod data retrieval from DEBUGINFOD_URLS or specified urls",
3244 "system"),
0e9b20b8
IM		 3245 OPT_END()
3246};
3247
e5b2c207
NK		 3248struct option *record_options = __record_options;
3249
7954f716
AB		 3250static void record__mmap_cpu_mask_init(struct mmap_cpu_mask *mask, struct perf_cpu_map *cpus)
3251{
3252 int c;
3253
3254 for (c = 0; c < cpus->nr; c++)
3255 set_bit(cpus->map[c].cpu, mask->bits);
3256}
3257
3258static void record__free_thread_masks(struct record *rec, int nr_threads)
3259{
3260 int t;
3261
3262 if (rec->thread_masks)
3263 for (t = 0; t < nr_threads; t++)
3264 record__thread_mask_free(&rec->thread_masks[t]);
3265
3266 zfree(&rec->thread_masks);
3267}
3268
3269static int record__alloc_thread_masks(struct record *rec, int nr_threads, int nr_bits)
3270{
3271 int t, ret;
3272
3273 rec->thread_masks = zalloc(nr_threads * sizeof(*(rec->thread_masks)));
3274 if (!rec->thread_masks) {
3275 pr_err("Failed to allocate thread masks\n");
3276 return -ENOMEM;
3277 }
3278
3279 for (t = 0; t < nr_threads; t++) {
3280 ret = record__thread_mask_alloc(&rec->thread_masks[t], nr_bits);
3281 if (ret) {
3282 pr_err("Failed to allocate thread masks[%d]\n", t);
3283 goto out_free;
3284 }
3285 }
3286
3287 return 0;
3288
3289out_free:
3290 record__free_thread_masks(rec, nr_threads);
3291
3292 return ret;
3293}
3294
3295static int record__init_thread_default_masks(struct record *rec, struct perf_cpu_map *cpus)
3296{
3297 int ret;
3298
3299 ret = record__alloc_thread_masks(rec, 1, cpu__max_cpu().cpu);
3300 if (ret)
3301 return ret;
3302
3303 record__mmap_cpu_mask_init(&rec->thread_masks->maps, cpus);
3304
3305 rec->nr_threads = 1;
3306
3307 return 0;
3308}
3309
3310static int record__init_thread_masks(struct record *rec)
3311{
3312 struct perf_cpu_map *cpus = rec->evlist->core.cpus;
3313
3314 return record__init_thread_default_masks(rec, cpus);
3315}
3316
b0ad8ea6 3317int cmd_record(int argc, const char **argv)
0e9b20b8 3318{
ef149c25 3319 int err;
8c6f45a7 3320 struct record *rec = &record;
16ad2ffb 3321 char errbuf[BUFSIZ];
0e9b20b8 3322
67230479
ACM		 3323 setlocale(LC_ALL, "");
3324
48e1cab1
WN		 3325#ifndef HAVE_LIBBPF_SUPPORT
3326# define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, "NO_LIBBPF=1", c)
3327 set_nobuild('\0', "clang-path", true);
3328 set_nobuild('\0', "clang-opt", true);
3329# undef set_nobuild
7efe0e03
HK		 3330#endif
3331
3332#ifndef HAVE_BPF_PROLOGUE
3333# if !defined (HAVE_DWARF_SUPPORT)
3334# define REASON "NO_DWARF=1"
3335# elif !defined (HAVE_LIBBPF_SUPPORT)
3336# define REASON "NO_LIBBPF=1"
3337# else
3338# define REASON "this architecture doesn't support BPF prologue"
3339# endif
3340# define set_nobuild(s, l, c) set_option_nobuild(record_options, s, l, REASON, c)
3341 set_nobuild('\0', "vmlinux", true);
3342# undef set_nobuild
3343# undef REASON
48e1cab1
WN		 3344#endif
3345
9d2ed645
AB		 3346 rec->opts.affinity = PERF_AFFINITY_SYS;
3347
0f98b11c 3348 rec->evlist = evlist__new();
3e2be2da 3349 if (rec->evlist == NULL)
361c99a6
ACM		 3350 return -ENOMEM;
3351
ecc4c561
ACM		 3352 err = perf_config(perf_record_config, rec);
3353 if (err)
3354 return err;
eb853e80 3355
bca647aa 3356 argc = parse_options(argc, argv, record_options, record_usage,
655000e7 3357 PARSE_OPT_STOP_AT_NON_OPTION);
68ba3235
NK		 3358 if (quiet)
3359 perf_quiet_option();
483635a9 3360
7cc72553
JC		 3361 err = symbol__validate_sym_arguments();
3362 if (err)
3363 return err;
3364
9bce13ea
JO		 3365 perf_debuginfod_setup(&record.debuginfod);
3366
483635a9 3367 /* Make system wide (-a) the default target. */
602ad878 3368 if (!argc && target__none(&rec->opts.target))
483635a9 3369 rec->opts.target.system_wide = true;
0e9b20b8 3370
bea03405 3371 if (nr_cgroups && !rec->opts.target.system_wide) {
c7118369
NK		 3372 usage_with_options_msg(record_usage, record_options,
3373 "cgroup monitoring only available in system-wide mode");
3374
023695d9 3375 }
504c1ad1 3376
e29386c8
JO		 3377 if (rec->buildid_mmap) {
3378 if (!perf_can_record_build_id()) {
3379 pr_err("Failed: no support to record build id in mmap events, update your kernel.\n");
3380 err = -EINVAL;
3381 goto out_opts;
3382 }
3383 pr_debug("Enabling build id in mmap2 events.\n");
3384 /* Enable mmap build id synthesizing. */
3385 symbol_conf.buildid_mmap2 = true;
3386 /* Enable perf_event_attr::build_id bit. */
3387 rec->opts.build_id = true;
3388 /* Disable build id cache. */
3389 rec->no_buildid = true;
3390 }
3391
4f2abe91
NK		 3392 if (rec->opts.record_cgroup && !perf_can_record_cgroup()) {
3393 pr_err("Kernel has no cgroup sampling support.\n");
3394 err = -EINVAL;
3395 goto out_opts;
3396 }
3397
56f735ff 3398 if (rec->opts.kcore || record__threads_enabled(rec))
eeb399b5
AH		 3399 rec->data.is_dir = true;
3400
504c1ad1
AB		 3401 if (rec->opts.comp_level != 0) {
3402 pr_debug("Compression enabled, disabling build id collection at the end of the session.\n");
3403 rec->no_buildid = true;
3404 }
3405
b757bb09
AH		 3406 if (rec->opts.record_switch_events &&
3407 !perf_can_record_switch_events()) {
c7118369
NK		 3408 ui__error("kernel does not support recording context switch events\n");
3409 parse_options_usage(record_usage, record_options, "switch-events", 0);
a8fcbd26
AH		 3410 err = -EINVAL;
3411 goto out_opts;
b757bb09 3412 }
023695d9 3413
cb4e1ebb
JO		 3414 if (switch_output_setup(rec)) {
3415 parse_options_usage(record_usage, record_options, "switch-output", 0);
a8fcbd26
AH		 3416 err = -EINVAL;
3417 goto out_opts;
cb4e1ebb
JO		 3418 }
3419
bfacbe3b
JO		 3420 if (rec->switch_output.time) {
3421 signal(SIGALRM, alarm_sig_handler);
3422 alarm(rec->switch_output.time);
3423 }
3424
03724b2e
AK		 3425 if (rec->switch_output.num_files) {
3426 rec->switch_output.filenames = calloc(sizeof(char *),
3427 rec->switch_output.num_files);
a8fcbd26
AH		 3428 if (!rec->switch_output.filenames) {
3429 err = -EINVAL;
3430 goto out_opts;
3431 }
03724b2e
AK		 3432 }
3433
1b36c03e
AH		 3434 /*
3435 * Allow aliases to facilitate the lookup of symbols for address
3436 * filters. Refer to auxtrace_parse_filters().
3437 */
3438 symbol_conf.allow_aliases = true;
3439
3440 symbol__init(NULL);
3441
4b5ea3bd 3442 err = record__auxtrace_init(rec);
1b36c03e
AH		 3443 if (err)
3444 goto out;
3445
0aab2136 3446 if (dry_run)
5c01ad60 3447 goto out;
0aab2136 3448
d7888573
WN		 3449 err = bpf__setup_stdout(rec->evlist);
3450 if (err) {
3451 bpf__strerror_setup_stdout(rec->evlist, err, errbuf, sizeof(errbuf));
3452 pr_err("ERROR: Setup BPF stdout failed: %s\n",
3453 errbuf);
5c01ad60 3454 goto out;
d7888573
WN		 3455 }
3456
ef149c25
AH		 3457 err = -ENOMEM;
3458
0c1d46a8 3459 if (rec->no_buildid_cache || rec->no_buildid) {
a1ac1d3c 3460 disable_buildid_cache();
dc0c6127 3461 } else if (rec->switch_output.enabled) {
0c1d46a8
WN		 3462 /*
3463 * In 'perf record --switch-output', disable buildid
3464 * generation by default to reduce data file switching
3465 * overhead. Still generate buildid if they are required
3466 * explicitly using
3467 *
60437ac0 3468 * perf record --switch-output --no-no-buildid \
0c1d46a8
WN		 3469 * --no-no-buildid-cache
3470 *
3471 * Following code equals to:
3472 *
3473 * if ((rec->no_buildid || !rec->no_buildid_set) &&
3474 * (rec->no_buildid_cache || !rec->no_buildid_cache_set))
3475 * disable_buildid_cache();
3476 */
3477 bool disable = true;
3478
3479 if (rec->no_buildid_set && !rec->no_buildid)
3480 disable = false;
3481 if (rec->no_buildid_cache_set && !rec->no_buildid_cache)
3482 disable = false;
3483 if (disable) {
3484 rec->no_buildid = true;
3485 rec->no_buildid_cache = true;
3486 disable_buildid_cache();
3487 }
3488 }
655000e7 3489
4ea648ae
WN		 3490 if (record.opts.overwrite)
3491 record.opts.tail_synthesize = true;
3492
b53a0755
JY		 3493 if (rec->evlist->core.nr_entries == 0) {
3494 if (perf_pmu__has_hybrid()) {
3495 err = evlist__add_default_hybrid(rec->evlist,
3496 !record.opts.no_samples);
3497 } else {
3498 err = __evlist__add_default(rec->evlist,
3499 !record.opts.no_samples);
3500 }
3501
3502 if (err < 0) {
3503 pr_err("Not enough memory for event selector list\n");
3504 goto out;
3505 }
bbd36e5e 3506 }
0e9b20b8 3507
69e7e5b0
AH		 3508 if (rec->opts.target.tid && !rec->opts.no_inherit_set)
3509 rec->opts.no_inherit = true;
3510
602ad878 3511 err = target__validate(&rec->opts.target);
16ad2ffb 3512 if (err) {
602ad878 3513 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
c3dec27b 3514 ui__warning("%s\n", errbuf);
16ad2ffb
NK		 3515 }
3516
602ad878 3517 err = target__parse_uid(&rec->opts.target);
16ad2ffb
NK		 3518 if (err) {
3519 int saved_errno = errno;
4bd0f2d2 3520
602ad878 3521 target__strerror(&rec->opts.target, err, errbuf, BUFSIZ);
3780f488 3522 ui__error("%s", errbuf);
16ad2ffb
NK		 3523
3524 err = -saved_errno;
394c01ed 3525 goto out;
16ad2ffb 3526 }
0d37aa34 3527
ca800068
MZ		 3528 /* Enable ignoring missing threads when -u/-p option is defined. */
3529 rec->opts.ignore_missing_thread = rec->opts.target.uid != UINT_MAX || rec->opts.target.pid;
23dc4f15 3530
1d3351e6
JY		 3531 if (evlist__fix_hybrid_cpus(rec->evlist, rec->opts.target.cpu_list)) {
3532 pr_err("failed to use cpu list %s\n",
3533 rec->opts.target.cpu_list);
3534 goto out;
3535 }
3536
3537 rec->opts.target.hybrid = perf_pmu__has_hybrid();
7248e308
AT		 3538
3539 if (callchain_param.enabled && callchain_param.record_mode == CALLCHAIN_FP)
3540 arch__add_leaf_frame_record_opts(&rec->opts);
3541
16ad2ffb 3542 err = -ENOMEM;
7748bb71 3543 if (evlist__create_maps(rec->evlist, &rec->opts.target) < 0)
dd7927f4 3544 usage_with_options(record_usage, record_options);
69aad6f1 3545
ef149c25
AH		 3546 err = auxtrace_record__options(rec->itr, rec->evlist, &rec->opts);
3547 if (err)
394c01ed 3548 goto out;
ef149c25 3549
6156681b
NK		 3550 /*
3551 * We take all buildids when the file contains
3552 * AUX area tracing data because we do not decode the
3553 * trace because it would take too long.
3554 */
3555 if (rec->opts.full_auxtrace)
3556 rec->buildid_all = true;
3557
246eba8e
AH		 3558 if (rec->opts.text_poke) {
3559 err = record__config_text_poke(rec->evlist);
3560 if (err) {
3561 pr_err("record__config_text_poke failed, error %d\n", err);
3562 goto out;
3563 }
3564 }
3565
b4006796 3566 if (record_opts__config(&rec->opts)) {
39d17dac 3567 err = -EINVAL;
394c01ed 3568 goto out;
7e4ff9e3
MG		 3569 }
3570
7954f716
AB		 3571 err = record__init_thread_masks(rec);
3572 if (err) {
3573 pr_err("Failed to initialize parallel data streaming masks\n");
3574 goto out;
3575 }
3576
93f20c0f
AB		 3577 if (rec->opts.nr_cblocks > nr_cblocks_max)
3578 rec->opts.nr_cblocks = nr_cblocks_max;
5d7f4116 3579 pr_debug("nr_cblocks: %d\n", rec->opts.nr_cblocks);
d3d1af6f 3580
9d2ed645 3581 pr_debug("affinity: %s\n", affinity_tags[rec->opts.affinity]);
470530bb 3582 pr_debug("mmap flush: %d\n", rec->opts.mmap_flush);
9d2ed645 3583
51255a8a
AB		 3584 if (rec->opts.comp_level > comp_level_max)
3585 rec->opts.comp_level = comp_level_max;
3586 pr_debug("comp level: %d\n", rec->opts.comp_level);
3587
d20deb64 3588 err = __cmd_record(&record, argc, argv);
394c01ed 3589out:
c12995a5 3590 evlist__delete(rec->evlist);
d65a458b 3591 symbol__exit();
ef149c25 3592 auxtrace_record__free(rec->itr);
a8fcbd26 3593out_opts:
7954f716
AB		 3594 record__free_thread_masks(rec, rec->nr_threads);
3595 rec->nr_threads = 0;
ee7fe31e 3596 evlist__close_control(rec->opts.ctl_fd, rec->opts.ctl_fd_ack, &rec->opts.ctl_fd_close);
39d17dac 3597 return err;
0e9b20b8 3598}
2dd6d8a1
AH		 3599
3600static void snapshot_sig_handler(int sig __maybe_unused)
3601{
dc0c6127
JO		 3602 struct record *rec = &record;
3603
d20aff15 3604 hit_auxtrace_snapshot_trigger(rec);
3c1cb7e3 3605
dc0c6127 3606 if (switch_output_signal(rec))
3c1cb7e3 3607 trigger_hit(&switch_output_trigger);
2dd6d8a1 3608}
bfacbe3b
JO		 3609
3610static void alarm_sig_handler(int sig __maybe_unused)
3611{
3612 struct record *rec = &record;
3613
3614 if (switch_output_time(rec))
3615 trigger_hit(&switch_output_trigger);
3616}
Linux 6.x block layer and io_uring tree(s)