tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <stdio.h>
  17 #include <stdbool.h>
  18 #include <errno.h>
  19 #include <linux/kernel.h>
  20 #include <linux/types.h>
  21
  22 #include "../perf.h"
  23 #include "session.h"
  24 #include "machine.h"
  25 #include "sort.h"
  26 #include "tool.h"
  27 #include "event.h"
  28 #include "evlist.h"
  29 #include "evsel.h"
  30 #include "map.h"
  31 #include "color.h"
  32 #include "util.h"
  33 #include "thread.h"
  34 #include "thread-stack.h"
  35 #include "symbol.h"
  36 #include "callchain.h"
  37 #include "dso.h"
  38 #include "debug.h"
  39 #include "auxtrace.h"
  40 #include "tsc.h"
  41 #include "intel-pt.h"
  42
  43 #include "intel-pt-decoder/intel-pt-log.h"
  44 #include "intel-pt-decoder/intel-pt-decoder.h"
  45 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  46 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  47
  48 #define MAX_TIMESTAMP (~0ULL)
  49
  50 struct intel_pt {
  51         struct auxtrace auxtrace;
  52         struct auxtrace_queues queues;
  53         struct auxtrace_heap heap;
  54         u32 auxtrace_type;
  55         struct perf_session *session;
  56         struct machine *machine;
  57         struct perf_evsel *switch_evsel;
  58         struct thread *unknown_thread;
  59         bool timeless_decoding;
  60         bool sampling_mode;
  61         bool snapshot_mode;
  62         bool per_cpu_mmaps;
  63         bool have_tsc;
  64         bool data_queued;
  65         bool est_tsc;
  66         bool sync_switch;
  67         bool mispred_all;
  68         int have_sched_switch;
  69         u32 pmu_type;
  70         u64 kernel_start;
  71         u64 switch_ip;
  72         u64 ptss_ip;
  73
  74         struct perf_tsc_conversion tc;
  75         bool cap_user_time_zero;
  76
  77         struct itrace_synth_opts synth_opts;
  78
  79         bool sample_instructions;
  80         u64 instructions_sample_type;
  81         u64 instructions_sample_period;
  82         u64 instructions_id;
  83
  84         bool sample_branches;
  85         u32 branches_filter;
  86         u64 branches_sample_type;
  87         u64 branches_id;
  88
  89         bool sample_transactions;
  90         u64 transactions_sample_type;
  91         u64 transactions_id;
  92
  93         bool synth_needs_swap;
  94
  95         u64 tsc_bit;
  96         u64 mtc_bit;
  97         u64 mtc_freq_bits;
  98         u32 tsc_ctc_ratio_n;
  99         u32 tsc_ctc_ratio_d;
 100         u64 cyc_bit;
 101         u64 noretcomp_bit;
 102         unsigned max_non_turbo_ratio;
 103 };
 104
 105 enum switch_state {
 106         INTEL_PT_SS_NOT_TRACING,
 107         INTEL_PT_SS_UNKNOWN,
 108         INTEL_PT_SS_TRACING,
 109         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 110         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 111 };
 112
 113 struct intel_pt_queue {
 114         struct intel_pt *pt;
 115         unsigned int queue_nr;
 116         struct auxtrace_buffer *buffer;
 117         void *decoder;
 118         const struct intel_pt_state *state;
 119         struct ip_callchain *chain;
 120         struct branch_stack *last_branch;
 121         struct branch_stack *last_branch_rb;
 122         size_t last_branch_pos;
 123         union perf_event *event_buf;
 124         bool on_heap;
 125         bool stop;
 126         bool step_through_buffers;
 127         bool use_buffer_pid_tid;
 128         pid_t pid, tid;
 129         int cpu;
 130         int switch_state;
 131         pid_t next_tid;
 132         struct thread *thread;
 133         bool exclude_kernel;
 134         bool have_sample;
 135         u64 time;
 136         u64 timestamp;
 137         u32 flags;
 138         u16 insn_len;
 139         u64 last_insn_cnt;
 140 };
 141
 142 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 143                           unsigned char *buf, size_t len)
 144 {
 145         struct intel_pt_pkt packet;
 146         size_t pos = 0;
 147         int ret, pkt_len, i;
 148         char desc[INTEL_PT_PKT_DESC_MAX];
 149         const char *color = PERF_COLOR_BLUE;
 150
 151         color_fprintf(stdout, color,
 152                       ". ... Intel Processor Trace data: size %zu bytes\n",
 153                       len);
 154
 155         while (len) {
 156                 ret = intel_pt_get_packet(buf, len, &packet);
 157                 if (ret > 0)
 158                         pkt_len = ret;
 159                 else
 160                         pkt_len = 1;
 161                 printf(".");
 162                 color_fprintf(stdout, color, "  %08x: ", pos);
 163                 for (i = 0; i < pkt_len; i++)
 164                         color_fprintf(stdout, color, " %02x", buf[i]);
 165                 for (; i < 16; i++)
 166                         color_fprintf(stdout, color, "   ");
 167                 if (ret > 0) {
 168                         ret = intel_pt_pkt_desc(&packet, desc,
 169                                                 INTEL_PT_PKT_DESC_MAX);
 170                         if (ret > 0)
 171                                 color_fprintf(stdout, color, " %s\n", desc);
 172                 } else {
 173                         color_fprintf(stdout, color, " Bad packet!\n");
 174                 }
 175                 pos += pkt_len;
 176                 buf += pkt_len;
 177                 len -= pkt_len;
 178         }
 179 }
 180
 181 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 182                                 size_t len)
 183 {
 184         printf(".\n");
 185         intel_pt_dump(pt, buf, len);
 186 }
 187
 188 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 189                                    struct auxtrace_buffer *b)
 190 {
 191         void *start;
 192
 193         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 194                                       pt->have_tsc);
 195         if (!start)
 196                 return -EINVAL;
 197         b->use_size = b->data + b->size - start;
 198         b->use_data = start;
 199         return 0;
 200 }
 201
 202 static void intel_pt_use_buffer_pid_tid(struct intel_pt_queue *ptq,
 203                                         struct auxtrace_queue *queue,
 204                                         struct auxtrace_buffer *buffer)
 205 {
 206         if (queue->cpu == -1 && buffer->cpu != -1)
 207                 ptq->cpu = buffer->cpu;
 208
 209         ptq->pid = buffer->pid;
 210         ptq->tid = buffer->tid;
 211
 212         intel_pt_log("queue %u cpu %d pid %d tid %d\n",
 213                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 214
 215         thread__zput(ptq->thread);
 216
 217         if (ptq->tid != -1) {
 218                 if (ptq->pid != -1)
 219                         ptq->thread = machine__findnew_thread(ptq->pt->machine,
 220                                                               ptq->pid,
 221                                                               ptq->tid);
 222                 else
 223                         ptq->thread = machine__find_thread(ptq->pt->machine, -1,
 224                                                            ptq->tid);
 225         }
 226 }
 227
 228 /* This function assumes data is processed sequentially only */
 229 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 230 {
 231         struct intel_pt_queue *ptq = data;
 232         struct auxtrace_buffer *buffer = ptq->buffer, *old_buffer = buffer;
 233         struct auxtrace_queue *queue;
 234
 235         if (ptq->stop) {
 236                 b->len = 0;
 237                 return 0;
 238         }
 239
 240         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 241
 242         buffer = auxtrace_buffer__next(queue, buffer);
 243         if (!buffer) {
 244                 if (old_buffer)
 245                         auxtrace_buffer__drop_data(old_buffer);
 246                 b->len = 0;
 247                 return 0;
 248         }
 249
 250         ptq->buffer = buffer;
 251
 252         if (!buffer->data) {
 253                 int fd = perf_data_file__fd(ptq->pt->session->file);
 254
 255                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 256                 if (!buffer->data)
 257                         return -ENOMEM;
 258         }
 259
 260         if (ptq->pt->snapshot_mode && !buffer->consecutive && old_buffer &&
 261             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 262                 return -ENOMEM;
 263
 264         if (old_buffer)
 265                 auxtrace_buffer__drop_data(old_buffer);
 266
 267         if (buffer->use_data) {
 268                 b->len = buffer->use_size;
 269                 b->buf = buffer->use_data;
 270         } else {
 271                 b->len = buffer->size;
 272                 b->buf = buffer->data;
 273         }
 274         b->ref_timestamp = buffer->reference;
 275
 276         if (!old_buffer || ptq->pt->sampling_mode || (ptq->pt->snapshot_mode &&
 277                                                       !buffer->consecutive)) {
 278                 b->consecutive = false;
 279                 b->trace_nr = buffer->buffer_nr + 1;
 280         } else {
 281                 b->consecutive = true;
 282         }
 283
 284         if (ptq->use_buffer_pid_tid && (ptq->pid != buffer->pid ||
 285                                         ptq->tid != buffer->tid))
 286                 intel_pt_use_buffer_pid_tid(ptq, queue, buffer);
 287
 288         if (ptq->step_through_buffers)
 289                 ptq->stop = true;
 290
 291         if (!b->len)
 292                 return intel_pt_get_trace(b, data);
 293
 294         return 0;
 295 }
 296
 297 struct intel_pt_cache_entry {
 298         struct auxtrace_cache_entry     entry;
 299         u64                             insn_cnt;
 300         u64                             byte_cnt;
 301         enum intel_pt_insn_op           op;
 302         enum intel_pt_insn_branch       branch;
 303         int                             length;
 304         int32_t                         rel;
 305 };
 306
 307 static int intel_pt_config_div(const char *var, const char *value, void *data)
 308 {
 309         int *d = data;
 310         long val;
 311
 312         if (!strcmp(var, "intel-pt.cache-divisor")) {
 313                 val = strtol(value, NULL, 0);
 314                 if (val > 0 && val <= INT_MAX)
 315                         *d = val;
 316         }
 317
 318         return 0;
 319 }
 320
 321 static int intel_pt_cache_divisor(void)
 322 {
 323         static int d;
 324
 325         if (d)
 326                 return d;
 327
 328         perf_config(intel_pt_config_div, &d);
 329
 330         if (!d)
 331                 d = 64;
 332
 333         return d;
 334 }
 335
 336 static unsigned int intel_pt_cache_size(struct dso *dso,
 337                                         struct machine *machine)
 338 {
 339         off_t size;
 340
 341         size = dso__data_size(dso, machine);
 342         size /= intel_pt_cache_divisor();
 343         if (size < 1000)
 344                 return 10;
 345         if (size > (1 << 21))
 346                 return 21;
 347         return 32 - __builtin_clz(size);
 348 }
 349
 350 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 351                                              struct machine *machine)
 352 {
 353         struct auxtrace_cache *c;
 354         unsigned int bits;
 355
 356         if (dso->auxtrace_cache)
 357                 return dso->auxtrace_cache;
 358
 359         bits = intel_pt_cache_size(dso, machine);
 360
 361         /* Ignoring cache creation failure */
 362         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 363
 364         dso->auxtrace_cache = c;
 365
 366         return c;
 367 }
 368
 369 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 370                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 371                               struct intel_pt_insn *intel_pt_insn)
 372 {
 373         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 374         struct intel_pt_cache_entry *e;
 375         int err;
 376
 377         if (!c)
 378                 return -ENOMEM;
 379
 380         e = auxtrace_cache__alloc_entry(c);
 381         if (!e)
 382                 return -ENOMEM;
 383
 384         e->insn_cnt = insn_cnt;
 385         e->byte_cnt = byte_cnt;
 386         e->op = intel_pt_insn->op;
 387         e->branch = intel_pt_insn->branch;
 388         e->length = intel_pt_insn->length;
 389         e->rel = intel_pt_insn->rel;
 390
 391         err = auxtrace_cache__add(c, offset, &e->entry);
 392         if (err)
 393                 auxtrace_cache__free_entry(c, e);
 394
 395         return err;
 396 }
 397
 398 static struct intel_pt_cache_entry *
 399 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 400 {
 401         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 402
 403         if (!c)
 404                 return NULL;
 405
 406         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 407 }
 408
 409 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 410                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 411                                    uint64_t to_ip, uint64_t max_insn_cnt,
 412                                    void *data)
 413 {
 414         struct intel_pt_queue *ptq = data;
 415         struct machine *machine = ptq->pt->machine;
 416         struct thread *thread;
 417         struct addr_location al;
 418         unsigned char buf[1024];
 419         size_t bufsz;
 420         ssize_t len;
 421         int x86_64;
 422         u8 cpumode;
 423         u64 offset, start_offset, start_ip;
 424         u64 insn_cnt = 0;
 425         bool one_map = true;
 426
 427         if (to_ip && *ip == to_ip)
 428                 goto out_no_cache;
 429
 430         bufsz = intel_pt_insn_max_size();
 431
 432         if (*ip >= ptq->pt->kernel_start)
 433                 cpumode = PERF_RECORD_MISC_KERNEL;
 434         else
 435                 cpumode = PERF_RECORD_MISC_USER;
 436
 437         thread = ptq->thread;
 438         if (!thread) {
 439                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 440                         return -EINVAL;
 441                 thread = ptq->pt->unknown_thread;
 442         }
 443
 444         while (1) {
 445                 thread__find_addr_map(thread, cpumode, MAP__FUNCTION, *ip, &al);
 446                 if (!al.map || !al.map->dso)
 447                         return -EINVAL;
 448
 449                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 450                     dso__data_status_seen(al.map->dso,
 451                                           DSO_DATA_STATUS_SEEN_ITRACE))
 452                         return -ENOENT;
 453
 454                 offset = al.map->map_ip(al.map, *ip);
 455
 456                 if (!to_ip && one_map) {
 457                         struct intel_pt_cache_entry *e;
 458
 459                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 460                         if (e &&
 461                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 462                                 *insn_cnt_ptr = e->insn_cnt;
 463                                 *ip += e->byte_cnt;
 464                                 intel_pt_insn->op = e->op;
 465                                 intel_pt_insn->branch = e->branch;
 466                                 intel_pt_insn->length = e->length;
 467                                 intel_pt_insn->rel = e->rel;
 468                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 469                                 return 0;
 470                         }
 471                 }
 472
 473                 start_offset = offset;
 474                 start_ip = *ip;
 475
 476                 /* Load maps to ensure dso->is_64_bit has been updated */
 477                 map__load(al.map, machine->symbol_filter);
 478
 479                 x86_64 = al.map->dso->is_64_bit;
 480
 481                 while (1) {
 482                         len = dso__data_read_offset(al.map->dso, machine,
 483                                                     offset, buf, bufsz);
 484                         if (len <= 0)
 485                                 return -EINVAL;
 486
 487                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 488                                 return -EINVAL;
 489
 490                         intel_pt_log_insn(intel_pt_insn, *ip);
 491
 492                         insn_cnt += 1;
 493
 494                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 495                                 goto out;
 496
 497                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 498                                 goto out_no_cache;
 499
 500                         *ip += intel_pt_insn->length;
 501
 502                         if (to_ip && *ip == to_ip)
 503                                 goto out_no_cache;
 504
 505                         if (*ip >= al.map->end)
 506                                 break;
 507
 508                         offset += intel_pt_insn->length;
 509                 }
 510                 one_map = false;
 511         }
 512 out:
 513         *insn_cnt_ptr = insn_cnt;
 514
 515         if (!one_map)
 516                 goto out_no_cache;
 517
 518         /*
 519          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 520          * entries.
 521          */
 522         if (to_ip) {
 523                 struct intel_pt_cache_entry *e;
 524
 525                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 526                 if (e)
 527                         return 0;
 528         }
 529
 530         /* Ignore cache errors */
 531         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 532                            *ip - start_ip, intel_pt_insn);
 533
 534         return 0;
 535
 536 out_no_cache:
 537         *insn_cnt_ptr = insn_cnt;
 538         return 0;
 539 }
 540
 541 static bool intel_pt_get_config(struct intel_pt *pt,
 542                                 struct perf_event_attr *attr, u64 *config)
 543 {
 544         if (attr->type == pt->pmu_type) {
 545                 if (config)
 546                         *config = attr->config;
 547                 return true;
 548         }
 549
 550         return false;
 551 }
 552
 553 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 554 {
 555         struct perf_evsel *evsel;
 556
 557         evlist__for_each(pt->session->evlist, evsel) {
 558                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 559                     !evsel->attr.exclude_kernel)
 560                         return false;
 561         }
 562         return true;
 563 }
 564
 565 static bool intel_pt_return_compression(struct intel_pt *pt)
 566 {
 567         struct perf_evsel *evsel;
 568         u64 config;
 569
 570         if (!pt->noretcomp_bit)
 571                 return true;
 572
 573         evlist__for_each(pt->session->evlist, evsel) {
 574                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 575                     (config & pt->noretcomp_bit))
 576                         return false;
 577         }
 578         return true;
 579 }
 580
 581 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 582 {
 583         struct perf_evsel *evsel;
 584         unsigned int shift;
 585         u64 config;
 586
 587         if (!pt->mtc_freq_bits)
 588                 return 0;
 589
 590         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 591                 config >>= 1;
 592
 593         evlist__for_each(pt->session->evlist, evsel) {
 594                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 595                         return (config & pt->mtc_freq_bits) >> shift;
 596         }
 597         return 0;
 598 }
 599
 600 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 601 {
 602         struct perf_evsel *evsel;
 603         bool timeless_decoding = true;
 604         u64 config;
 605
 606         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 607                 return true;
 608
 609         evlist__for_each(pt->session->evlist, evsel) {
 610                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 611                         return true;
 612                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 613                         if (config & pt->tsc_bit)
 614                                 timeless_decoding = false;
 615                         else
 616                                 return true;
 617                 }
 618         }
 619         return timeless_decoding;
 620 }
 621
 622 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 623 {
 624         struct perf_evsel *evsel;
 625
 626         evlist__for_each(pt->session->evlist, evsel) {
 627                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 628                     !evsel->attr.exclude_kernel)
 629                         return true;
 630         }
 631         return false;
 632 }
 633
 634 static bool intel_pt_have_tsc(struct intel_pt *pt)
 635 {
 636         struct perf_evsel *evsel;
 637         bool have_tsc = false;
 638         u64 config;
 639
 640         if (!pt->tsc_bit)
 641                 return false;
 642
 643         evlist__for_each(pt->session->evlist, evsel) {
 644                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 645                         if (config & pt->tsc_bit)
 646                                 have_tsc = true;
 647                         else
 648                                 return false;
 649                 }
 650         }
 651         return have_tsc;
 652 }
 653
 654 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 655 {
 656         u64 quot, rem;
 657
 658         quot = ns / pt->tc.time_mult;
 659         rem  = ns % pt->tc.time_mult;
 660         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 661                 pt->tc.time_mult;
 662 }
 663
 664 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 665                                                    unsigned int queue_nr)
 666 {
 667         struct intel_pt_params params = { .get_trace = 0, };
 668         struct intel_pt_queue *ptq;
 669
 670         ptq = zalloc(sizeof(struct intel_pt_queue));
 671         if (!ptq)
 672                 return NULL;
 673
 674         if (pt->synth_opts.callchain) {
 675                 size_t sz = sizeof(struct ip_callchain);
 676
 677                 sz += pt->synth_opts.callchain_sz * sizeof(u64);
 678                 ptq->chain = zalloc(sz);
 679                 if (!ptq->chain)
 680                         goto out_free;
 681         }
 682
 683         if (pt->synth_opts.last_branch) {
 684                 size_t sz = sizeof(struct branch_stack);
 685
 686                 sz += pt->synth_opts.last_branch_sz *
 687                       sizeof(struct branch_entry);
 688                 ptq->last_branch = zalloc(sz);
 689                 if (!ptq->last_branch)
 690                         goto out_free;
 691                 ptq->last_branch_rb = zalloc(sz);
 692                 if (!ptq->last_branch_rb)
 693                         goto out_free;
 694         }
 695
 696         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 697         if (!ptq->event_buf)
 698                 goto out_free;
 699
 700         ptq->pt = pt;
 701         ptq->queue_nr = queue_nr;
 702         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 703         ptq->pid = -1;
 704         ptq->tid = -1;
 705         ptq->cpu = -1;
 706         ptq->next_tid = -1;
 707
 708         params.get_trace = intel_pt_get_trace;
 709         params.walk_insn = intel_pt_walk_next_insn;
 710         params.data = ptq;
 711         params.return_compression = intel_pt_return_compression(pt);
 712         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 713         params.mtc_period = intel_pt_mtc_period(pt);
 714         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 715         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 716
 717         if (pt->synth_opts.instructions) {
 718                 if (pt->synth_opts.period) {
 719                         switch (pt->synth_opts.period_type) {
 720                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 721                                 params.period_type =
 722                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 723                                 params.period = pt->synth_opts.period;
 724                                 break;
 725                         case PERF_ITRACE_PERIOD_TICKS:
 726                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 727                                 params.period = pt->synth_opts.period;
 728                                 break;
 729                         case PERF_ITRACE_PERIOD_NANOSECS:
 730                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 731                                 params.period = intel_pt_ns_to_ticks(pt,
 732                                                         pt->synth_opts.period);
 733                                 break;
 734                         default:
 735                                 break;
 736                         }
 737                 }
 738
 739                 if (!params.period) {
 740                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 741                         params.period = 1;
 742                 }
 743         }
 744
 745         ptq->decoder = intel_pt_decoder_new(&params);
 746         if (!ptq->decoder)
 747                 goto out_free;
 748
 749         return ptq;
 750
 751 out_free:
 752         zfree(&ptq->event_buf);
 753         zfree(&ptq->last_branch);
 754         zfree(&ptq->last_branch_rb);
 755         zfree(&ptq->chain);
 756         free(ptq);
 757         return NULL;
 758 }
 759
 760 static void intel_pt_free_queue(void *priv)
 761 {
 762         struct intel_pt_queue *ptq = priv;
 763
 764         if (!ptq)
 765                 return;
 766         thread__zput(ptq->thread);
 767         intel_pt_decoder_free(ptq->decoder);
 768         zfree(&ptq->event_buf);
 769         zfree(&ptq->last_branch);
 770         zfree(&ptq->last_branch_rb);
 771         zfree(&ptq->chain);
 772         free(ptq);
 773 }
 774
 775 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 776                                      struct auxtrace_queue *queue)
 777 {
 778         struct intel_pt_queue *ptq = queue->priv;
 779
 780         if (queue->tid == -1 || pt->have_sched_switch) {
 781                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 782                 thread__zput(ptq->thread);
 783         }
 784
 785         if (!ptq->thread && ptq->tid != -1)
 786                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 787
 788         if (ptq->thread) {
 789                 ptq->pid = ptq->thread->pid_;
 790                 if (queue->cpu == -1)
 791                         ptq->cpu = ptq->thread->cpu;
 792         }
 793 }
 794
 795 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 796 {
 797         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 798                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 799         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 800                 if (ptq->state->to_ip)
 801                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 802                                      PERF_IP_FLAG_ASYNC |
 803                                      PERF_IP_FLAG_INTERRUPT;
 804                 else
 805                         ptq->flags = PERF_IP_FLAG_BRANCH |
 806                                      PERF_IP_FLAG_TRACE_END;
 807                 ptq->insn_len = 0;
 808         } else {
 809                 if (ptq->state->from_ip)
 810                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 811                 else
 812                         ptq->flags = PERF_IP_FLAG_BRANCH |
 813                                      PERF_IP_FLAG_TRACE_BEGIN;
 814                 if (ptq->state->flags & INTEL_PT_IN_TX)
 815                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 816                 ptq->insn_len = ptq->state->insn_len;
 817         }
 818 }
 819
 820 static int intel_pt_setup_queue(struct intel_pt *pt,
 821                                 struct auxtrace_queue *queue,
 822                                 unsigned int queue_nr)
 823 {
 824         struct intel_pt_queue *ptq = queue->priv;
 825
 826         if (list_empty(&queue->head))
 827                 return 0;
 828
 829         if (!ptq) {
 830                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 831                 if (!ptq)
 832                         return -ENOMEM;
 833                 queue->priv = ptq;
 834
 835                 if (queue->cpu != -1)
 836                         ptq->cpu = queue->cpu;
 837                 ptq->tid = queue->tid;
 838
 839                 if (pt->sampling_mode) {
 840                         if (pt->timeless_decoding)
 841                                 ptq->step_through_buffers = true;
 842                         if (pt->timeless_decoding || !pt->have_sched_switch)
 843                                 ptq->use_buffer_pid_tid = true;
 844                 }
 845         }
 846
 847         if (!ptq->on_heap &&
 848             (!pt->sync_switch ||
 849              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 850                 const struct intel_pt_state *state;
 851                 int ret;
 852
 853                 if (pt->timeless_decoding)
 854                         return 0;
 855
 856                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 857                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 858                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 859                 while (1) {
 860                         state = intel_pt_decode(ptq->decoder);
 861                         if (state->err) {
 862                                 if (state->err == INTEL_PT_ERR_NODATA) {
 863                                         intel_pt_log("queue %u has no timestamp\n",
 864                                                      queue_nr);
 865                                         return 0;
 866                                 }
 867                                 continue;
 868                         }
 869                         if (state->timestamp)
 870                                 break;
 871                 }
 872
 873                 ptq->timestamp = state->timestamp;
 874                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 875                              queue_nr, ptq->timestamp);
 876                 ptq->state = state;
 877                 ptq->have_sample = true;
 878                 intel_pt_sample_flags(ptq);
 879                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 880                 if (ret)
 881                         return ret;
 882                 ptq->on_heap = true;
 883         }
 884
 885         return 0;
 886 }
 887
 888 static int intel_pt_setup_queues(struct intel_pt *pt)
 889 {
 890         unsigned int i;
 891         int ret;
 892
 893         for (i = 0; i < pt->queues.nr_queues; i++) {
 894                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
 895                 if (ret)
 896                         return ret;
 897         }
 898         return 0;
 899 }
 900
 901 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
 902 {
 903         struct branch_stack *bs_src = ptq->last_branch_rb;
 904         struct branch_stack *bs_dst = ptq->last_branch;
 905         size_t nr = 0;
 906
 907         bs_dst->nr = bs_src->nr;
 908
 909         if (!bs_src->nr)
 910                 return;
 911
 912         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
 913         memcpy(&bs_dst->entries[0],
 914                &bs_src->entries[ptq->last_branch_pos],
 915                sizeof(struct branch_entry) * nr);
 916
 917         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
 918                 memcpy(&bs_dst->entries[nr],
 919                        &bs_src->entries[0],
 920                        sizeof(struct branch_entry) * ptq->last_branch_pos);
 921         }
 922 }
 923
 924 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
 925 {
 926         ptq->last_branch_pos = 0;
 927         ptq->last_branch_rb->nr = 0;
 928 }
 929
 930 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
 931 {
 932         const struct intel_pt_state *state = ptq->state;
 933         struct branch_stack *bs = ptq->last_branch_rb;
 934         struct branch_entry *be;
 935
 936         if (!ptq->last_branch_pos)
 937                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
 938
 939         ptq->last_branch_pos -= 1;
 940
 941         be              = &bs->entries[ptq->last_branch_pos];
 942         be->from        = state->from_ip;
 943         be->to          = state->to_ip;
 944         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
 945         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
 946         /* No support for mispredict */
 947         be->flags.mispred = ptq->pt->mispred_all;
 948
 949         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
 950                 bs->nr += 1;
 951 }
 952
 953 static int intel_pt_inject_event(union perf_event *event,
 954                                  struct perf_sample *sample, u64 type,
 955                                  bool swapped)
 956 {
 957         event->header.size = perf_event__sample_event_size(sample, type, 0);
 958         return perf_event__synthesize_sample(event, type, 0, sample, swapped);
 959 }
 960
 961 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
 962 {
 963         int ret;
 964         struct intel_pt *pt = ptq->pt;
 965         union perf_event *event = ptq->event_buf;
 966         struct perf_sample sample = { .ip = 0, };
 967         struct dummy_branch_stack {
 968                 u64                     nr;
 969                 struct branch_entry     entries;
 970         } dummy_bs;
 971
 972         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
 973                 return 0;
 974
 975         event->sample.header.type = PERF_RECORD_SAMPLE;
 976         event->sample.header.misc = PERF_RECORD_MISC_USER;
 977         event->sample.header.size = sizeof(struct perf_event_header);
 978
 979         if (!pt->timeless_decoding)
 980                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
 981
 982         sample.cpumode = PERF_RECORD_MISC_USER;
 983         sample.ip = ptq->state->from_ip;
 984         sample.pid = ptq->pid;
 985         sample.tid = ptq->tid;
 986         sample.addr = ptq->state->to_ip;
 987         sample.id = ptq->pt->branches_id;
 988         sample.stream_id = ptq->pt->branches_id;
 989         sample.period = 1;
 990         sample.cpu = ptq->cpu;
 991         sample.flags = ptq->flags;
 992         sample.insn_len = ptq->insn_len;
 993
 994         /*
 995          * perf report cannot handle events without a branch stack when using
 996          * SORT_MODE__BRANCH so make a dummy one.
 997          */
 998         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
 999                 dummy_bs = (struct dummy_branch_stack){
1000                         .nr = 1,
1001                         .entries = {
1002                                 .from = sample.ip,
1003                                 .to = sample.addr,
1004                         },
1005                 };
1006                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1007         }
1008
1009         if (pt->synth_opts.inject) {
1010                 ret = intel_pt_inject_event(event, &sample,
1011                                             pt->branches_sample_type,
1012                                             pt->synth_needs_swap);
1013                 if (ret)
1014                         return ret;
1015         }
1016
1017         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1018         if (ret)
1019                 pr_err("Intel Processor Trace: failed to deliver branch event, error %d\n",
1020                        ret);
1021
1022         return ret;
1023 }
1024
1025 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1026 {
1027         int ret;
1028         struct intel_pt *pt = ptq->pt;
1029         union perf_event *event = ptq->event_buf;
1030         struct perf_sample sample = { .ip = 0, };
1031
1032         event->sample.header.type = PERF_RECORD_SAMPLE;
1033         event->sample.header.misc = PERF_RECORD_MISC_USER;
1034         event->sample.header.size = sizeof(struct perf_event_header);
1035
1036         if (!pt->timeless_decoding)
1037                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1038
1039         sample.cpumode = PERF_RECORD_MISC_USER;
1040         sample.ip = ptq->state->from_ip;
1041         sample.pid = ptq->pid;
1042         sample.tid = ptq->tid;
1043         sample.addr = ptq->state->to_ip;
1044         sample.id = ptq->pt->instructions_id;
1045         sample.stream_id = ptq->pt->instructions_id;
1046         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1047         sample.cpu = ptq->cpu;
1048         sample.flags = ptq->flags;
1049         sample.insn_len = ptq->insn_len;
1050
1051         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1052
1053         if (pt->synth_opts.callchain) {
1054                 thread_stack__sample(ptq->thread, ptq->chain,
1055                                      pt->synth_opts.callchain_sz, sample.ip);
1056                 sample.callchain = ptq->chain;
1057         }
1058
1059         if (pt->synth_opts.last_branch) {
1060                 intel_pt_copy_last_branch_rb(ptq);
1061                 sample.branch_stack = ptq->last_branch;
1062         }
1063
1064         if (pt->synth_opts.inject) {
1065                 ret = intel_pt_inject_event(event, &sample,
1066                                             pt->instructions_sample_type,
1067                                             pt->synth_needs_swap);
1068                 if (ret)
1069                         return ret;
1070         }
1071
1072         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1073         if (ret)
1074                 pr_err("Intel Processor Trace: failed to deliver instruction event, error %d\n",
1075                        ret);
1076
1077         if (pt->synth_opts.last_branch)
1078                 intel_pt_reset_last_branch_rb(ptq);
1079
1080         return ret;
1081 }
1082
1083 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1084 {
1085         int ret;
1086         struct intel_pt *pt = ptq->pt;
1087         union perf_event *event = ptq->event_buf;
1088         struct perf_sample sample = { .ip = 0, };
1089
1090         event->sample.header.type = PERF_RECORD_SAMPLE;
1091         event->sample.header.misc = PERF_RECORD_MISC_USER;
1092         event->sample.header.size = sizeof(struct perf_event_header);
1093
1094         if (!pt->timeless_decoding)
1095                 sample.time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1096
1097         sample.cpumode = PERF_RECORD_MISC_USER;
1098         sample.ip = ptq->state->from_ip;
1099         sample.pid = ptq->pid;
1100         sample.tid = ptq->tid;
1101         sample.addr = ptq->state->to_ip;
1102         sample.id = ptq->pt->transactions_id;
1103         sample.stream_id = ptq->pt->transactions_id;
1104         sample.period = 1;
1105         sample.cpu = ptq->cpu;
1106         sample.flags = ptq->flags;
1107         sample.insn_len = ptq->insn_len;
1108
1109         if (pt->synth_opts.callchain) {
1110                 thread_stack__sample(ptq->thread, ptq->chain,
1111                                      pt->synth_opts.callchain_sz, sample.ip);
1112                 sample.callchain = ptq->chain;
1113         }
1114
1115         if (pt->synth_opts.last_branch) {
1116                 intel_pt_copy_last_branch_rb(ptq);
1117                 sample.branch_stack = ptq->last_branch;
1118         }
1119
1120         if (pt->synth_opts.inject) {
1121                 ret = intel_pt_inject_event(event, &sample,
1122                                             pt->transactions_sample_type,
1123                                             pt->synth_needs_swap);
1124                 if (ret)
1125                         return ret;
1126         }
1127
1128         ret = perf_session__deliver_synth_event(pt->session, event, &sample);
1129         if (ret)
1130                 pr_err("Intel Processor Trace: failed to deliver transaction event, error %d\n",
1131                        ret);
1132
1133         if (pt->synth_opts.callchain)
1134                 intel_pt_reset_last_branch_rb(ptq);
1135
1136         return ret;
1137 }
1138
1139 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1140                                 pid_t pid, pid_t tid, u64 ip)
1141 {
1142         union perf_event event;
1143         char msg[MAX_AUXTRACE_ERROR_MSG];
1144         int err;
1145
1146         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1147
1148         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1149                              code, cpu, pid, tid, ip, msg);
1150
1151         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1152         if (err)
1153                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1154                        err);
1155
1156         return err;
1157 }
1158
1159 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1160 {
1161         struct auxtrace_queue *queue;
1162         pid_t tid = ptq->next_tid;
1163         int err;
1164
1165         if (tid == -1)
1166                 return 0;
1167
1168         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1169
1170         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1171
1172         queue = &pt->queues.queue_array[ptq->queue_nr];
1173         intel_pt_set_pid_tid_cpu(pt, queue);
1174
1175         ptq->next_tid = -1;
1176
1177         return err;
1178 }
1179
1180 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1181 {
1182         struct intel_pt *pt = ptq->pt;
1183
1184         return ip == pt->switch_ip &&
1185                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1186                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1187                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1188 }
1189
1190 static int intel_pt_sample(struct intel_pt_queue *ptq)
1191 {
1192         const struct intel_pt_state *state = ptq->state;
1193         struct intel_pt *pt = ptq->pt;
1194         int err;
1195
1196         if (!ptq->have_sample)
1197                 return 0;
1198
1199         ptq->have_sample = false;
1200
1201         if (pt->sample_instructions &&
1202             (state->type & INTEL_PT_INSTRUCTION)) {
1203                 err = intel_pt_synth_instruction_sample(ptq);
1204                 if (err)
1205                         return err;
1206         }
1207
1208         if (pt->sample_transactions &&
1209             (state->type & INTEL_PT_TRANSACTION)) {
1210                 err = intel_pt_synth_transaction_sample(ptq);
1211                 if (err)
1212                         return err;
1213         }
1214
1215         if (!(state->type & INTEL_PT_BRANCH))
1216                 return 0;
1217
1218         if (pt->synth_opts.callchain)
1219                 thread_stack__event(ptq->thread, ptq->flags, state->from_ip,
1220                                     state->to_ip, ptq->insn_len,
1221                                     state->trace_nr);
1222         else
1223                 thread_stack__set_trace_nr(ptq->thread, state->trace_nr);
1224
1225         if (pt->sample_branches) {
1226                 err = intel_pt_synth_branch_sample(ptq);
1227                 if (err)
1228                         return err;
1229         }
1230
1231         if (pt->synth_opts.last_branch)
1232                 intel_pt_update_last_branch_rb(ptq);
1233
1234         if (!pt->sync_switch)
1235                 return 0;
1236
1237         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1238                 switch (ptq->switch_state) {
1239                 case INTEL_PT_SS_UNKNOWN:
1240                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1241                         err = intel_pt_next_tid(pt, ptq);
1242                         if (err)
1243                                 return err;
1244                         ptq->switch_state = INTEL_PT_SS_TRACING;
1245                         break;
1246                 default:
1247                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1248                         return 1;
1249                 }
1250         } else if (!state->to_ip) {
1251                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1252         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1253                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1254         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1255                    state->to_ip == pt->ptss_ip &&
1256                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1257                 ptq->switch_state = INTEL_PT_SS_TRACING;
1258         }
1259
1260         return 0;
1261 }
1262
1263 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1264 {
1265         struct machine *machine = pt->machine;
1266         struct map *map;
1267         struct symbol *sym, *start;
1268         u64 ip, switch_ip = 0;
1269         const char *ptss;
1270
1271         if (ptss_ip)
1272                 *ptss_ip = 0;
1273
1274         map = machine__kernel_map(machine);
1275         if (!map)
1276                 return 0;
1277
1278         if (map__load(map, machine->symbol_filter))
1279                 return 0;
1280
1281         start = dso__first_symbol(map->dso, MAP__FUNCTION);
1282
1283         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1284                 if (sym->binding == STB_GLOBAL &&
1285                     !strcmp(sym->name, "__switch_to")) {
1286                         ip = map->unmap_ip(map, sym->start);
1287                         if (ip >= map->start && ip < map->end) {
1288                                 switch_ip = ip;
1289                                 break;
1290                         }
1291                 }
1292         }
1293
1294         if (!switch_ip || !ptss_ip)
1295                 return 0;
1296
1297         if (pt->have_sched_switch == 1)
1298                 ptss = "perf_trace_sched_switch";
1299         else
1300                 ptss = "__perf_event_task_sched_out";
1301
1302         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1303                 if (!strcmp(sym->name, ptss)) {
1304                         ip = map->unmap_ip(map, sym->start);
1305                         if (ip >= map->start && ip < map->end) {
1306                                 *ptss_ip = ip;
1307                                 break;
1308                         }
1309                 }
1310         }
1311
1312         return switch_ip;
1313 }
1314
1315 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1316 {
1317         const struct intel_pt_state *state = ptq->state;
1318         struct intel_pt *pt = ptq->pt;
1319         int err;
1320
1321         if (!pt->kernel_start) {
1322                 pt->kernel_start = machine__kernel_start(pt->machine);
1323                 if (pt->per_cpu_mmaps &&
1324                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1325                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1326                     !pt->sampling_mode) {
1327                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1328                         if (pt->switch_ip) {
1329                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1330                                              pt->switch_ip, pt->ptss_ip);
1331                                 pt->sync_switch = true;
1332                         }
1333                 }
1334         }
1335
1336         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1337                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1338         while (1) {
1339                 err = intel_pt_sample(ptq);
1340                 if (err)
1341                         return err;
1342
1343                 state = intel_pt_decode(ptq->decoder);
1344                 if (state->err) {
1345                         if (state->err == INTEL_PT_ERR_NODATA)
1346                                 return 1;
1347                         if (pt->sync_switch &&
1348                             state->from_ip >= pt->kernel_start) {
1349                                 pt->sync_switch = false;
1350                                 intel_pt_next_tid(pt, ptq);
1351                         }
1352                         if (pt->synth_opts.errors) {
1353                                 err = intel_pt_synth_error(pt, state->err,
1354                                                            ptq->cpu, ptq->pid,
1355                                                            ptq->tid,
1356                                                            state->from_ip);
1357                                 if (err)
1358                                         return err;
1359                         }
1360                         continue;
1361                 }
1362
1363                 ptq->state = state;
1364                 ptq->have_sample = true;
1365                 intel_pt_sample_flags(ptq);
1366
1367                 /* Use estimated TSC upon return to user space */
1368                 if (pt->est_tsc &&
1369                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1370                     state->to_ip && state->to_ip < pt->kernel_start) {
1371                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1372                                      state->timestamp, state->est_timestamp);
1373                         ptq->timestamp = state->est_timestamp;
1374                 /* Use estimated TSC in unknown switch state */
1375                 } else if (pt->sync_switch &&
1376                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1377                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1378                            ptq->next_tid == -1) {
1379                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1380                                      state->timestamp, state->est_timestamp);
1381                         ptq->timestamp = state->est_timestamp;
1382                 } else if (state->timestamp > ptq->timestamp) {
1383                         ptq->timestamp = state->timestamp;
1384                 }
1385
1386                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1387                         *timestamp = ptq->timestamp;
1388                         return 0;
1389                 }
1390         }
1391         return 0;
1392 }
1393
1394 static inline int intel_pt_update_queues(struct intel_pt *pt)
1395 {
1396         if (pt->queues.new_data) {
1397                 pt->queues.new_data = false;
1398                 return intel_pt_setup_queues(pt);
1399         }
1400         return 0;
1401 }
1402
1403 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1404 {
1405         unsigned int queue_nr;
1406         u64 ts;
1407         int ret;
1408
1409         while (1) {
1410                 struct auxtrace_queue *queue;
1411                 struct intel_pt_queue *ptq;
1412
1413                 if (!pt->heap.heap_cnt)
1414                         return 0;
1415
1416                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1417                         return 0;
1418
1419                 queue_nr = pt->heap.heap_array[0].queue_nr;
1420                 queue = &pt->queues.queue_array[queue_nr];
1421                 ptq = queue->priv;
1422
1423                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1424                              queue_nr, pt->heap.heap_array[0].ordinal,
1425                              timestamp);
1426
1427                 auxtrace_heap__pop(&pt->heap);
1428
1429                 if (pt->heap.heap_cnt) {
1430                         ts = pt->heap.heap_array[0].ordinal + 1;
1431                         if (ts > timestamp)
1432                                 ts = timestamp;
1433                 } else {
1434                         ts = timestamp;
1435                 }
1436
1437                 intel_pt_set_pid_tid_cpu(pt, queue);
1438
1439                 ret = intel_pt_run_decoder(ptq, &ts);
1440
1441                 if (ret < 0) {
1442                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1443                         return ret;
1444                 }
1445
1446                 if (!ret) {
1447                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1448                         if (ret < 0)
1449                                 return ret;
1450                 } else {
1451                         ptq->on_heap = false;
1452                 }
1453         }
1454
1455         return 0;
1456 }
1457
1458 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1459                                             u64 time_)
1460 {
1461         struct auxtrace_queues *queues = &pt->queues;
1462         unsigned int i;
1463         u64 ts = 0;
1464
1465         for (i = 0; i < queues->nr_queues; i++) {
1466                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1467                 struct intel_pt_queue *ptq = queue->priv;
1468
1469                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1470                         ptq->time = time_;
1471                         intel_pt_set_pid_tid_cpu(pt, queue);
1472                         intel_pt_run_decoder(ptq, &ts);
1473                 }
1474         }
1475         return 0;
1476 }
1477
1478 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1479 {
1480         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1481                                     sample->pid, sample->tid, 0);
1482 }
1483
1484 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1485 {
1486         unsigned i, j;
1487
1488         if (cpu < 0 || !pt->queues.nr_queues)
1489                 return NULL;
1490
1491         if ((unsigned)cpu >= pt->queues.nr_queues)
1492                 i = pt->queues.nr_queues - 1;
1493         else
1494                 i = cpu;
1495
1496         if (pt->queues.queue_array[i].cpu == cpu)
1497                 return pt->queues.queue_array[i].priv;
1498
1499         for (j = 0; i > 0; j++) {
1500                 if (pt->queues.queue_array[--i].cpu == cpu)
1501                         return pt->queues.queue_array[i].priv;
1502         }
1503
1504         for (; j < pt->queues.nr_queues; j++) {
1505                 if (pt->queues.queue_array[j].cpu == cpu)
1506                         return pt->queues.queue_array[j].priv;
1507         }
1508
1509         return NULL;
1510 }
1511
1512 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1513                                 u64 timestamp)
1514 {
1515         struct intel_pt_queue *ptq;
1516         int err;
1517
1518         if (!pt->sync_switch)
1519                 return 1;
1520
1521         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1522         if (!ptq)
1523                 return 1;
1524
1525         switch (ptq->switch_state) {
1526         case INTEL_PT_SS_NOT_TRACING:
1527                 ptq->next_tid = -1;
1528                 break;
1529         case INTEL_PT_SS_UNKNOWN:
1530         case INTEL_PT_SS_TRACING:
1531                 ptq->next_tid = tid;
1532                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1533                 return 0;
1534         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1535                 if (!ptq->on_heap) {
1536                         ptq->timestamp = perf_time_to_tsc(timestamp,
1537                                                           &pt->tc);
1538                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1539                                                  ptq->timestamp);
1540                         if (err)
1541                                 return err;
1542                         ptq->on_heap = true;
1543                 }
1544                 ptq->switch_state = INTEL_PT_SS_TRACING;
1545                 break;
1546         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1547                 ptq->next_tid = tid;
1548                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1549                 break;
1550         default:
1551                 break;
1552         }
1553
1554         return 1;
1555 }
1556
1557 static int intel_pt_process_switch(struct intel_pt *pt,
1558                                    struct perf_sample *sample)
1559 {
1560         struct perf_evsel *evsel;
1561         pid_t tid;
1562         int cpu, ret;
1563
1564         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1565         if (evsel != pt->switch_evsel)
1566                 return 0;
1567
1568         tid = perf_evsel__intval(evsel, sample, "next_pid");
1569         cpu = sample->cpu;
1570
1571         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1572                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1573                      &pt->tc));
1574
1575         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1576         if (ret <= 0)
1577                 return ret;
1578
1579         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1580 }
1581
1582 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1583                                    struct perf_sample *sample)
1584 {
1585         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1586         pid_t pid, tid;
1587         int cpu, ret;
1588
1589         cpu = sample->cpu;
1590
1591         if (pt->have_sched_switch == 3) {
1592                 if (!out)
1593                         return 0;
1594                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1595                         pr_err("Expecting CPU-wide context switch event\n");
1596                         return -EINVAL;
1597                 }
1598                 pid = event->context_switch.next_prev_pid;
1599                 tid = event->context_switch.next_prev_tid;
1600         } else {
1601                 if (out)
1602                         return 0;
1603                 pid = sample->pid;
1604                 tid = sample->tid;
1605         }
1606
1607         if (tid == -1) {
1608                 pr_err("context_switch event has no tid\n");
1609                 return -EINVAL;
1610         }
1611
1612         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1613                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
1614                      &pt->tc));
1615
1616         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1617         if (ret <= 0)
1618                 return ret;
1619
1620         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1621 }
1622
1623 static int intel_pt_process_itrace_start(struct intel_pt *pt,
1624                                          union perf_event *event,
1625                                          struct perf_sample *sample)
1626 {
1627         if (!pt->per_cpu_mmaps)
1628                 return 0;
1629
1630         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1631                      sample->cpu, event->itrace_start.pid,
1632                      event->itrace_start.tid, sample->time,
1633                      perf_time_to_tsc(sample->time, &pt->tc));
1634
1635         return machine__set_current_tid(pt->machine, sample->cpu,
1636                                         event->itrace_start.pid,
1637                                         event->itrace_start.tid);
1638 }
1639
1640 static int intel_pt_process_event(struct perf_session *session,
1641                                   union perf_event *event,
1642                                   struct perf_sample *sample,
1643                                   struct perf_tool *tool)
1644 {
1645         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1646                                            auxtrace);
1647         u64 timestamp;
1648         int err = 0;
1649
1650         if (dump_trace)
1651                 return 0;
1652
1653         if (!tool->ordered_events) {
1654                 pr_err("Intel Processor Trace requires ordered events\n");
1655                 return -EINVAL;
1656         }
1657
1658         if (sample->time && sample->time != (u64)-1)
1659                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
1660         else
1661                 timestamp = 0;
1662
1663         if (timestamp || pt->timeless_decoding) {
1664                 err = intel_pt_update_queues(pt);
1665                 if (err)
1666                         return err;
1667         }
1668
1669         if (pt->timeless_decoding) {
1670                 if (event->header.type == PERF_RECORD_EXIT) {
1671                         err = intel_pt_process_timeless_queues(pt,
1672                                                                event->fork.tid,
1673                                                                sample->time);
1674                 }
1675         } else if (timestamp) {
1676                 err = intel_pt_process_queues(pt, timestamp);
1677         }
1678         if (err)
1679                 return err;
1680
1681         if (event->header.type == PERF_RECORD_AUX &&
1682             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
1683             pt->synth_opts.errors) {
1684                 err = intel_pt_lost(pt, sample);
1685                 if (err)
1686                         return err;
1687         }
1688
1689         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
1690                 err = intel_pt_process_switch(pt, sample);
1691         else if (event->header.type == PERF_RECORD_ITRACE_START)
1692                 err = intel_pt_process_itrace_start(pt, event, sample);
1693         else if (event->header.type == PERF_RECORD_SWITCH ||
1694                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
1695                 err = intel_pt_context_switch(pt, event, sample);
1696
1697         intel_pt_log("event %s (%u): cpu %d time %"PRIu64" tsc %#"PRIx64"\n",
1698                      perf_event__name(event->header.type), event->header.type,
1699                      sample->cpu, sample->time, timestamp);
1700
1701         return err;
1702 }
1703
1704 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
1705 {
1706         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1707                                            auxtrace);
1708         int ret;
1709
1710         if (dump_trace)
1711                 return 0;
1712
1713         if (!tool->ordered_events)
1714                 return -EINVAL;
1715
1716         ret = intel_pt_update_queues(pt);
1717         if (ret < 0)
1718                 return ret;
1719
1720         if (pt->timeless_decoding)
1721                 return intel_pt_process_timeless_queues(pt, -1,
1722                                                         MAX_TIMESTAMP - 1);
1723
1724         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
1725 }
1726
1727 static void intel_pt_free_events(struct perf_session *session)
1728 {
1729         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1730                                            auxtrace);
1731         struct auxtrace_queues *queues = &pt->queues;
1732         unsigned int i;
1733
1734         for (i = 0; i < queues->nr_queues; i++) {
1735                 intel_pt_free_queue(queues->queue_array[i].priv);
1736                 queues->queue_array[i].priv = NULL;
1737         }
1738         intel_pt_log_disable();
1739         auxtrace_queues__free(queues);
1740 }
1741
1742 static void intel_pt_free(struct perf_session *session)
1743 {
1744         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1745                                            auxtrace);
1746
1747         auxtrace_heap__free(&pt->heap);
1748         intel_pt_free_events(session);
1749         session->auxtrace = NULL;
1750         thread__put(pt->unknown_thread);
1751         free(pt);
1752 }
1753
1754 static int intel_pt_process_auxtrace_event(struct perf_session *session,
1755                                            union perf_event *event,
1756                                            struct perf_tool *tool __maybe_unused)
1757 {
1758         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
1759                                            auxtrace);
1760
1761         if (pt->sampling_mode)
1762                 return 0;
1763
1764         if (!pt->data_queued) {
1765                 struct auxtrace_buffer *buffer;
1766                 off_t data_offset;
1767                 int fd = perf_data_file__fd(session->file);
1768                 int err;
1769
1770                 if (perf_data_file__is_pipe(session->file)) {
1771                         data_offset = 0;
1772                 } else {
1773                         data_offset = lseek(fd, 0, SEEK_CUR);
1774                         if (data_offset == -1)
1775                                 return -errno;
1776                 }
1777
1778                 err = auxtrace_queues__add_event(&pt->queues, session, event,
1779                                                  data_offset, &buffer);
1780                 if (err)
1781                         return err;
1782
1783                 /* Dump here now we have copied a piped trace out of the pipe */
1784                 if (dump_trace) {
1785                         if (auxtrace_buffer__get_data(buffer, fd)) {
1786                                 intel_pt_dump_event(pt, buffer->data,
1787                                                     buffer->size);
1788                                 auxtrace_buffer__put_data(buffer);
1789                         }
1790                 }
1791         }
1792
1793         return 0;
1794 }
1795
1796 struct intel_pt_synth {
1797         struct perf_tool dummy_tool;
1798         struct perf_session *session;
1799 };
1800
1801 static int intel_pt_event_synth(struct perf_tool *tool,
1802                                 union perf_event *event,
1803                                 struct perf_sample *sample __maybe_unused,
1804                                 struct machine *machine __maybe_unused)
1805 {
1806         struct intel_pt_synth *intel_pt_synth =
1807                         container_of(tool, struct intel_pt_synth, dummy_tool);
1808
1809         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
1810                                                  NULL);
1811 }
1812
1813 static int intel_pt_synth_event(struct perf_session *session,
1814                                 struct perf_event_attr *attr, u64 id)
1815 {
1816         struct intel_pt_synth intel_pt_synth;
1817
1818         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
1819         intel_pt_synth.session = session;
1820
1821         return perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
1822                                            &id, intel_pt_event_synth);
1823 }
1824
1825 static int intel_pt_synth_events(struct intel_pt *pt,
1826                                  struct perf_session *session)
1827 {
1828         struct perf_evlist *evlist = session->evlist;
1829         struct perf_evsel *evsel;
1830         struct perf_event_attr attr;
1831         bool found = false;
1832         u64 id;
1833         int err;
1834
1835         evlist__for_each(evlist, evsel) {
1836                 if (evsel->attr.type == pt->pmu_type && evsel->ids) {
1837                         found = true;
1838                         break;
1839                 }
1840         }
1841
1842         if (!found) {
1843                 pr_debug("There are no selected events with Intel Processor Trace data\n");
1844                 return 0;
1845         }
1846
1847         memset(&attr, 0, sizeof(struct perf_event_attr));
1848         attr.size = sizeof(struct perf_event_attr);
1849         attr.type = PERF_TYPE_HARDWARE;
1850         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
1851         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
1852                             PERF_SAMPLE_PERIOD;
1853         if (pt->timeless_decoding)
1854                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
1855         else
1856                 attr.sample_type |= PERF_SAMPLE_TIME;
1857         if (!pt->per_cpu_mmaps)
1858                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
1859         attr.exclude_user = evsel->attr.exclude_user;
1860         attr.exclude_kernel = evsel->attr.exclude_kernel;
1861         attr.exclude_hv = evsel->attr.exclude_hv;
1862         attr.exclude_host = evsel->attr.exclude_host;
1863         attr.exclude_guest = evsel->attr.exclude_guest;
1864         attr.sample_id_all = evsel->attr.sample_id_all;
1865         attr.read_format = evsel->attr.read_format;
1866
1867         id = evsel->id[0] + 1000000000;
1868         if (!id)
1869                 id = 1;
1870
1871         if (pt->synth_opts.instructions) {
1872                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1873                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
1874                         attr.sample_period =
1875                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
1876                 else
1877                         attr.sample_period = pt->synth_opts.period;
1878                 pt->instructions_sample_period = attr.sample_period;
1879                 if (pt->synth_opts.callchain)
1880                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1881                 if (pt->synth_opts.last_branch)
1882                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1883                 pr_debug("Synthesizing 'instructions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1884                          id, (u64)attr.sample_type);
1885                 err = intel_pt_synth_event(session, &attr, id);
1886                 if (err) {
1887                         pr_err("%s: failed to synthesize 'instructions' event type\n",
1888                                __func__);
1889                         return err;
1890                 }
1891                 pt->sample_instructions = true;
1892                 pt->instructions_sample_type = attr.sample_type;
1893                 pt->instructions_id = id;
1894                 id += 1;
1895         }
1896
1897         if (pt->synth_opts.transactions) {
1898                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
1899                 attr.sample_period = 1;
1900                 if (pt->synth_opts.callchain)
1901                         attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
1902                 if (pt->synth_opts.last_branch)
1903                         attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
1904                 pr_debug("Synthesizing 'transactions' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1905                          id, (u64)attr.sample_type);
1906                 err = intel_pt_synth_event(session, &attr, id);
1907                 if (err) {
1908                         pr_err("%s: failed to synthesize 'transactions' event type\n",
1909                                __func__);
1910                         return err;
1911                 }
1912                 pt->sample_transactions = true;
1913                 pt->transactions_id = id;
1914                 id += 1;
1915                 evlist__for_each(evlist, evsel) {
1916                         if (evsel->id && evsel->id[0] == pt->transactions_id) {
1917                                 if (evsel->name)
1918                                         zfree(&evsel->name);
1919                                 evsel->name = strdup("transactions");
1920                                 break;
1921                         }
1922                 }
1923         }
1924
1925         if (pt->synth_opts.branches) {
1926                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
1927                 attr.sample_period = 1;
1928                 attr.sample_type |= PERF_SAMPLE_ADDR;
1929                 attr.sample_type &= ~(u64)PERF_SAMPLE_CALLCHAIN;
1930                 attr.sample_type &= ~(u64)PERF_SAMPLE_BRANCH_STACK;
1931                 pr_debug("Synthesizing 'branches' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
1932                          id, (u64)attr.sample_type);
1933                 err = intel_pt_synth_event(session, &attr, id);
1934                 if (err) {
1935                         pr_err("%s: failed to synthesize 'branches' event type\n",
1936                                __func__);
1937                         return err;
1938                 }
1939                 pt->sample_branches = true;
1940                 pt->branches_sample_type = attr.sample_type;
1941                 pt->branches_id = id;
1942         }
1943
1944         pt->synth_needs_swap = evsel->needs_swap;
1945
1946         return 0;
1947 }
1948
1949 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
1950 {
1951         struct perf_evsel *evsel;
1952
1953         evlist__for_each_reverse(evlist, evsel) {
1954                 const char *name = perf_evsel__name(evsel);
1955
1956                 if (!strcmp(name, "sched:sched_switch"))
1957                         return evsel;
1958         }
1959
1960         return NULL;
1961 }
1962
1963 static bool intel_pt_find_switch(struct perf_evlist *evlist)
1964 {
1965         struct perf_evsel *evsel;
1966
1967         evlist__for_each(evlist, evsel) {
1968                 if (evsel->attr.context_switch)
1969                         return true;
1970         }
1971
1972         return false;
1973 }
1974
1975 static int intel_pt_perf_config(const char *var, const char *value, void *data)
1976 {
1977         struct intel_pt *pt = data;
1978
1979         if (!strcmp(var, "intel-pt.mispred-all"))
1980                 pt->mispred_all = perf_config_bool(var, value);
1981
1982         return 0;
1983 }
1984
1985 static const char * const intel_pt_info_fmts[] = {
1986         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
1987         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
1988         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
1989         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
1990         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
1991         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
1992         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
1993         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
1994         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
1995         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
1996         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
1997         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
1998         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
1999         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
2000 };
2001
2002 static void intel_pt_print_info(u64 *arr, int start, int finish)
2003 {
2004         int i;
2005
2006         if (!dump_trace)
2007                 return;
2008
2009         for (i = start; i <= finish; i++)
2010                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2011 }
2012
2013 int intel_pt_process_auxtrace_info(union perf_event *event,
2014                                    struct perf_session *session)
2015 {
2016         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2017         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2018         struct intel_pt *pt;
2019         int err;
2020
2021         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2022                                         min_sz)
2023                 return -EINVAL;
2024
2025         pt = zalloc(sizeof(struct intel_pt));
2026         if (!pt)
2027                 return -ENOMEM;
2028
2029         perf_config(intel_pt_perf_config, pt);
2030
2031         err = auxtrace_queues__init(&pt->queues);
2032         if (err)
2033                 goto err_free;
2034
2035         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2036
2037         pt->session = session;
2038         pt->machine = &session->machines.host; /* No kvm support */
2039         pt->auxtrace_type = auxtrace_info->type;
2040         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2041         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2042         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2043         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2044         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2045         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2046         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2047         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2048         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2049         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2050         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2051                             INTEL_PT_PER_CPU_MMAPS);
2052
2053         if (auxtrace_info->header.size >= sizeof(struct auxtrace_info_event) +
2054                                         (sizeof(u64) * INTEL_PT_CYC_BIT)) {
2055                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2056                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2057                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2058                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2059                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2060                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2061                                     INTEL_PT_CYC_BIT);
2062         }
2063
2064         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2065         pt->have_tsc = intel_pt_have_tsc(pt);
2066         pt->sampling_mode = false;
2067         pt->est_tsc = !pt->timeless_decoding;
2068
2069         pt->unknown_thread = thread__new(999999999, 999999999);
2070         if (!pt->unknown_thread) {
2071                 err = -ENOMEM;
2072                 goto err_free_queues;
2073         }
2074
2075         /*
2076          * Since this thread will not be kept in any rbtree not in a
2077          * list, initialize its list node so that at thread__put() the
2078          * current thread lifetime assuption is kept and we don't segfault
2079          * at list_del_init().
2080          */
2081         INIT_LIST_HEAD(&pt->unknown_thread->node);
2082
2083         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2084         if (err)
2085                 goto err_delete_thread;
2086         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2087                 err = -ENOMEM;
2088                 goto err_delete_thread;
2089         }
2090
2091         pt->auxtrace.process_event = intel_pt_process_event;
2092         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2093         pt->auxtrace.flush_events = intel_pt_flush;
2094         pt->auxtrace.free_events = intel_pt_free_events;
2095         pt->auxtrace.free = intel_pt_free;
2096         session->auxtrace = &pt->auxtrace;
2097
2098         if (dump_trace)
2099                 return 0;
2100
2101         if (pt->have_sched_switch == 1) {
2102                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2103                 if (!pt->switch_evsel) {
2104                         pr_err("%s: missing sched_switch event\n", __func__);
2105                         goto err_delete_thread;
2106                 }
2107         } else if (pt->have_sched_switch == 2 &&
2108                    !intel_pt_find_switch(session->evlist)) {
2109                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2110                 goto err_delete_thread;
2111         }
2112
2113         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2114                 pt->synth_opts = *session->itrace_synth_opts;
2115         } else {
2116                 itrace_synth_opts__set_default(&pt->synth_opts);
2117                 if (use_browser != -1) {
2118                         pt->synth_opts.branches = false;
2119                         pt->synth_opts.callchain = true;
2120                 }
2121         }
2122
2123         if (pt->synth_opts.log)
2124                 intel_pt_log_enable();
2125
2126         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2127         if (pt->tc.time_mult) {
2128                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2129
2130                 pt->max_non_turbo_ratio = (tsc_freq + 50000000) / 100000000;
2131                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2132                 intel_pt_log("Maximum non-turbo ratio %u\n",
2133                              pt->max_non_turbo_ratio);
2134         }
2135
2136         if (pt->synth_opts.calls)
2137                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2138                                        PERF_IP_FLAG_TRACE_END;
2139         if (pt->synth_opts.returns)
2140                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2141                                        PERF_IP_FLAG_TRACE_BEGIN;
2142
2143         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2144                 symbol_conf.use_callchain = true;
2145                 if (callchain_register_param(&callchain_param) < 0) {
2146                         symbol_conf.use_callchain = false;
2147                         pt->synth_opts.callchain = false;
2148                 }
2149         }
2150
2151         err = intel_pt_synth_events(pt, session);
2152         if (err)
2153                 goto err_delete_thread;
2154
2155         err = auxtrace_queues__process_index(&pt->queues, session);
2156         if (err)
2157                 goto err_delete_thread;
2158
2159         if (pt->queues.populated)
2160                 pt->data_queued = true;
2161
2162         if (pt->timeless_decoding)
2163                 pr_debug2("Intel PT decoding without timestamps\n");
2164
2165         return 0;
2166
2167 err_delete_thread:
2168         thread__zput(pt->unknown_thread);
2169 err_free_queues:
2170         intel_pt_log_disable();
2171         auxtrace_queues__free(&pt->queues);
2172         session->auxtrace = NULL;
2173 err_free:
2174         free(pt);
2175         return err;
2176 }