tools/perf/util/intel-pt.c

   1 /*
   2  * intel_pt.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2015, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <inttypes.h>
  17 #include <stdio.h>
  18 #include <stdbool.h>
  19 #include <errno.h>
  20 #include <linux/kernel.h>
  21 #include <linux/types.h>
  22
  23 #include "../perf.h"
  24 #include "session.h"
  25 #include "machine.h"
  26 #include "memswap.h"
  27 #include "sort.h"
  28 #include "tool.h"
  29 #include "event.h"
  30 #include "evlist.h"
  31 #include "evsel.h"
  32 #include "map.h"
  33 #include "color.h"
  34 #include "util.h"
  35 #include "thread.h"
  36 #include "thread-stack.h"
  37 #include "symbol.h"
  38 #include "callchain.h"
  39 #include "dso.h"
  40 #include "debug.h"
  41 #include "auxtrace.h"
  42 #include "tsc.h"
  43 #include "intel-pt.h"
  44 #include "config.h"
  45
  46 #include "intel-pt-decoder/intel-pt-log.h"
  47 #include "intel-pt-decoder/intel-pt-decoder.h"
  48 #include "intel-pt-decoder/intel-pt-insn-decoder.h"
  49 #include "intel-pt-decoder/intel-pt-pkt-decoder.h"
  50
  51 #define MAX_TIMESTAMP (~0ULL)
  52
  53 struct intel_pt {
  54         struct auxtrace auxtrace;
  55         struct auxtrace_queues queues;
  56         struct auxtrace_heap heap;
  57         u32 auxtrace_type;
  58         struct perf_session *session;
  59         struct machine *machine;
  60         struct perf_evsel *switch_evsel;
  61         struct thread *unknown_thread;
  62         bool timeless_decoding;
  63         bool sampling_mode;
  64         bool snapshot_mode;
  65         bool per_cpu_mmaps;
  66         bool have_tsc;
  67         bool data_queued;
  68         bool est_tsc;
  69         bool sync_switch;
  70         bool mispred_all;
  71         int have_sched_switch;
  72         u32 pmu_type;
  73         u64 kernel_start;
  74         u64 switch_ip;
  75         u64 ptss_ip;
  76
  77         struct perf_tsc_conversion tc;
  78         bool cap_user_time_zero;
  79
  80         struct itrace_synth_opts synth_opts;
  81
  82         bool sample_instructions;
  83         u64 instructions_sample_type;
  84         u64 instructions_id;
  85
  86         bool sample_branches;
  87         u32 branches_filter;
  88         u64 branches_sample_type;
  89         u64 branches_id;
  90
  91         bool sample_transactions;
  92         u64 transactions_sample_type;
  93         u64 transactions_id;
  94
  95         bool sample_ptwrites;
  96         u64 ptwrites_sample_type;
  97         u64 ptwrites_id;
  98
  99         bool sample_pwr_events;
 100         u64 pwr_events_sample_type;
 101         u64 mwait_id;
 102         u64 pwre_id;
 103         u64 exstop_id;
 104         u64 pwrx_id;
 105         u64 cbr_id;
 106
 107         u64 tsc_bit;
 108         u64 mtc_bit;
 109         u64 mtc_freq_bits;
 110         u32 tsc_ctc_ratio_n;
 111         u32 tsc_ctc_ratio_d;
 112         u64 cyc_bit;
 113         u64 noretcomp_bit;
 114         unsigned max_non_turbo_ratio;
 115         unsigned cbr2khz;
 116
 117         unsigned long num_events;
 118
 119         char *filter;
 120         struct addr_filters filts;
 121 };
 122
 123 enum switch_state {
 124         INTEL_PT_SS_NOT_TRACING,
 125         INTEL_PT_SS_UNKNOWN,
 126         INTEL_PT_SS_TRACING,
 127         INTEL_PT_SS_EXPECTING_SWITCH_EVENT,
 128         INTEL_PT_SS_EXPECTING_SWITCH_IP,
 129 };
 130
 131 struct intel_pt_queue {
 132         struct intel_pt *pt;
 133         unsigned int queue_nr;
 134         struct auxtrace_buffer *buffer;
 135         struct auxtrace_buffer *old_buffer;
 136         void *decoder;
 137         const struct intel_pt_state *state;
 138         struct ip_callchain *chain;
 139         struct branch_stack *last_branch;
 140         struct branch_stack *last_branch_rb;
 141         size_t last_branch_pos;
 142         union perf_event *event_buf;
 143         bool on_heap;
 144         bool stop;
 145         bool step_through_buffers;
 146         bool use_buffer_pid_tid;
 147         bool sync_switch;
 148         pid_t pid, tid;
 149         int cpu;
 150         int switch_state;
 151         pid_t next_tid;
 152         struct thread *thread;
 153         bool exclude_kernel;
 154         bool have_sample;
 155         u64 time;
 156         u64 timestamp;
 157         u32 flags;
 158         u16 insn_len;
 159         u64 last_insn_cnt;
 160         u64 ipc_insn_cnt;
 161         u64 ipc_cyc_cnt;
 162         u64 last_in_insn_cnt;
 163         u64 last_in_cyc_cnt;
 164         u64 last_br_insn_cnt;
 165         u64 last_br_cyc_cnt;
 166         char insn[INTEL_PT_INSN_BUF_SZ];
 167 };
 168
 169 static void intel_pt_dump(struct intel_pt *pt __maybe_unused,
 170                           unsigned char *buf, size_t len)
 171 {
 172         struct intel_pt_pkt packet;
 173         size_t pos = 0;
 174         int ret, pkt_len, i;
 175         char desc[INTEL_PT_PKT_DESC_MAX];
 176         const char *color = PERF_COLOR_BLUE;
 177
 178         color_fprintf(stdout, color,
 179                       ". ... Intel Processor Trace data: size %zu bytes\n",
 180                       len);
 181
 182         while (len) {
 183                 ret = intel_pt_get_packet(buf, len, &packet);
 184                 if (ret > 0)
 185                         pkt_len = ret;
 186                 else
 187                         pkt_len = 1;
 188                 printf(".");
 189                 color_fprintf(stdout, color, "  %08x: ", pos);
 190                 for (i = 0; i < pkt_len; i++)
 191                         color_fprintf(stdout, color, " %02x", buf[i]);
 192                 for (; i < 16; i++)
 193                         color_fprintf(stdout, color, "   ");
 194                 if (ret > 0) {
 195                         ret = intel_pt_pkt_desc(&packet, desc,
 196                                                 INTEL_PT_PKT_DESC_MAX);
 197                         if (ret > 0)
 198                                 color_fprintf(stdout, color, " %s\n", desc);
 199                 } else {
 200                         color_fprintf(stdout, color, " Bad packet!\n");
 201                 }
 202                 pos += pkt_len;
 203                 buf += pkt_len;
 204                 len -= pkt_len;
 205         }
 206 }
 207
 208 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf,
 209                                 size_t len)
 210 {
 211         printf(".\n");
 212         intel_pt_dump(pt, buf, len);
 213 }
 214
 215 static void intel_pt_log_event(union perf_event *event)
 216 {
 217         FILE *f = intel_pt_log_fp();
 218
 219         if (!intel_pt_enable_logging || !f)
 220                 return;
 221
 222         perf_event__fprintf(event, f);
 223 }
 224
 225 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a,
 226                                    struct auxtrace_buffer *b)
 227 {
 228         bool consecutive = false;
 229         void *start;
 230
 231         start = intel_pt_find_overlap(a->data, a->size, b->data, b->size,
 232                                       pt->have_tsc, &consecutive);
 233         if (!start)
 234                 return -EINVAL;
 235         b->use_size = b->data + b->size - start;
 236         b->use_data = start;
 237         if (b->use_size && consecutive)
 238                 b->consecutive = true;
 239         return 0;
 240 }
 241
 242 /* This function assumes data is processed sequentially only */
 243 static int intel_pt_get_trace(struct intel_pt_buffer *b, void *data)
 244 {
 245         struct intel_pt_queue *ptq = data;
 246         struct auxtrace_buffer *buffer = ptq->buffer;
 247         struct auxtrace_buffer *old_buffer = ptq->old_buffer;
 248         struct auxtrace_queue *queue;
 249         bool might_overlap;
 250
 251         if (ptq->stop) {
 252                 b->len = 0;
 253                 return 0;
 254         }
 255
 256         queue = &ptq->pt->queues.queue_array[ptq->queue_nr];
 257
 258         buffer = auxtrace_buffer__next(queue, buffer);
 259         if (!buffer) {
 260                 if (old_buffer)
 261                         auxtrace_buffer__drop_data(old_buffer);
 262                 b->len = 0;
 263                 return 0;
 264         }
 265
 266         ptq->buffer = buffer;
 267
 268         if (!buffer->data) {
 269                 int fd = perf_data__fd(ptq->pt->session->data);
 270
 271                 buffer->data = auxtrace_buffer__get_data(buffer, fd);
 272                 if (!buffer->data)
 273                         return -ENOMEM;
 274         }
 275
 276         might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;
 277         if (might_overlap && !buffer->consecutive && old_buffer &&
 278             intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))
 279                 return -ENOMEM;
 280
 281         if (buffer->use_data) {
 282                 b->len = buffer->use_size;
 283                 b->buf = buffer->use_data;
 284         } else {
 285                 b->len = buffer->size;
 286                 b->buf = buffer->data;
 287         }
 288         b->ref_timestamp = buffer->reference;
 289
 290         if (!old_buffer || (might_overlap && !buffer->consecutive)) {
 291                 b->consecutive = false;
 292                 b->trace_nr = buffer->buffer_nr + 1;
 293         } else {
 294                 b->consecutive = true;
 295         }
 296
 297         if (ptq->step_through_buffers)
 298                 ptq->stop = true;
 299
 300         if (b->len) {
 301                 if (old_buffer)
 302                         auxtrace_buffer__drop_data(old_buffer);
 303                 ptq->old_buffer = buffer;
 304         } else {
 305                 auxtrace_buffer__drop_data(buffer);
 306                 return intel_pt_get_trace(b, data);
 307         }
 308
 309         return 0;
 310 }
 311
 312 struct intel_pt_cache_entry {
 313         struct auxtrace_cache_entry     entry;
 314         u64                             insn_cnt;
 315         u64                             byte_cnt;
 316         enum intel_pt_insn_op           op;
 317         enum intel_pt_insn_branch       branch;
 318         int                             length;
 319         int32_t                         rel;
 320         char                            insn[INTEL_PT_INSN_BUF_SZ];
 321 };
 322
 323 static int intel_pt_config_div(const char *var, const char *value, void *data)
 324 {
 325         int *d = data;
 326         long val;
 327
 328         if (!strcmp(var, "intel-pt.cache-divisor")) {
 329                 val = strtol(value, NULL, 0);
 330                 if (val > 0 && val <= INT_MAX)
 331                         *d = val;
 332         }
 333
 334         return 0;
 335 }
 336
 337 static int intel_pt_cache_divisor(void)
 338 {
 339         static int d;
 340
 341         if (d)
 342                 return d;
 343
 344         perf_config(intel_pt_config_div, &d);
 345
 346         if (!d)
 347                 d = 64;
 348
 349         return d;
 350 }
 351
 352 static unsigned int intel_pt_cache_size(struct dso *dso,
 353                                         struct machine *machine)
 354 {
 355         off_t size;
 356
 357         size = dso__data_size(dso, machine);
 358         size /= intel_pt_cache_divisor();
 359         if (size < 1000)
 360                 return 10;
 361         if (size > (1 << 21))
 362                 return 21;
 363         return 32 - __builtin_clz(size);
 364 }
 365
 366 static struct auxtrace_cache *intel_pt_cache(struct dso *dso,
 367                                              struct machine *machine)
 368 {
 369         struct auxtrace_cache *c;
 370         unsigned int bits;
 371
 372         if (dso->auxtrace_cache)
 373                 return dso->auxtrace_cache;
 374
 375         bits = intel_pt_cache_size(dso, machine);
 376
 377         /* Ignoring cache creation failure */
 378         c = auxtrace_cache__new(bits, sizeof(struct intel_pt_cache_entry), 200);
 379
 380         dso->auxtrace_cache = c;
 381
 382         return c;
 383 }
 384
 385 static int intel_pt_cache_add(struct dso *dso, struct machine *machine,
 386                               u64 offset, u64 insn_cnt, u64 byte_cnt,
 387                               struct intel_pt_insn *intel_pt_insn)
 388 {
 389         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 390         struct intel_pt_cache_entry *e;
 391         int err;
 392
 393         if (!c)
 394                 return -ENOMEM;
 395
 396         e = auxtrace_cache__alloc_entry(c);
 397         if (!e)
 398                 return -ENOMEM;
 399
 400         e->insn_cnt = insn_cnt;
 401         e->byte_cnt = byte_cnt;
 402         e->op = intel_pt_insn->op;
 403         e->branch = intel_pt_insn->branch;
 404         e->length = intel_pt_insn->length;
 405         e->rel = intel_pt_insn->rel;
 406         memcpy(e->insn, intel_pt_insn->buf, INTEL_PT_INSN_BUF_SZ);
 407
 408         err = auxtrace_cache__add(c, offset, &e->entry);
 409         if (err)
 410                 auxtrace_cache__free_entry(c, e);
 411
 412         return err;
 413 }
 414
 415 static struct intel_pt_cache_entry *
 416 intel_pt_cache_lookup(struct dso *dso, struct machine *machine, u64 offset)
 417 {
 418         struct auxtrace_cache *c = intel_pt_cache(dso, machine);
 419
 420         if (!c)
 421                 return NULL;
 422
 423         return auxtrace_cache__lookup(dso->auxtrace_cache, offset);
 424 }
 425
 426 static inline u8 intel_pt_cpumode(struct intel_pt *pt, uint64_t ip)
 427 {
 428         return ip >= pt->kernel_start ?
 429                PERF_RECORD_MISC_KERNEL :
 430                PERF_RECORD_MISC_USER;
 431 }
 432
 433 static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn,
 434                                    uint64_t *insn_cnt_ptr, uint64_t *ip,
 435                                    uint64_t to_ip, uint64_t max_insn_cnt,
 436                                    void *data)
 437 {
 438         struct intel_pt_queue *ptq = data;
 439         struct machine *machine = ptq->pt->machine;
 440         struct thread *thread;
 441         struct addr_location al;
 442         unsigned char buf[INTEL_PT_INSN_BUF_SZ];
 443         ssize_t len;
 444         int x86_64;
 445         u8 cpumode;
 446         u64 offset, start_offset, start_ip;
 447         u64 insn_cnt = 0;
 448         bool one_map = true;
 449
 450         intel_pt_insn->length = 0;
 451
 452         if (to_ip && *ip == to_ip)
 453                 goto out_no_cache;
 454
 455         cpumode = intel_pt_cpumode(ptq->pt, *ip);
 456
 457         thread = ptq->thread;
 458         if (!thread) {
 459                 if (cpumode != PERF_RECORD_MISC_KERNEL)
 460                         return -EINVAL;
 461                 thread = ptq->pt->unknown_thread;
 462         }
 463
 464         while (1) {
 465                 if (!thread__find_map(thread, cpumode, *ip, &al) || !al.map->dso)
 466                         return -EINVAL;
 467
 468                 if (al.map->dso->data.status == DSO_DATA_STATUS_ERROR &&
 469                     dso__data_status_seen(al.map->dso,
 470                                           DSO_DATA_STATUS_SEEN_ITRACE))
 471                         return -ENOENT;
 472
 473                 offset = al.map->map_ip(al.map, *ip);
 474
 475                 if (!to_ip && one_map) {
 476                         struct intel_pt_cache_entry *e;
 477
 478                         e = intel_pt_cache_lookup(al.map->dso, machine, offset);
 479                         if (e &&
 480                             (!max_insn_cnt || e->insn_cnt <= max_insn_cnt)) {
 481                                 *insn_cnt_ptr = e->insn_cnt;
 482                                 *ip += e->byte_cnt;
 483                                 intel_pt_insn->op = e->op;
 484                                 intel_pt_insn->branch = e->branch;
 485                                 intel_pt_insn->length = e->length;
 486                                 intel_pt_insn->rel = e->rel;
 487                                 memcpy(intel_pt_insn->buf, e->insn,
 488                                        INTEL_PT_INSN_BUF_SZ);
 489                                 intel_pt_log_insn_no_data(intel_pt_insn, *ip);
 490                                 return 0;
 491                         }
 492                 }
 493
 494                 start_offset = offset;
 495                 start_ip = *ip;
 496
 497                 /* Load maps to ensure dso->is_64_bit has been updated */
 498                 map__load(al.map);
 499
 500                 x86_64 = al.map->dso->is_64_bit;
 501
 502                 while (1) {
 503                         len = dso__data_read_offset(al.map->dso, machine,
 504                                                     offset, buf,
 505                                                     INTEL_PT_INSN_BUF_SZ);
 506                         if (len <= 0)
 507                                 return -EINVAL;
 508
 509                         if (intel_pt_get_insn(buf, len, x86_64, intel_pt_insn))
 510                                 return -EINVAL;
 511
 512                         intel_pt_log_insn(intel_pt_insn, *ip);
 513
 514                         insn_cnt += 1;
 515
 516                         if (intel_pt_insn->branch != INTEL_PT_BR_NO_BRANCH)
 517                                 goto out;
 518
 519                         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 520                                 goto out_no_cache;
 521
 522                         *ip += intel_pt_insn->length;
 523
 524                         if (to_ip && *ip == to_ip)
 525                                 goto out_no_cache;
 526
 527                         if (*ip >= al.map->end)
 528                                 break;
 529
 530                         offset += intel_pt_insn->length;
 531                 }
 532                 one_map = false;
 533         }
 534 out:
 535         *insn_cnt_ptr = insn_cnt;
 536
 537         if (!one_map)
 538                 goto out_no_cache;
 539
 540         /*
 541          * Didn't lookup in the 'to_ip' case, so do it now to prevent duplicate
 542          * entries.
 543          */
 544         if (to_ip) {
 545                 struct intel_pt_cache_entry *e;
 546
 547                 e = intel_pt_cache_lookup(al.map->dso, machine, start_offset);
 548                 if (e)
 549                         return 0;
 550         }
 551
 552         /* Ignore cache errors */
 553         intel_pt_cache_add(al.map->dso, machine, start_offset, insn_cnt,
 554                            *ip - start_ip, intel_pt_insn);
 555
 556         return 0;
 557
 558 out_no_cache:
 559         *insn_cnt_ptr = insn_cnt;
 560         return 0;
 561 }
 562
 563 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip,
 564                                   uint64_t offset, const char *filename)
 565 {
 566         struct addr_filter *filt;
 567         bool have_filter   = false;
 568         bool hit_tracestop = false;
 569         bool hit_filter    = false;
 570
 571         list_for_each_entry(filt, &pt->filts.head, list) {
 572                 if (filt->start)
 573                         have_filter = true;
 574
 575                 if ((filename && !filt->filename) ||
 576                     (!filename && filt->filename) ||
 577                     (filename && strcmp(filename, filt->filename)))
 578                         continue;
 579
 580                 if (!(offset >= filt->addr && offset < filt->addr + filt->size))
 581                         continue;
 582
 583                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s hit filter: %s offset %#"PRIx64" size %#"PRIx64"\n",
 584                              ip, offset, filename ? filename : "[kernel]",
 585                              filt->start ? "filter" : "stop",
 586                              filt->addr, filt->size);
 587
 588                 if (filt->start)
 589                         hit_filter = true;
 590                 else
 591                         hit_tracestop = true;
 592         }
 593
 594         if (!hit_tracestop && !hit_filter)
 595                 intel_pt_log("TIP.PGD ip %#"PRIx64" offset %#"PRIx64" in %s is not in a filter region\n",
 596                              ip, offset, filename ? filename : "[kernel]");
 597
 598         return hit_tracestop || (have_filter && !hit_filter);
 599 }
 600
 601 static int __intel_pt_pgd_ip(uint64_t ip, void *data)
 602 {
 603         struct intel_pt_queue *ptq = data;
 604         struct thread *thread;
 605         struct addr_location al;
 606         u8 cpumode;
 607         u64 offset;
 608
 609         if (ip >= ptq->pt->kernel_start)
 610                 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);
 611
 612         cpumode = PERF_RECORD_MISC_USER;
 613
 614         thread = ptq->thread;
 615         if (!thread)
 616                 return -EINVAL;
 617
 618         if (!thread__find_map(thread, cpumode, ip, &al) || !al.map->dso)
 619                 return -EINVAL;
 620
 621         offset = al.map->map_ip(al.map, ip);
 622
 623         return intel_pt_match_pgd_ip(ptq->pt, ip, offset,
 624                                      al.map->dso->long_name);
 625 }
 626
 627 static bool intel_pt_pgd_ip(uint64_t ip, void *data)
 628 {
 629         return __intel_pt_pgd_ip(ip, data) > 0;
 630 }
 631
 632 static bool intel_pt_get_config(struct intel_pt *pt,
 633                                 struct perf_event_attr *attr, u64 *config)
 634 {
 635         if (attr->type == pt->pmu_type) {
 636                 if (config)
 637                         *config = attr->config;
 638                 return true;
 639         }
 640
 641         return false;
 642 }
 643
 644 static bool intel_pt_exclude_kernel(struct intel_pt *pt)
 645 {
 646         struct perf_evsel *evsel;
 647
 648         evlist__for_each_entry(pt->session->evlist, evsel) {
 649                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 650                     !evsel->attr.exclude_kernel)
 651                         return false;
 652         }
 653         return true;
 654 }
 655
 656 static bool intel_pt_return_compression(struct intel_pt *pt)
 657 {
 658         struct perf_evsel *evsel;
 659         u64 config;
 660
 661         if (!pt->noretcomp_bit)
 662                 return true;
 663
 664         evlist__for_each_entry(pt->session->evlist, evsel) {
 665                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 666                     (config & pt->noretcomp_bit))
 667                         return false;
 668         }
 669         return true;
 670 }
 671
 672 static bool intel_pt_branch_enable(struct intel_pt *pt)
 673 {
 674         struct perf_evsel *evsel;
 675         u64 config;
 676
 677         evlist__for_each_entry(pt->session->evlist, evsel) {
 678                 if (intel_pt_get_config(pt, &evsel->attr, &config) &&
 679                     (config & 1) && !(config & 0x2000))
 680                         return false;
 681         }
 682         return true;
 683 }
 684
 685 static unsigned int intel_pt_mtc_period(struct intel_pt *pt)
 686 {
 687         struct perf_evsel *evsel;
 688         unsigned int shift;
 689         u64 config;
 690
 691         if (!pt->mtc_freq_bits)
 692                 return 0;
 693
 694         for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++)
 695                 config >>= 1;
 696
 697         evlist__for_each_entry(pt->session->evlist, evsel) {
 698                 if (intel_pt_get_config(pt, &evsel->attr, &config))
 699                         return (config & pt->mtc_freq_bits) >> shift;
 700         }
 701         return 0;
 702 }
 703
 704 static bool intel_pt_timeless_decoding(struct intel_pt *pt)
 705 {
 706         struct perf_evsel *evsel;
 707         bool timeless_decoding = true;
 708         u64 config;
 709
 710         if (!pt->tsc_bit || !pt->cap_user_time_zero)
 711                 return true;
 712
 713         evlist__for_each_entry(pt->session->evlist, evsel) {
 714                 if (!(evsel->attr.sample_type & PERF_SAMPLE_TIME))
 715                         return true;
 716                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 717                         if (config & pt->tsc_bit)
 718                                 timeless_decoding = false;
 719                         else
 720                                 return true;
 721                 }
 722         }
 723         return timeless_decoding;
 724 }
 725
 726 static bool intel_pt_tracing_kernel(struct intel_pt *pt)
 727 {
 728         struct perf_evsel *evsel;
 729
 730         evlist__for_each_entry(pt->session->evlist, evsel) {
 731                 if (intel_pt_get_config(pt, &evsel->attr, NULL) &&
 732                     !evsel->attr.exclude_kernel)
 733                         return true;
 734         }
 735         return false;
 736 }
 737
 738 static bool intel_pt_have_tsc(struct intel_pt *pt)
 739 {
 740         struct perf_evsel *evsel;
 741         bool have_tsc = false;
 742         u64 config;
 743
 744         if (!pt->tsc_bit)
 745                 return false;
 746
 747         evlist__for_each_entry(pt->session->evlist, evsel) {
 748                 if (intel_pt_get_config(pt, &evsel->attr, &config)) {
 749                         if (config & pt->tsc_bit)
 750                                 have_tsc = true;
 751                         else
 752                                 return false;
 753                 }
 754         }
 755         return have_tsc;
 756 }
 757
 758 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns)
 759 {
 760         u64 quot, rem;
 761
 762         quot = ns / pt->tc.time_mult;
 763         rem  = ns % pt->tc.time_mult;
 764         return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) /
 765                 pt->tc.time_mult;
 766 }
 767
 768 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt,
 769                                                    unsigned int queue_nr)
 770 {
 771         struct intel_pt_params params = { .get_trace = 0, };
 772         struct perf_env *env = pt->machine->env;
 773         struct intel_pt_queue *ptq;
 774
 775         ptq = zalloc(sizeof(struct intel_pt_queue));
 776         if (!ptq)
 777                 return NULL;
 778
 779         if (pt->synth_opts.callchain) {
 780                 size_t sz = sizeof(struct ip_callchain);
 781
 782                 /* Add 1 to callchain_sz for callchain context */
 783                 sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64);
 784                 ptq->chain = zalloc(sz);
 785                 if (!ptq->chain)
 786                         goto out_free;
 787         }
 788
 789         if (pt->synth_opts.last_branch) {
 790                 size_t sz = sizeof(struct branch_stack);
 791
 792                 sz += pt->synth_opts.last_branch_sz *
 793                       sizeof(struct branch_entry);
 794                 ptq->last_branch = zalloc(sz);
 795                 if (!ptq->last_branch)
 796                         goto out_free;
 797                 ptq->last_branch_rb = zalloc(sz);
 798                 if (!ptq->last_branch_rb)
 799                         goto out_free;
 800         }
 801
 802         ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);
 803         if (!ptq->event_buf)
 804                 goto out_free;
 805
 806         ptq->pt = pt;
 807         ptq->queue_nr = queue_nr;
 808         ptq->exclude_kernel = intel_pt_exclude_kernel(pt);
 809         ptq->pid = -1;
 810         ptq->tid = -1;
 811         ptq->cpu = -1;
 812         ptq->next_tid = -1;
 813
 814         params.get_trace = intel_pt_get_trace;
 815         params.walk_insn = intel_pt_walk_next_insn;
 816         params.data = ptq;
 817         params.return_compression = intel_pt_return_compression(pt);
 818         params.branch_enable = intel_pt_branch_enable(pt);
 819         params.max_non_turbo_ratio = pt->max_non_turbo_ratio;
 820         params.mtc_period = intel_pt_mtc_period(pt);
 821         params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n;
 822         params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d;
 823
 824         if (pt->filts.cnt > 0)
 825                 params.pgd_ip = intel_pt_pgd_ip;
 826
 827         if (pt->synth_opts.instructions) {
 828                 if (pt->synth_opts.period) {
 829                         switch (pt->synth_opts.period_type) {
 830                         case PERF_ITRACE_PERIOD_INSTRUCTIONS:
 831                                 params.period_type =
 832                                                 INTEL_PT_PERIOD_INSTRUCTIONS;
 833                                 params.period = pt->synth_opts.period;
 834                                 break;
 835                         case PERF_ITRACE_PERIOD_TICKS:
 836                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 837                                 params.period = pt->synth_opts.period;
 838                                 break;
 839                         case PERF_ITRACE_PERIOD_NANOSECS:
 840                                 params.period_type = INTEL_PT_PERIOD_TICKS;
 841                                 params.period = intel_pt_ns_to_ticks(pt,
 842                                                         pt->synth_opts.period);
 843                                 break;
 844                         default:
 845                                 break;
 846                         }
 847                 }
 848
 849                 if (!params.period) {
 850                         params.period_type = INTEL_PT_PERIOD_INSTRUCTIONS;
 851                         params.period = 1;
 852                 }
 853         }
 854
 855         if (env->cpuid && !strncmp(env->cpuid, "GenuineIntel,6,92,", 18))
 856                 params.flags |= INTEL_PT_FUP_WITH_NLIP;
 857
 858         ptq->decoder = intel_pt_decoder_new(&params);
 859         if (!ptq->decoder)
 860                 goto out_free;
 861
 862         return ptq;
 863
 864 out_free:
 865         zfree(&ptq->event_buf);
 866         zfree(&ptq->last_branch);
 867         zfree(&ptq->last_branch_rb);
 868         zfree(&ptq->chain);
 869         free(ptq);
 870         return NULL;
 871 }
 872
 873 static void intel_pt_free_queue(void *priv)
 874 {
 875         struct intel_pt_queue *ptq = priv;
 876
 877         if (!ptq)
 878                 return;
 879         thread__zput(ptq->thread);
 880         intel_pt_decoder_free(ptq->decoder);
 881         zfree(&ptq->event_buf);
 882         zfree(&ptq->last_branch);
 883         zfree(&ptq->last_branch_rb);
 884         zfree(&ptq->chain);
 885         free(ptq);
 886 }
 887
 888 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt,
 889                                      struct auxtrace_queue *queue)
 890 {
 891         struct intel_pt_queue *ptq = queue->priv;
 892
 893         if (queue->tid == -1 || pt->have_sched_switch) {
 894                 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);
 895                 thread__zput(ptq->thread);
 896         }
 897
 898         if (!ptq->thread && ptq->tid != -1)
 899                 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);
 900
 901         if (ptq->thread) {
 902                 ptq->pid = ptq->thread->pid_;
 903                 if (queue->cpu == -1)
 904                         ptq->cpu = ptq->thread->cpu;
 905         }
 906 }
 907
 908 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)
 909 {
 910         if (ptq->state->flags & INTEL_PT_ABORT_TX) {
 911                 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;
 912         } else if (ptq->state->flags & INTEL_PT_ASYNC) {
 913                 if (ptq->state->to_ip)
 914                         ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |
 915                                      PERF_IP_FLAG_ASYNC |
 916                                      PERF_IP_FLAG_INTERRUPT;
 917                 else
 918                         ptq->flags = PERF_IP_FLAG_BRANCH |
 919                                      PERF_IP_FLAG_TRACE_END;
 920                 ptq->insn_len = 0;
 921         } else {
 922                 if (ptq->state->from_ip)
 923                         ptq->flags = intel_pt_insn_type(ptq->state->insn_op);
 924                 else
 925                         ptq->flags = PERF_IP_FLAG_BRANCH |
 926                                      PERF_IP_FLAG_TRACE_BEGIN;
 927                 if (ptq->state->flags & INTEL_PT_IN_TX)
 928                         ptq->flags |= PERF_IP_FLAG_IN_TX;
 929                 ptq->insn_len = ptq->state->insn_len;
 930                 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);
 931         }
 932
 933         if (ptq->state->type & INTEL_PT_TRACE_BEGIN)
 934                 ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN;
 935         if (ptq->state->type & INTEL_PT_TRACE_END)
 936                 ptq->flags |= PERF_IP_FLAG_TRACE_END;
 937 }
 938
 939 static int intel_pt_setup_queue(struct intel_pt *pt,
 940                                 struct auxtrace_queue *queue,
 941                                 unsigned int queue_nr)
 942 {
 943         struct intel_pt_queue *ptq = queue->priv;
 944
 945         if (list_empty(&queue->head))
 946                 return 0;
 947
 948         if (!ptq) {
 949                 ptq = intel_pt_alloc_queue(pt, queue_nr);
 950                 if (!ptq)
 951                         return -ENOMEM;
 952                 queue->priv = ptq;
 953
 954                 if (queue->cpu != -1)
 955                         ptq->cpu = queue->cpu;
 956                 ptq->tid = queue->tid;
 957
 958                 if (pt->sampling_mode && !pt->snapshot_mode &&
 959                     pt->timeless_decoding)
 960                         ptq->step_through_buffers = true;
 961
 962                 ptq->sync_switch = pt->sync_switch;
 963         }
 964
 965         if (!ptq->on_heap &&
 966             (!ptq->sync_switch ||
 967              ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {
 968                 const struct intel_pt_state *state;
 969                 int ret;
 970
 971                 if (pt->timeless_decoding)
 972                         return 0;
 973
 974                 intel_pt_log("queue %u getting timestamp\n", queue_nr);
 975                 intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
 976                              queue_nr, ptq->cpu, ptq->pid, ptq->tid);
 977                 while (1) {
 978                         state = intel_pt_decode(ptq->decoder);
 979                         if (state->err) {
 980                                 if (state->err == INTEL_PT_ERR_NODATA) {
 981                                         intel_pt_log("queue %u has no timestamp\n",
 982                                                      queue_nr);
 983                                         return 0;
 984                                 }
 985                                 continue;
 986                         }
 987                         if (state->timestamp)
 988                                 break;
 989                 }
 990
 991                 ptq->timestamp = state->timestamp;
 992                 intel_pt_log("queue %u timestamp 0x%" PRIx64 "\n",
 993                              queue_nr, ptq->timestamp);
 994                 ptq->state = state;
 995                 ptq->have_sample = true;
 996                 intel_pt_sample_flags(ptq);
 997                 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);
 998                 if (ret)
 999                         return ret;
1000                 ptq->on_heap = true;
1001         }
1002
1003         return 0;
1004 }
1005
1006 static int intel_pt_setup_queues(struct intel_pt *pt)
1007 {
1008         unsigned int i;
1009         int ret;
1010
1011         for (i = 0; i < pt->queues.nr_queues; i++) {
1012                 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i);
1013                 if (ret)
1014                         return ret;
1015         }
1016         return 0;
1017 }
1018
1019 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)
1020 {
1021         struct branch_stack *bs_src = ptq->last_branch_rb;
1022         struct branch_stack *bs_dst = ptq->last_branch;
1023         size_t nr = 0;
1024
1025         bs_dst->nr = bs_src->nr;
1026
1027         if (!bs_src->nr)
1028                 return;
1029
1030         nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;
1031         memcpy(&bs_dst->entries[0],
1032                &bs_src->entries[ptq->last_branch_pos],
1033                sizeof(struct branch_entry) * nr);
1034
1035         if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {
1036                 memcpy(&bs_dst->entries[nr],
1037                        &bs_src->entries[0],
1038                        sizeof(struct branch_entry) * ptq->last_branch_pos);
1039         }
1040 }
1041
1042 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)
1043 {
1044         ptq->last_branch_pos = 0;
1045         ptq->last_branch_rb->nr = 0;
1046 }
1047
1048 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)
1049 {
1050         const struct intel_pt_state *state = ptq->state;
1051         struct branch_stack *bs = ptq->last_branch_rb;
1052         struct branch_entry *be;
1053
1054         if (!ptq->last_branch_pos)
1055                 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;
1056
1057         ptq->last_branch_pos -= 1;
1058
1059         be              = &bs->entries[ptq->last_branch_pos];
1060         be->from        = state->from_ip;
1061         be->to          = state->to_ip;
1062         be->flags.abort = !!(state->flags & INTEL_PT_ABORT_TX);
1063         be->flags.in_tx = !!(state->flags & INTEL_PT_IN_TX);
1064         /* No support for mispredict */
1065         be->flags.mispred = ptq->pt->mispred_all;
1066
1067         if (bs->nr < ptq->pt->synth_opts.last_branch_sz)
1068                 bs->nr += 1;
1069 }
1070
1071 static inline bool intel_pt_skip_event(struct intel_pt *pt)
1072 {
1073         return pt->synth_opts.initial_skip &&
1074                pt->num_events++ < pt->synth_opts.initial_skip;
1075 }
1076
1077 static void intel_pt_prep_b_sample(struct intel_pt *pt,
1078                                    struct intel_pt_queue *ptq,
1079                                    union perf_event *event,
1080                                    struct perf_sample *sample)
1081 {
1082         if (!pt->timeless_decoding)
1083                 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);
1084
1085         sample->ip = ptq->state->from_ip;
1086         sample->cpumode = intel_pt_cpumode(pt, sample->ip);
1087         sample->pid = ptq->pid;
1088         sample->tid = ptq->tid;
1089         sample->addr = ptq->state->to_ip;
1090         sample->period = 1;
1091         sample->cpu = ptq->cpu;
1092         sample->flags = ptq->flags;
1093         sample->insn_len = ptq->insn_len;
1094         memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);
1095
1096         event->sample.header.type = PERF_RECORD_SAMPLE;
1097         event->sample.header.misc = sample->cpumode;
1098         event->sample.header.size = sizeof(struct perf_event_header);
1099 }
1100
1101 static int intel_pt_inject_event(union perf_event *event,
1102                                  struct perf_sample *sample, u64 type)
1103 {
1104         event->header.size = perf_event__sample_event_size(sample, type, 0);
1105         return perf_event__synthesize_sample(event, type, 0, sample);
1106 }
1107
1108 static inline int intel_pt_opt_inject(struct intel_pt *pt,
1109                                       union perf_event *event,
1110                                       struct perf_sample *sample, u64 type)
1111 {
1112         if (!pt->synth_opts.inject)
1113                 return 0;
1114
1115         return intel_pt_inject_event(event, sample, type);
1116 }
1117
1118 static int intel_pt_deliver_synth_b_event(struct intel_pt *pt,
1119                                           union perf_event *event,
1120                                           struct perf_sample *sample, u64 type)
1121 {
1122         int ret;
1123
1124         ret = intel_pt_opt_inject(pt, event, sample, type);
1125         if (ret)
1126                 return ret;
1127
1128         ret = perf_session__deliver_synth_event(pt->session, event, sample);
1129         if (ret)
1130                 pr_err("Intel PT: failed to deliver event, error %d\n", ret);
1131
1132         return ret;
1133 }
1134
1135 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)
1136 {
1137         struct intel_pt *pt = ptq->pt;
1138         union perf_event *event = ptq->event_buf;
1139         struct perf_sample sample = { .ip = 0, };
1140         struct dummy_branch_stack {
1141                 u64                     nr;
1142                 struct branch_entry     entries;
1143         } dummy_bs;
1144
1145         if (pt->branches_filter && !(pt->branches_filter & ptq->flags))
1146                 return 0;
1147
1148         if (intel_pt_skip_event(pt))
1149                 return 0;
1150
1151         intel_pt_prep_b_sample(pt, ptq, event, &sample);
1152
1153         sample.id = ptq->pt->branches_id;
1154         sample.stream_id = ptq->pt->branches_id;
1155
1156         /*
1157          * perf report cannot handle events without a branch stack when using
1158          * SORT_MODE__BRANCH so make a dummy one.
1159          */
1160         if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) {
1161                 dummy_bs = (struct dummy_branch_stack){
1162                         .nr = 1,
1163                         .entries = {
1164                                 .from = sample.ip,
1165                                 .to = sample.addr,
1166                         },
1167                 };
1168                 sample.branch_stack = (struct branch_stack *)&dummy_bs;
1169         }
1170
1171         sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;
1172         if (sample.cyc_cnt) {
1173                 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;
1174                 ptq->last_br_insn_cnt = ptq->ipc_insn_cnt;
1175                 ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt;
1176         }
1177
1178         return intel_pt_deliver_synth_b_event(pt, event, &sample,
1179                                               pt->branches_sample_type);
1180 }
1181
1182 static void intel_pt_prep_sample(struct intel_pt *pt,
1183                                  struct intel_pt_queue *ptq,
1184                                  union perf_event *event,
1185                                  struct perf_sample *sample)
1186 {
1187         intel_pt_prep_b_sample(pt, ptq, event, sample);
1188
1189         if (pt->synth_opts.callchain) {
1190                 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain,
1191                                      pt->synth_opts.callchain_sz + 1,
1192                                      sample->ip, pt->kernel_start);
1193                 sample->callchain = ptq->chain;
1194         }
1195
1196         if (pt->synth_opts.last_branch) {
1197                 intel_pt_copy_last_branch_rb(ptq);
1198                 sample->branch_stack = ptq->last_branch;
1199         }
1200 }
1201
1202 static inline int intel_pt_deliver_synth_event(struct intel_pt *pt,
1203                                                struct intel_pt_queue *ptq,
1204                                                union perf_event *event,
1205                                                struct perf_sample *sample,
1206                                                u64 type)
1207 {
1208         int ret;
1209
1210         ret = intel_pt_deliver_synth_b_event(pt, event, sample, type);
1211
1212         if (pt->synth_opts.last_branch)
1213                 intel_pt_reset_last_branch_rb(ptq);
1214
1215         return ret;
1216 }
1217
1218 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)
1219 {
1220         struct intel_pt *pt = ptq->pt;
1221         union perf_event *event = ptq->event_buf;
1222         struct perf_sample sample = { .ip = 0, };
1223
1224         if (intel_pt_skip_event(pt))
1225                 return 0;
1226
1227         intel_pt_prep_sample(pt, ptq, event, &sample);
1228
1229         sample.id = ptq->pt->instructions_id;
1230         sample.stream_id = ptq->pt->instructions_id;
1231         sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;
1232
1233         sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;
1234         if (sample.cyc_cnt) {
1235                 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;
1236                 ptq->last_in_insn_cnt = ptq->ipc_insn_cnt;
1237                 ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt;
1238         }
1239
1240         ptq->last_insn_cnt = ptq->state->tot_insn_cnt;
1241
1242         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1243                                             pt->instructions_sample_type);
1244 }
1245
1246 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)
1247 {
1248         struct intel_pt *pt = ptq->pt;
1249         union perf_event *event = ptq->event_buf;
1250         struct perf_sample sample = { .ip = 0, };
1251
1252         if (intel_pt_skip_event(pt))
1253                 return 0;
1254
1255         intel_pt_prep_sample(pt, ptq, event, &sample);
1256
1257         sample.id = ptq->pt->transactions_id;
1258         sample.stream_id = ptq->pt->transactions_id;
1259
1260         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1261                                             pt->transactions_sample_type);
1262 }
1263
1264 static void intel_pt_prep_p_sample(struct intel_pt *pt,
1265                                    struct intel_pt_queue *ptq,
1266                                    union perf_event *event,
1267                                    struct perf_sample *sample)
1268 {
1269         intel_pt_prep_sample(pt, ptq, event, sample);
1270
1271         /*
1272          * Zero IP is used to mean "trace start" but that is not the case for
1273          * power or PTWRITE events with no IP, so clear the flags.
1274          */
1275         if (!sample->ip)
1276                 sample->flags = 0;
1277 }
1278
1279 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)
1280 {
1281         struct intel_pt *pt = ptq->pt;
1282         union perf_event *event = ptq->event_buf;
1283         struct perf_sample sample = { .ip = 0, };
1284         struct perf_synth_intel_ptwrite raw;
1285
1286         if (intel_pt_skip_event(pt))
1287                 return 0;
1288
1289         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1290
1291         sample.id = ptq->pt->ptwrites_id;
1292         sample.stream_id = ptq->pt->ptwrites_id;
1293
1294         raw.flags = 0;
1295         raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1296         raw.payload = cpu_to_le64(ptq->state->ptw_payload);
1297
1298         sample.raw_size = perf_synth__raw_size(raw);
1299         sample.raw_data = perf_synth__raw_data(&raw);
1300
1301         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1302                                             pt->ptwrites_sample_type);
1303 }
1304
1305 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)
1306 {
1307         struct intel_pt *pt = ptq->pt;
1308         union perf_event *event = ptq->event_buf;
1309         struct perf_sample sample = { .ip = 0, };
1310         struct perf_synth_intel_cbr raw;
1311         u32 flags;
1312
1313         if (intel_pt_skip_event(pt))
1314                 return 0;
1315
1316         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1317
1318         sample.id = ptq->pt->cbr_id;
1319         sample.stream_id = ptq->pt->cbr_id;
1320
1321         flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);
1322         raw.flags = cpu_to_le32(flags);
1323         raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz);
1324         raw.reserved3 = 0;
1325
1326         sample.raw_size = perf_synth__raw_size(raw);
1327         sample.raw_data = perf_synth__raw_data(&raw);
1328
1329         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1330                                             pt->pwr_events_sample_type);
1331 }
1332
1333 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)
1334 {
1335         struct intel_pt *pt = ptq->pt;
1336         union perf_event *event = ptq->event_buf;
1337         struct perf_sample sample = { .ip = 0, };
1338         struct perf_synth_intel_mwait raw;
1339
1340         if (intel_pt_skip_event(pt))
1341                 return 0;
1342
1343         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1344
1345         sample.id = ptq->pt->mwait_id;
1346         sample.stream_id = ptq->pt->mwait_id;
1347
1348         raw.reserved = 0;
1349         raw.payload = cpu_to_le64(ptq->state->mwait_payload);
1350
1351         sample.raw_size = perf_synth__raw_size(raw);
1352         sample.raw_data = perf_synth__raw_data(&raw);
1353
1354         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1355                                             pt->pwr_events_sample_type);
1356 }
1357
1358 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)
1359 {
1360         struct intel_pt *pt = ptq->pt;
1361         union perf_event *event = ptq->event_buf;
1362         struct perf_sample sample = { .ip = 0, };
1363         struct perf_synth_intel_pwre raw;
1364
1365         if (intel_pt_skip_event(pt))
1366                 return 0;
1367
1368         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1369
1370         sample.id = ptq->pt->pwre_id;
1371         sample.stream_id = ptq->pt->pwre_id;
1372
1373         raw.reserved = 0;
1374         raw.payload = cpu_to_le64(ptq->state->pwre_payload);
1375
1376         sample.raw_size = perf_synth__raw_size(raw);
1377         sample.raw_data = perf_synth__raw_data(&raw);
1378
1379         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1380                                             pt->pwr_events_sample_type);
1381 }
1382
1383 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)
1384 {
1385         struct intel_pt *pt = ptq->pt;
1386         union perf_event *event = ptq->event_buf;
1387         struct perf_sample sample = { .ip = 0, };
1388         struct perf_synth_intel_exstop raw;
1389
1390         if (intel_pt_skip_event(pt))
1391                 return 0;
1392
1393         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1394
1395         sample.id = ptq->pt->exstop_id;
1396         sample.stream_id = ptq->pt->exstop_id;
1397
1398         raw.flags = 0;
1399         raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);
1400
1401         sample.raw_size = perf_synth__raw_size(raw);
1402         sample.raw_data = perf_synth__raw_data(&raw);
1403
1404         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1405                                             pt->pwr_events_sample_type);
1406 }
1407
1408 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)
1409 {
1410         struct intel_pt *pt = ptq->pt;
1411         union perf_event *event = ptq->event_buf;
1412         struct perf_sample sample = { .ip = 0, };
1413         struct perf_synth_intel_pwrx raw;
1414
1415         if (intel_pt_skip_event(pt))
1416                 return 0;
1417
1418         intel_pt_prep_p_sample(pt, ptq, event, &sample);
1419
1420         sample.id = ptq->pt->pwrx_id;
1421         sample.stream_id = ptq->pt->pwrx_id;
1422
1423         raw.reserved = 0;
1424         raw.payload = cpu_to_le64(ptq->state->pwrx_payload);
1425
1426         sample.raw_size = perf_synth__raw_size(raw);
1427         sample.raw_data = perf_synth__raw_data(&raw);
1428
1429         return intel_pt_deliver_synth_event(pt, ptq, event, &sample,
1430                                             pt->pwr_events_sample_type);
1431 }
1432
1433 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu,
1434                                 pid_t pid, pid_t tid, u64 ip, u64 timestamp)
1435 {
1436         union perf_event event;
1437         char msg[MAX_AUXTRACE_ERROR_MSG];
1438         int err;
1439
1440         intel_pt__strerror(code, msg, MAX_AUXTRACE_ERROR_MSG);
1441
1442         auxtrace_synth_error(&event.auxtrace_error, PERF_AUXTRACE_ERROR_ITRACE,
1443                              code, cpu, pid, tid, ip, msg, timestamp);
1444
1445         err = perf_session__deliver_synth_event(pt->session, &event, NULL);
1446         if (err)
1447                 pr_err("Intel Processor Trace: failed to deliver error event, error %d\n",
1448                        err);
1449
1450         return err;
1451 }
1452
1453 static int intel_ptq_synth_error(struct intel_pt_queue *ptq,
1454                                  const struct intel_pt_state *state)
1455 {
1456         struct intel_pt *pt = ptq->pt;
1457         u64 tm = ptq->timestamp;
1458
1459         tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc);
1460
1461         return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid,
1462                                     ptq->tid, state->from_ip, tm);
1463 }
1464
1465 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)
1466 {
1467         struct auxtrace_queue *queue;
1468         pid_t tid = ptq->next_tid;
1469         int err;
1470
1471         if (tid == -1)
1472                 return 0;
1473
1474         intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);
1475
1476         err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);
1477
1478         queue = &pt->queues.queue_array[ptq->queue_nr];
1479         intel_pt_set_pid_tid_cpu(pt, queue);
1480
1481         ptq->next_tid = -1;
1482
1483         return err;
1484 }
1485
1486 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)
1487 {
1488         struct intel_pt *pt = ptq->pt;
1489
1490         return ip == pt->switch_ip &&
1491                (ptq->flags & PERF_IP_FLAG_BRANCH) &&
1492                !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |
1493                                PERF_IP_FLAG_INTERRUPT | PERF_IP_FLAG_TX_ABORT));
1494 }
1495
1496 #define INTEL_PT_PWR_EVT (INTEL_PT_MWAIT_OP | INTEL_PT_PWR_ENTRY | \
1497                           INTEL_PT_EX_STOP | INTEL_PT_PWR_EXIT | \
1498                           INTEL_PT_CBR_CHG)
1499
1500 static int intel_pt_sample(struct intel_pt_queue *ptq)
1501 {
1502         const struct intel_pt_state *state = ptq->state;
1503         struct intel_pt *pt = ptq->pt;
1504         int err;
1505
1506         if (!ptq->have_sample)
1507                 return 0;
1508
1509         ptq->have_sample = false;
1510
1511         if (ptq->state->tot_cyc_cnt > ptq->ipc_cyc_cnt) {
1512                 /*
1513                  * Cycle count and instruction count only go together to create
1514                  * a valid IPC ratio when the cycle count changes.
1515                  */
1516                 ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt;
1517                 ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt;
1518         }
1519
1520         if (pt->sample_pwr_events && (state->type & INTEL_PT_PWR_EVT)) {
1521                 if (state->type & INTEL_PT_CBR_CHG) {
1522                         err = intel_pt_synth_cbr_sample(ptq);
1523                         if (err)
1524                                 return err;
1525                 }
1526                 if (state->type & INTEL_PT_MWAIT_OP) {
1527                         err = intel_pt_synth_mwait_sample(ptq);
1528                         if (err)
1529                                 return err;
1530                 }
1531                 if (state->type & INTEL_PT_PWR_ENTRY) {
1532                         err = intel_pt_synth_pwre_sample(ptq);
1533                         if (err)
1534                                 return err;
1535                 }
1536                 if (state->type & INTEL_PT_EX_STOP) {
1537                         err = intel_pt_synth_exstop_sample(ptq);
1538                         if (err)
1539                                 return err;
1540                 }
1541                 if (state->type & INTEL_PT_PWR_EXIT) {
1542                         err = intel_pt_synth_pwrx_sample(ptq);
1543                         if (err)
1544                                 return err;
1545                 }
1546         }
1547
1548         if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) {
1549                 err = intel_pt_synth_instruction_sample(ptq);
1550                 if (err)
1551                         return err;
1552         }
1553
1554         if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) {
1555                 err = intel_pt_synth_transaction_sample(ptq);
1556                 if (err)
1557                         return err;
1558         }
1559
1560         if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) {
1561                 err = intel_pt_synth_ptwrite_sample(ptq);
1562                 if (err)
1563                         return err;
1564         }
1565
1566         if (!(state->type & INTEL_PT_BRANCH))
1567                 return 0;
1568
1569         if (pt->synth_opts.callchain || pt->synth_opts.thread_stack)
1570                 thread_stack__event(ptq->thread, ptq->cpu, ptq->flags, state->from_ip,
1571                                     state->to_ip, ptq->insn_len,
1572                                     state->trace_nr);
1573         else
1574                 thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr);
1575
1576         if (pt->sample_branches) {
1577                 err = intel_pt_synth_branch_sample(ptq);
1578                 if (err)
1579                         return err;
1580         }
1581
1582         if (pt->synth_opts.last_branch)
1583                 intel_pt_update_last_branch_rb(ptq);
1584
1585         if (!ptq->sync_switch)
1586                 return 0;
1587
1588         if (intel_pt_is_switch_ip(ptq, state->to_ip)) {
1589                 switch (ptq->switch_state) {
1590                 case INTEL_PT_SS_NOT_TRACING:
1591                 case INTEL_PT_SS_UNKNOWN:
1592                 case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1593                         err = intel_pt_next_tid(pt, ptq);
1594                         if (err)
1595                                 return err;
1596                         ptq->switch_state = INTEL_PT_SS_TRACING;
1597                         break;
1598                 default:
1599                         ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;
1600                         return 1;
1601                 }
1602         } else if (!state->to_ip) {
1603                 ptq->switch_state = INTEL_PT_SS_NOT_TRACING;
1604         } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {
1605                 ptq->switch_state = INTEL_PT_SS_UNKNOWN;
1606         } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1607                    state->to_ip == pt->ptss_ip &&
1608                    (ptq->flags & PERF_IP_FLAG_CALL)) {
1609                 ptq->switch_state = INTEL_PT_SS_TRACING;
1610         }
1611
1612         return 0;
1613 }
1614
1615 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip)
1616 {
1617         struct machine *machine = pt->machine;
1618         struct map *map;
1619         struct symbol *sym, *start;
1620         u64 ip, switch_ip = 0;
1621         const char *ptss;
1622
1623         if (ptss_ip)
1624                 *ptss_ip = 0;
1625
1626         map = machine__kernel_map(machine);
1627         if (!map)
1628                 return 0;
1629
1630         if (map__load(map))
1631                 return 0;
1632
1633         start = dso__first_symbol(map->dso);
1634
1635         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1636                 if (sym->binding == STB_GLOBAL &&
1637                     !strcmp(sym->name, "__switch_to")) {
1638                         ip = map->unmap_ip(map, sym->start);
1639                         if (ip >= map->start && ip < map->end) {
1640                                 switch_ip = ip;
1641                                 break;
1642                         }
1643                 }
1644         }
1645
1646         if (!switch_ip || !ptss_ip)
1647                 return 0;
1648
1649         if (pt->have_sched_switch == 1)
1650                 ptss = "perf_trace_sched_switch";
1651         else
1652                 ptss = "__perf_event_task_sched_out";
1653
1654         for (sym = start; sym; sym = dso__next_symbol(sym)) {
1655                 if (!strcmp(sym->name, ptss)) {
1656                         ip = map->unmap_ip(map, sym->start);
1657                         if (ip >= map->start && ip < map->end) {
1658                                 *ptss_ip = ip;
1659                                 break;
1660                         }
1661                 }
1662         }
1663
1664         return switch_ip;
1665 }
1666
1667 static void intel_pt_enable_sync_switch(struct intel_pt *pt)
1668 {
1669         unsigned int i;
1670
1671         pt->sync_switch = true;
1672
1673         for (i = 0; i < pt->queues.nr_queues; i++) {
1674                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1675                 struct intel_pt_queue *ptq = queue->priv;
1676
1677                 if (ptq)
1678                         ptq->sync_switch = true;
1679         }
1680 }
1681
1682 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)
1683 {
1684         const struct intel_pt_state *state = ptq->state;
1685         struct intel_pt *pt = ptq->pt;
1686         int err;
1687
1688         if (!pt->kernel_start) {
1689                 pt->kernel_start = machine__kernel_start(pt->machine);
1690                 if (pt->per_cpu_mmaps &&
1691                     (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) &&
1692                     !pt->timeless_decoding && intel_pt_tracing_kernel(pt) &&
1693                     !pt->sampling_mode) {
1694                         pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip);
1695                         if (pt->switch_ip) {
1696                                 intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n",
1697                                              pt->switch_ip, pt->ptss_ip);
1698                                 intel_pt_enable_sync_switch(pt);
1699                         }
1700                 }
1701         }
1702
1703         intel_pt_log("queue %u decoding cpu %d pid %d tid %d\n",
1704                      ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);
1705         while (1) {
1706                 err = intel_pt_sample(ptq);
1707                 if (err)
1708                         return err;
1709
1710                 state = intel_pt_decode(ptq->decoder);
1711                 if (state->err) {
1712                         if (state->err == INTEL_PT_ERR_NODATA)
1713                                 return 1;
1714                         if (ptq->sync_switch &&
1715                             state->from_ip >= pt->kernel_start) {
1716                                 ptq->sync_switch = false;
1717                                 intel_pt_next_tid(pt, ptq);
1718                         }
1719                         if (pt->synth_opts.errors) {
1720                                 err = intel_ptq_synth_error(ptq, state);
1721                                 if (err)
1722                                         return err;
1723                         }
1724                         continue;
1725                 }
1726
1727                 ptq->state = state;
1728                 ptq->have_sample = true;
1729                 intel_pt_sample_flags(ptq);
1730
1731                 /* Use estimated TSC upon return to user space */
1732                 if (pt->est_tsc &&
1733                     (state->from_ip >= pt->kernel_start || !state->from_ip) &&
1734                     state->to_ip && state->to_ip < pt->kernel_start) {
1735                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1736                                      state->timestamp, state->est_timestamp);
1737                         ptq->timestamp = state->est_timestamp;
1738                 /* Use estimated TSC in unknown switch state */
1739                 } else if (ptq->sync_switch &&
1740                            ptq->switch_state == INTEL_PT_SS_UNKNOWN &&
1741                            intel_pt_is_switch_ip(ptq, state->to_ip) &&
1742                            ptq->next_tid == -1) {
1743                         intel_pt_log("TSC %"PRIx64" est. TSC %"PRIx64"\n",
1744                                      state->timestamp, state->est_timestamp);
1745                         ptq->timestamp = state->est_timestamp;
1746                 } else if (state->timestamp > ptq->timestamp) {
1747                         ptq->timestamp = state->timestamp;
1748                 }
1749
1750                 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {
1751                         *timestamp = ptq->timestamp;
1752                         return 0;
1753                 }
1754         }
1755         return 0;
1756 }
1757
1758 static inline int intel_pt_update_queues(struct intel_pt *pt)
1759 {
1760         if (pt->queues.new_data) {
1761                 pt->queues.new_data = false;
1762                 return intel_pt_setup_queues(pt);
1763         }
1764         return 0;
1765 }
1766
1767 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp)
1768 {
1769         unsigned int queue_nr;
1770         u64 ts;
1771         int ret;
1772
1773         while (1) {
1774                 struct auxtrace_queue *queue;
1775                 struct intel_pt_queue *ptq;
1776
1777                 if (!pt->heap.heap_cnt)
1778                         return 0;
1779
1780                 if (pt->heap.heap_array[0].ordinal >= timestamp)
1781                         return 0;
1782
1783                 queue_nr = pt->heap.heap_array[0].queue_nr;
1784                 queue = &pt->queues.queue_array[queue_nr];
1785                 ptq = queue->priv;
1786
1787                 intel_pt_log("queue %u processing 0x%" PRIx64 " to 0x%" PRIx64 "\n",
1788                              queue_nr, pt->heap.heap_array[0].ordinal,
1789                              timestamp);
1790
1791                 auxtrace_heap__pop(&pt->heap);
1792
1793                 if (pt->heap.heap_cnt) {
1794                         ts = pt->heap.heap_array[0].ordinal + 1;
1795                         if (ts > timestamp)
1796                                 ts = timestamp;
1797                 } else {
1798                         ts = timestamp;
1799                 }
1800
1801                 intel_pt_set_pid_tid_cpu(pt, queue);
1802
1803                 ret = intel_pt_run_decoder(ptq, &ts);
1804
1805                 if (ret < 0) {
1806                         auxtrace_heap__add(&pt->heap, queue_nr, ts);
1807                         return ret;
1808                 }
1809
1810                 if (!ret) {
1811                         ret = auxtrace_heap__add(&pt->heap, queue_nr, ts);
1812                         if (ret < 0)
1813                                 return ret;
1814                 } else {
1815                         ptq->on_heap = false;
1816                 }
1817         }
1818
1819         return 0;
1820 }
1821
1822 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid,
1823                                             u64 time_)
1824 {
1825         struct auxtrace_queues *queues = &pt->queues;
1826         unsigned int i;
1827         u64 ts = 0;
1828
1829         for (i = 0; i < queues->nr_queues; i++) {
1830                 struct auxtrace_queue *queue = &pt->queues.queue_array[i];
1831                 struct intel_pt_queue *ptq = queue->priv;
1832
1833                 if (ptq && (tid == -1 || ptq->tid == tid)) {
1834                         ptq->time = time_;
1835                         intel_pt_set_pid_tid_cpu(pt, queue);
1836                         intel_pt_run_decoder(ptq, &ts);
1837                 }
1838         }
1839         return 0;
1840 }
1841
1842 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)
1843 {
1844         return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,
1845                                     sample->pid, sample->tid, 0, sample->time);
1846 }
1847
1848 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu)
1849 {
1850         unsigned i, j;
1851
1852         if (cpu < 0 || !pt->queues.nr_queues)
1853                 return NULL;
1854
1855         if ((unsigned)cpu >= pt->queues.nr_queues)
1856                 i = pt->queues.nr_queues - 1;
1857         else
1858                 i = cpu;
1859
1860         if (pt->queues.queue_array[i].cpu == cpu)
1861                 return pt->queues.queue_array[i].priv;
1862
1863         for (j = 0; i > 0; j++) {
1864                 if (pt->queues.queue_array[--i].cpu == cpu)
1865                         return pt->queues.queue_array[i].priv;
1866         }
1867
1868         for (; j < pt->queues.nr_queues; j++) {
1869                 if (pt->queues.queue_array[j].cpu == cpu)
1870                         return pt->queues.queue_array[j].priv;
1871         }
1872
1873         return NULL;
1874 }
1875
1876 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid,
1877                                 u64 timestamp)
1878 {
1879         struct intel_pt_queue *ptq;
1880         int err;
1881
1882         if (!pt->sync_switch)
1883                 return 1;
1884
1885         ptq = intel_pt_cpu_to_ptq(pt, cpu);
1886         if (!ptq || !ptq->sync_switch)
1887                 return 1;
1888
1889         switch (ptq->switch_state) {
1890         case INTEL_PT_SS_NOT_TRACING:
1891                 break;
1892         case INTEL_PT_SS_UNKNOWN:
1893         case INTEL_PT_SS_TRACING:
1894                 ptq->next_tid = tid;
1895                 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;
1896                 return 0;
1897         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1898                 if (!ptq->on_heap) {
1899                         ptq->timestamp = perf_time_to_tsc(timestamp,
1900                                                           &pt->tc);
1901                         err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,
1902                                                  ptq->timestamp);
1903                         if (err)
1904                                 return err;
1905                         ptq->on_heap = true;
1906                 }
1907                 ptq->switch_state = INTEL_PT_SS_TRACING;
1908                 break;
1909         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1910                 intel_pt_log("ERROR: cpu %d expecting switch ip\n", cpu);
1911                 break;
1912         default:
1913                 break;
1914         }
1915
1916         ptq->next_tid = -1;
1917
1918         return 1;
1919 }
1920
1921 static int intel_pt_process_switch(struct intel_pt *pt,
1922                                    struct perf_sample *sample)
1923 {
1924         struct perf_evsel *evsel;
1925         pid_t tid;
1926         int cpu, ret;
1927
1928         evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);
1929         if (evsel != pt->switch_evsel)
1930                 return 0;
1931
1932         tid = perf_evsel__intval(evsel, sample, "next_pid");
1933         cpu = sample->cpu;
1934
1935         intel_pt_log("sched_switch: cpu %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
1936                      cpu, tid, sample->time, perf_time_to_tsc(sample->time,
1937                      &pt->tc));
1938
1939         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
1940         if (ret <= 0)
1941                 return ret;
1942
1943         return machine__set_current_tid(pt->machine, cpu, -1, tid);
1944 }
1945
1946 static int intel_pt_context_switch_in(struct intel_pt *pt,
1947                                       struct perf_sample *sample)
1948 {
1949         pid_t pid = sample->pid;
1950         pid_t tid = sample->tid;
1951         int cpu = sample->cpu;
1952
1953         if (pt->sync_switch) {
1954                 struct intel_pt_queue *ptq;
1955
1956                 ptq = intel_pt_cpu_to_ptq(pt, cpu);
1957                 if (ptq && ptq->sync_switch) {
1958                         ptq->next_tid = -1;
1959                         switch (ptq->switch_state) {
1960                         case INTEL_PT_SS_NOT_TRACING:
1961                         case INTEL_PT_SS_UNKNOWN:
1962                         case INTEL_PT_SS_TRACING:
1963                                 break;
1964                         case INTEL_PT_SS_EXPECTING_SWITCH_EVENT:
1965                         case INTEL_PT_SS_EXPECTING_SWITCH_IP:
1966                                 ptq->switch_state = INTEL_PT_SS_TRACING;
1967                                 break;
1968                         default:
1969                                 break;
1970                         }
1971                 }
1972         }
1973
1974         /*
1975          * If the current tid has not been updated yet, ensure it is now that
1976          * a "switch in" event has occurred.
1977          */
1978         if (machine__get_current_tid(pt->machine, cpu) == tid)
1979                 return 0;
1980
1981         return machine__set_current_tid(pt->machine, cpu, pid, tid);
1982 }
1983
1984 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event,
1985                                    struct perf_sample *sample)
1986 {
1987         bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT;
1988         pid_t pid, tid;
1989         int cpu, ret;
1990
1991         cpu = sample->cpu;
1992
1993         if (pt->have_sched_switch == 3) {
1994                 if (!out)
1995                         return intel_pt_context_switch_in(pt, sample);
1996                 if (event->header.type != PERF_RECORD_SWITCH_CPU_WIDE) {
1997                         pr_err("Expecting CPU-wide context switch event\n");
1998                         return -EINVAL;
1999                 }
2000                 pid = event->context_switch.next_prev_pid;
2001                 tid = event->context_switch.next_prev_tid;
2002         } else {
2003                 if (out)
2004                         return 0;
2005                 pid = sample->pid;
2006                 tid = sample->tid;
2007         }
2008
2009         if (tid == -1) {
2010                 pr_err("context_switch event has no tid\n");
2011                 return -EINVAL;
2012         }
2013
2014         intel_pt_log("context_switch: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
2015                      cpu, pid, tid, sample->time, perf_time_to_tsc(sample->time,
2016                      &pt->tc));
2017
2018         ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);
2019         if (ret <= 0)
2020                 return ret;
2021
2022         return machine__set_current_tid(pt->machine, cpu, pid, tid);
2023 }
2024
2025 static int intel_pt_process_itrace_start(struct intel_pt *pt,
2026                                          union perf_event *event,
2027                                          struct perf_sample *sample)
2028 {
2029         if (!pt->per_cpu_mmaps)
2030                 return 0;
2031
2032         intel_pt_log("itrace_start: cpu %d pid %d tid %d time %"PRIu64" tsc %#"PRIx64"\n",
2033                      sample->cpu, event->itrace_start.pid,
2034                      event->itrace_start.tid, sample->time,
2035                      perf_time_to_tsc(sample->time, &pt->tc));
2036
2037         return machine__set_current_tid(pt->machine, sample->cpu,
2038                                         event->itrace_start.pid,
2039                                         event->itrace_start.tid);
2040 }
2041
2042 static int intel_pt_process_event(struct perf_session *session,
2043                                   union perf_event *event,
2044                                   struct perf_sample *sample,
2045                                   struct perf_tool *tool)
2046 {
2047         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2048                                            auxtrace);
2049         u64 timestamp;
2050         int err = 0;
2051
2052         if (dump_trace)
2053                 return 0;
2054
2055         if (!tool->ordered_events) {
2056                 pr_err("Intel Processor Trace requires ordered events\n");
2057                 return -EINVAL;
2058         }
2059
2060         if (sample->time && sample->time != (u64)-1)
2061                 timestamp = perf_time_to_tsc(sample->time, &pt->tc);
2062         else
2063                 timestamp = 0;
2064
2065         if (timestamp || pt->timeless_decoding) {
2066                 err = intel_pt_update_queues(pt);
2067                 if (err)
2068                         return err;
2069         }
2070
2071         if (pt->timeless_decoding) {
2072                 if (event->header.type == PERF_RECORD_EXIT) {
2073                         err = intel_pt_process_timeless_queues(pt,
2074                                                                event->fork.tid,
2075                                                                sample->time);
2076                 }
2077         } else if (timestamp) {
2078                 err = intel_pt_process_queues(pt, timestamp);
2079         }
2080         if (err)
2081                 return err;
2082
2083         if (event->header.type == PERF_RECORD_AUX &&
2084             (event->aux.flags & PERF_AUX_FLAG_TRUNCATED) &&
2085             pt->synth_opts.errors) {
2086                 err = intel_pt_lost(pt, sample);
2087                 if (err)
2088                         return err;
2089         }
2090
2091         if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE)
2092                 err = intel_pt_process_switch(pt, sample);
2093         else if (event->header.type == PERF_RECORD_ITRACE_START)
2094                 err = intel_pt_process_itrace_start(pt, event, sample);
2095         else if (event->header.type == PERF_RECORD_SWITCH ||
2096                  event->header.type == PERF_RECORD_SWITCH_CPU_WIDE)
2097                 err = intel_pt_context_switch(pt, event, sample);
2098
2099         intel_pt_log("event %u: cpu %d time %"PRIu64" tsc %#"PRIx64" ",
2100                      event->header.type, sample->cpu, sample->time, timestamp);
2101         intel_pt_log_event(event);
2102
2103         return err;
2104 }
2105
2106 static int intel_pt_flush(struct perf_session *session, struct perf_tool *tool)
2107 {
2108         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2109                                            auxtrace);
2110         int ret;
2111
2112         if (dump_trace)
2113                 return 0;
2114
2115         if (!tool->ordered_events)
2116                 return -EINVAL;
2117
2118         ret = intel_pt_update_queues(pt);
2119         if (ret < 0)
2120                 return ret;
2121
2122         if (pt->timeless_decoding)
2123                 return intel_pt_process_timeless_queues(pt, -1,
2124                                                         MAX_TIMESTAMP - 1);
2125
2126         return intel_pt_process_queues(pt, MAX_TIMESTAMP);
2127 }
2128
2129 static void intel_pt_free_events(struct perf_session *session)
2130 {
2131         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2132                                            auxtrace);
2133         struct auxtrace_queues *queues = &pt->queues;
2134         unsigned int i;
2135
2136         for (i = 0; i < queues->nr_queues; i++) {
2137                 intel_pt_free_queue(queues->queue_array[i].priv);
2138                 queues->queue_array[i].priv = NULL;
2139         }
2140         intel_pt_log_disable();
2141         auxtrace_queues__free(queues);
2142 }
2143
2144 static void intel_pt_free(struct perf_session *session)
2145 {
2146         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2147                                            auxtrace);
2148
2149         auxtrace_heap__free(&pt->heap);
2150         intel_pt_free_events(session);
2151         session->auxtrace = NULL;
2152         thread__put(pt->unknown_thread);
2153         addr_filters__exit(&pt->filts);
2154         zfree(&pt->filter);
2155         free(pt);
2156 }
2157
2158 static int intel_pt_process_auxtrace_event(struct perf_session *session,
2159                                            union perf_event *event,
2160                                            struct perf_tool *tool __maybe_unused)
2161 {
2162         struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt,
2163                                            auxtrace);
2164
2165         if (!pt->data_queued) {
2166                 struct auxtrace_buffer *buffer;
2167                 off_t data_offset;
2168                 int fd = perf_data__fd(session->data);
2169                 int err;
2170
2171                 if (perf_data__is_pipe(session->data)) {
2172                         data_offset = 0;
2173                 } else {
2174                         data_offset = lseek(fd, 0, SEEK_CUR);
2175                         if (data_offset == -1)
2176                                 return -errno;
2177                 }
2178
2179                 err = auxtrace_queues__add_event(&pt->queues, session, event,
2180                                                  data_offset, &buffer);
2181                 if (err)
2182                         return err;
2183
2184                 /* Dump here now we have copied a piped trace out of the pipe */
2185                 if (dump_trace) {
2186                         if (auxtrace_buffer__get_data(buffer, fd)) {
2187                                 intel_pt_dump_event(pt, buffer->data,
2188                                                     buffer->size);
2189                                 auxtrace_buffer__put_data(buffer);
2190                         }
2191                 }
2192         }
2193
2194         return 0;
2195 }
2196
2197 struct intel_pt_synth {
2198         struct perf_tool dummy_tool;
2199         struct perf_session *session;
2200 };
2201
2202 static int intel_pt_event_synth(struct perf_tool *tool,
2203                                 union perf_event *event,
2204                                 struct perf_sample *sample __maybe_unused,
2205                                 struct machine *machine __maybe_unused)
2206 {
2207         struct intel_pt_synth *intel_pt_synth =
2208                         container_of(tool, struct intel_pt_synth, dummy_tool);
2209
2210         return perf_session__deliver_synth_event(intel_pt_synth->session, event,
2211                                                  NULL);
2212 }
2213
2214 static int intel_pt_synth_event(struct perf_session *session, const char *name,
2215                                 struct perf_event_attr *attr, u64 id)
2216 {
2217         struct intel_pt_synth intel_pt_synth;
2218         int err;
2219
2220         pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",
2221                  name, id, (u64)attr->sample_type);
2222
2223         memset(&intel_pt_synth, 0, sizeof(struct intel_pt_synth));
2224         intel_pt_synth.session = session;
2225
2226         err = perf_event__synthesize_attr(&intel_pt_synth.dummy_tool, attr, 1,
2227                                           &id, intel_pt_event_synth);
2228         if (err)
2229                 pr_err("%s: failed to synthesize '%s' event type\n",
2230                        __func__, name);
2231
2232         return err;
2233 }
2234
2235 static void intel_pt_set_event_name(struct perf_evlist *evlist, u64 id,
2236                                     const char *name)
2237 {
2238         struct perf_evsel *evsel;
2239
2240         evlist__for_each_entry(evlist, evsel) {
2241                 if (evsel->id && evsel->id[0] == id) {
2242                         if (evsel->name)
2243                                 zfree(&evsel->name);
2244                         evsel->name = strdup(name);
2245                         break;
2246                 }
2247         }
2248 }
2249
2250 static struct perf_evsel *intel_pt_evsel(struct intel_pt *pt,
2251                                          struct perf_evlist *evlist)
2252 {
2253         struct perf_evsel *evsel;
2254
2255         evlist__for_each_entry(evlist, evsel) {
2256                 if (evsel->attr.type == pt->pmu_type && evsel->ids)
2257                         return evsel;
2258         }
2259
2260         return NULL;
2261 }
2262
2263 static int intel_pt_synth_events(struct intel_pt *pt,
2264                                  struct perf_session *session)
2265 {
2266         struct perf_evlist *evlist = session->evlist;
2267         struct perf_evsel *evsel = intel_pt_evsel(pt, evlist);
2268         struct perf_event_attr attr;
2269         u64 id;
2270         int err;
2271
2272         if (!evsel) {
2273                 pr_debug("There are no selected events with Intel Processor Trace data\n");
2274                 return 0;
2275         }
2276
2277         memset(&attr, 0, sizeof(struct perf_event_attr));
2278         attr.size = sizeof(struct perf_event_attr);
2279         attr.type = PERF_TYPE_HARDWARE;
2280         attr.sample_type = evsel->attr.sample_type & PERF_SAMPLE_MASK;
2281         attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID |
2282                             PERF_SAMPLE_PERIOD;
2283         if (pt->timeless_decoding)
2284                 attr.sample_type &= ~(u64)PERF_SAMPLE_TIME;
2285         else
2286                 attr.sample_type |= PERF_SAMPLE_TIME;
2287         if (!pt->per_cpu_mmaps)
2288                 attr.sample_type &= ~(u64)PERF_SAMPLE_CPU;
2289         attr.exclude_user = evsel->attr.exclude_user;
2290         attr.exclude_kernel = evsel->attr.exclude_kernel;
2291         attr.exclude_hv = evsel->attr.exclude_hv;
2292         attr.exclude_host = evsel->attr.exclude_host;
2293         attr.exclude_guest = evsel->attr.exclude_guest;
2294         attr.sample_id_all = evsel->attr.sample_id_all;
2295         attr.read_format = evsel->attr.read_format;
2296
2297         id = evsel->id[0] + 1000000000;
2298         if (!id)
2299                 id = 1;
2300
2301         if (pt->synth_opts.branches) {
2302                 attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS;
2303                 attr.sample_period = 1;
2304                 attr.sample_type |= PERF_SAMPLE_ADDR;
2305                 err = intel_pt_synth_event(session, "branches", &attr, id);
2306                 if (err)
2307                         return err;
2308                 pt->sample_branches = true;
2309                 pt->branches_sample_type = attr.sample_type;
2310                 pt->branches_id = id;
2311                 id += 1;
2312                 attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR;
2313         }
2314
2315         if (pt->synth_opts.callchain)
2316                 attr.sample_type |= PERF_SAMPLE_CALLCHAIN;
2317         if (pt->synth_opts.last_branch)
2318                 attr.sample_type |= PERF_SAMPLE_BRANCH_STACK;
2319
2320         if (pt->synth_opts.instructions) {
2321                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2322                 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS)
2323                         attr.sample_period =
2324                                 intel_pt_ns_to_ticks(pt, pt->synth_opts.period);
2325                 else
2326                         attr.sample_period = pt->synth_opts.period;
2327                 err = intel_pt_synth_event(session, "instructions", &attr, id);
2328                 if (err)
2329                         return err;
2330                 pt->sample_instructions = true;
2331                 pt->instructions_sample_type = attr.sample_type;
2332                 pt->instructions_id = id;
2333                 id += 1;
2334         }
2335
2336         attr.sample_type &= ~(u64)PERF_SAMPLE_PERIOD;
2337         attr.sample_period = 1;
2338
2339         if (pt->synth_opts.transactions) {
2340                 attr.config = PERF_COUNT_HW_INSTRUCTIONS;
2341                 err = intel_pt_synth_event(session, "transactions", &attr, id);
2342                 if (err)
2343                         return err;
2344                 pt->sample_transactions = true;
2345                 pt->transactions_sample_type = attr.sample_type;
2346                 pt->transactions_id = id;
2347                 intel_pt_set_event_name(evlist, id, "transactions");
2348                 id += 1;
2349         }
2350
2351         attr.type = PERF_TYPE_SYNTH;
2352         attr.sample_type |= PERF_SAMPLE_RAW;
2353
2354         if (pt->synth_opts.ptwrites) {
2355                 attr.config = PERF_SYNTH_INTEL_PTWRITE;
2356                 err = intel_pt_synth_event(session, "ptwrite", &attr, id);
2357                 if (err)
2358                         return err;
2359                 pt->sample_ptwrites = true;
2360                 pt->ptwrites_sample_type = attr.sample_type;
2361                 pt->ptwrites_id = id;
2362                 intel_pt_set_event_name(evlist, id, "ptwrite");
2363                 id += 1;
2364         }
2365
2366         if (pt->synth_opts.pwr_events) {
2367                 pt->sample_pwr_events = true;
2368                 pt->pwr_events_sample_type = attr.sample_type;
2369
2370                 attr.config = PERF_SYNTH_INTEL_CBR;
2371                 err = intel_pt_synth_event(session, "cbr", &attr, id);
2372                 if (err)
2373                         return err;
2374                 pt->cbr_id = id;
2375                 intel_pt_set_event_name(evlist, id, "cbr");
2376                 id += 1;
2377         }
2378
2379         if (pt->synth_opts.pwr_events && (evsel->attr.config & 0x10)) {
2380                 attr.config = PERF_SYNTH_INTEL_MWAIT;
2381                 err = intel_pt_synth_event(session, "mwait", &attr, id);
2382                 if (err)
2383                         return err;
2384                 pt->mwait_id = id;
2385                 intel_pt_set_event_name(evlist, id, "mwait");
2386                 id += 1;
2387
2388                 attr.config = PERF_SYNTH_INTEL_PWRE;
2389                 err = intel_pt_synth_event(session, "pwre", &attr, id);
2390                 if (err)
2391                         return err;
2392                 pt->pwre_id = id;
2393                 intel_pt_set_event_name(evlist, id, "pwre");
2394                 id += 1;
2395
2396                 attr.config = PERF_SYNTH_INTEL_EXSTOP;
2397                 err = intel_pt_synth_event(session, "exstop", &attr, id);
2398                 if (err)
2399                         return err;
2400                 pt->exstop_id = id;
2401                 intel_pt_set_event_name(evlist, id, "exstop");
2402                 id += 1;
2403
2404                 attr.config = PERF_SYNTH_INTEL_PWRX;
2405                 err = intel_pt_synth_event(session, "pwrx", &attr, id);
2406                 if (err)
2407                         return err;
2408                 pt->pwrx_id = id;
2409                 intel_pt_set_event_name(evlist, id, "pwrx");
2410                 id += 1;
2411         }
2412
2413         return 0;
2414 }
2415
2416 static struct perf_evsel *intel_pt_find_sched_switch(struct perf_evlist *evlist)
2417 {
2418         struct perf_evsel *evsel;
2419
2420         evlist__for_each_entry_reverse(evlist, evsel) {
2421                 const char *name = perf_evsel__name(evsel);
2422
2423                 if (!strcmp(name, "sched:sched_switch"))
2424                         return evsel;
2425         }
2426
2427         return NULL;
2428 }
2429
2430 static bool intel_pt_find_switch(struct perf_evlist *evlist)
2431 {
2432         struct perf_evsel *evsel;
2433
2434         evlist__for_each_entry(evlist, evsel) {
2435                 if (evsel->attr.context_switch)
2436                         return true;
2437         }
2438
2439         return false;
2440 }
2441
2442 static int intel_pt_perf_config(const char *var, const char *value, void *data)
2443 {
2444         struct intel_pt *pt = data;
2445
2446         if (!strcmp(var, "intel-pt.mispred-all"))
2447                 pt->mispred_all = perf_config_bool(var, value);
2448
2449         return 0;
2450 }
2451
2452 static const char * const intel_pt_info_fmts[] = {
2453         [INTEL_PT_PMU_TYPE]             = "  PMU Type            %"PRId64"\n",
2454         [INTEL_PT_TIME_SHIFT]           = "  Time Shift          %"PRIu64"\n",
2455         [INTEL_PT_TIME_MULT]            = "  Time Muliplier      %"PRIu64"\n",
2456         [INTEL_PT_TIME_ZERO]            = "  Time Zero           %"PRIu64"\n",
2457         [INTEL_PT_CAP_USER_TIME_ZERO]   = "  Cap Time Zero       %"PRId64"\n",
2458         [INTEL_PT_TSC_BIT]              = "  TSC bit             %#"PRIx64"\n",
2459         [INTEL_PT_NORETCOMP_BIT]        = "  NoRETComp bit       %#"PRIx64"\n",
2460         [INTEL_PT_HAVE_SCHED_SWITCH]    = "  Have sched_switch   %"PRId64"\n",
2461         [INTEL_PT_SNAPSHOT_MODE]        = "  Snapshot mode       %"PRId64"\n",
2462         [INTEL_PT_PER_CPU_MMAPS]        = "  Per-cpu maps        %"PRId64"\n",
2463         [INTEL_PT_MTC_BIT]              = "  MTC bit             %#"PRIx64"\n",
2464         [INTEL_PT_TSC_CTC_N]            = "  TSC:CTC numerator   %"PRIu64"\n",
2465         [INTEL_PT_TSC_CTC_D]            = "  TSC:CTC denominator %"PRIu64"\n",
2466         [INTEL_PT_CYC_BIT]              = "  CYC bit             %#"PRIx64"\n",
2467         [INTEL_PT_MAX_NONTURBO_RATIO]   = "  Max non-turbo ratio %"PRIu64"\n",
2468         [INTEL_PT_FILTER_STR_LEN]       = "  Filter string len.  %"PRIu64"\n",
2469 };
2470
2471 static void intel_pt_print_info(u64 *arr, int start, int finish)
2472 {
2473         int i;
2474
2475         if (!dump_trace)
2476                 return;
2477
2478         for (i = start; i <= finish; i++)
2479                 fprintf(stdout, intel_pt_info_fmts[i], arr[i]);
2480 }
2481
2482 static void intel_pt_print_info_str(const char *name, const char *str)
2483 {
2484         if (!dump_trace)
2485                 return;
2486
2487         fprintf(stdout, "  %-20s%s\n", name, str ? str : "");
2488 }
2489
2490 static bool intel_pt_has(struct auxtrace_info_event *auxtrace_info, int pos)
2491 {
2492         return auxtrace_info->header.size >=
2493                 sizeof(struct auxtrace_info_event) + (sizeof(u64) * (pos + 1));
2494 }
2495
2496 int intel_pt_process_auxtrace_info(union perf_event *event,
2497                                    struct perf_session *session)
2498 {
2499         struct auxtrace_info_event *auxtrace_info = &event->auxtrace_info;
2500         size_t min_sz = sizeof(u64) * INTEL_PT_PER_CPU_MMAPS;
2501         struct intel_pt *pt;
2502         void *info_end;
2503         u64 *info;
2504         int err;
2505
2506         if (auxtrace_info->header.size < sizeof(struct auxtrace_info_event) +
2507                                         min_sz)
2508                 return -EINVAL;
2509
2510         pt = zalloc(sizeof(struct intel_pt));
2511         if (!pt)
2512                 return -ENOMEM;
2513
2514         addr_filters__init(&pt->filts);
2515
2516         err = perf_config(intel_pt_perf_config, pt);
2517         if (err)
2518                 goto err_free;
2519
2520         err = auxtrace_queues__init(&pt->queues);
2521         if (err)
2522                 goto err_free;
2523
2524         intel_pt_log_set_name(INTEL_PT_PMU_NAME);
2525
2526         pt->session = session;
2527         pt->machine = &session->machines.host; /* No kvm support */
2528         pt->auxtrace_type = auxtrace_info->type;
2529         pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE];
2530         pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT];
2531         pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT];
2532         pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO];
2533         pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO];
2534         pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT];
2535         pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT];
2536         pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH];
2537         pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE];
2538         pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS];
2539         intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_PMU_TYPE,
2540                             INTEL_PT_PER_CPU_MMAPS);
2541
2542         if (intel_pt_has(auxtrace_info, INTEL_PT_CYC_BIT)) {
2543                 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT];
2544                 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS];
2545                 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N];
2546                 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D];
2547                 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT];
2548                 intel_pt_print_info(&auxtrace_info->priv[0], INTEL_PT_MTC_BIT,
2549                                     INTEL_PT_CYC_BIT);
2550         }
2551
2552         if (intel_pt_has(auxtrace_info, INTEL_PT_MAX_NONTURBO_RATIO)) {
2553                 pt->max_non_turbo_ratio =
2554                         auxtrace_info->priv[INTEL_PT_MAX_NONTURBO_RATIO];
2555                 intel_pt_print_info(&auxtrace_info->priv[0],
2556                                     INTEL_PT_MAX_NONTURBO_RATIO,
2557                                     INTEL_PT_MAX_NONTURBO_RATIO);
2558         }
2559
2560         info = &auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN] + 1;
2561         info_end = (void *)info + auxtrace_info->header.size;
2562
2563         if (intel_pt_has(auxtrace_info, INTEL_PT_FILTER_STR_LEN)) {
2564                 size_t len;
2565
2566                 len = auxtrace_info->priv[INTEL_PT_FILTER_STR_LEN];
2567                 intel_pt_print_info(&auxtrace_info->priv[0],
2568                                     INTEL_PT_FILTER_STR_LEN,
2569                                     INTEL_PT_FILTER_STR_LEN);
2570                 if (len) {
2571                         const char *filter = (const char *)info;
2572
2573                         len = roundup(len + 1, 8);
2574                         info += len >> 3;
2575                         if ((void *)info > info_end) {
2576                                 pr_err("%s: bad filter string length\n", __func__);
2577                                 err = -EINVAL;
2578                                 goto err_free_queues;
2579                         }
2580                         pt->filter = memdup(filter, len);
2581                         if (!pt->filter) {
2582                                 err = -ENOMEM;
2583                                 goto err_free_queues;
2584                         }
2585                         if (session->header.needs_swap)
2586                                 mem_bswap_64(pt->filter, len);
2587                         if (pt->filter[len - 1]) {
2588                                 pr_err("%s: filter string not null terminated\n", __func__);
2589                                 err = -EINVAL;
2590                                 goto err_free_queues;
2591                         }
2592                         err = addr_filters__parse_bare_filter(&pt->filts,
2593                                                               filter);
2594                         if (err)
2595                                 goto err_free_queues;
2596                 }
2597                 intel_pt_print_info_str("Filter string", pt->filter);
2598         }
2599
2600         pt->timeless_decoding = intel_pt_timeless_decoding(pt);
2601         if (pt->timeless_decoding && !pt->tc.time_mult)
2602                 pt->tc.time_mult = 1;
2603         pt->have_tsc = intel_pt_have_tsc(pt);
2604         pt->sampling_mode = false;
2605         pt->est_tsc = !pt->timeless_decoding;
2606
2607         pt->unknown_thread = thread__new(999999999, 999999999);
2608         if (!pt->unknown_thread) {
2609                 err = -ENOMEM;
2610                 goto err_free_queues;
2611         }
2612
2613         /*
2614          * Since this thread will not be kept in any rbtree not in a
2615          * list, initialize its list node so that at thread__put() the
2616          * current thread lifetime assuption is kept and we don't segfault
2617          * at list_del_init().
2618          */
2619         INIT_LIST_HEAD(&pt->unknown_thread->node);
2620
2621         err = thread__set_comm(pt->unknown_thread, "unknown", 0);
2622         if (err)
2623                 goto err_delete_thread;
2624         if (thread__init_map_groups(pt->unknown_thread, pt->machine)) {
2625                 err = -ENOMEM;
2626                 goto err_delete_thread;
2627         }
2628
2629         pt->auxtrace.process_event = intel_pt_process_event;
2630         pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event;
2631         pt->auxtrace.flush_events = intel_pt_flush;
2632         pt->auxtrace.free_events = intel_pt_free_events;
2633         pt->auxtrace.free = intel_pt_free;
2634         session->auxtrace = &pt->auxtrace;
2635
2636         if (dump_trace)
2637                 return 0;
2638
2639         if (pt->have_sched_switch == 1) {
2640                 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist);
2641                 if (!pt->switch_evsel) {
2642                         pr_err("%s: missing sched_switch event\n", __func__);
2643                         err = -EINVAL;
2644                         goto err_delete_thread;
2645                 }
2646         } else if (pt->have_sched_switch == 2 &&
2647                    !intel_pt_find_switch(session->evlist)) {
2648                 pr_err("%s: missing context_switch attribute flag\n", __func__);
2649                 err = -EINVAL;
2650                 goto err_delete_thread;
2651         }
2652
2653         if (session->itrace_synth_opts && session->itrace_synth_opts->set) {
2654                 pt->synth_opts = *session->itrace_synth_opts;
2655         } else {
2656                 itrace_synth_opts__set_default(&pt->synth_opts,
2657                                 session->itrace_synth_opts->default_no_sample);
2658                 if (!session->itrace_synth_opts->default_no_sample &&
2659                     !session->itrace_synth_opts->inject) {
2660                         pt->synth_opts.branches = false;
2661                         pt->synth_opts.callchain = true;
2662                 }
2663                 if (session->itrace_synth_opts)
2664                         pt->synth_opts.thread_stack =
2665                                 session->itrace_synth_opts->thread_stack;
2666         }
2667
2668         if (pt->synth_opts.log)
2669                 intel_pt_log_enable();
2670
2671         /* Maximum non-turbo ratio is TSC freq / 100 MHz */
2672         if (pt->tc.time_mult) {
2673                 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000);
2674
2675                 if (!pt->max_non_turbo_ratio)
2676                         pt->max_non_turbo_ratio =
2677                                         (tsc_freq + 50000000) / 100000000;
2678                 intel_pt_log("TSC frequency %"PRIu64"\n", tsc_freq);
2679                 intel_pt_log("Maximum non-turbo ratio %u\n",
2680                              pt->max_non_turbo_ratio);
2681                 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000;
2682         }
2683
2684         if (pt->synth_opts.calls)
2685                 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC |
2686                                        PERF_IP_FLAG_TRACE_END;
2687         if (pt->synth_opts.returns)
2688                 pt->branches_filter |= PERF_IP_FLAG_RETURN |
2689                                        PERF_IP_FLAG_TRACE_BEGIN;
2690
2691         if (pt->synth_opts.callchain && !symbol_conf.use_callchain) {
2692                 symbol_conf.use_callchain = true;
2693                 if (callchain_register_param(&callchain_param) < 0) {
2694                         symbol_conf.use_callchain = false;
2695                         pt->synth_opts.callchain = false;
2696                 }
2697         }
2698
2699         err = intel_pt_synth_events(pt, session);
2700         if (err)
2701                 goto err_delete_thread;
2702
2703         err = auxtrace_queues__process_index(&pt->queues, session);
2704         if (err)
2705                 goto err_delete_thread;
2706
2707         if (pt->queues.populated)
2708                 pt->data_queued = true;
2709
2710         if (pt->timeless_decoding)
2711                 pr_debug2("Intel PT decoding without timestamps\n");
2712
2713         return 0;
2714
2715 err_delete_thread:
2716         thread__zput(pt->unknown_thread);
2717 err_free_queues:
2718         intel_pt_log_disable();
2719         auxtrace_queues__free(&pt->queues);
2720         session->auxtrace = NULL;
2721 err_free:
2722         addr_filters__exit(&pt->filts);
2723         zfree(&pt->filter);
2724         free(pt);
2725         return err;
2726 }