tools/perf/util/intel-pt-decoder/intel-pt-decoder.c

   1 /*
   2  * intel_pt_decoder.c: Intel Processor Trace support
   3  * Copyright (c) 2013-2014, Intel Corporation.
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #ifndef _GNU_SOURCE
  17 #define _GNU_SOURCE
  18 #endif
  19 #include <stdlib.h>
  20 #include <stdbool.h>
  21 #include <string.h>
  22 #include <errno.h>
  23 #include <stdint.h>
  24 #include <inttypes.h>
  25 #include <linux/compiler.h>
  26
  27 #include "../cache.h"
  28 #include "../util.h"
  29 #include "../auxtrace.h"
  30
  31 #include "intel-pt-insn-decoder.h"
  32 #include "intel-pt-pkt-decoder.h"
  33 #include "intel-pt-decoder.h"
  34 #include "intel-pt-log.h"
  35
  36 #define INTEL_PT_BLK_SIZE 1024
  37
  38 #define BIT63 (((uint64_t)1 << 63))
  39
  40 #define INTEL_PT_RETURN 1
  41
  42 /* Maximum number of loops with no packets consumed i.e. stuck in a loop */
  43 #define INTEL_PT_MAX_LOOPS 10000
  44
  45 struct intel_pt_blk {
  46         struct intel_pt_blk *prev;
  47         uint64_t ip[INTEL_PT_BLK_SIZE];
  48 };
  49
  50 struct intel_pt_stack {
  51         struct intel_pt_blk *blk;
  52         struct intel_pt_blk *spare;
  53         int pos;
  54 };
  55
  56 enum intel_pt_pkt_state {
  57         INTEL_PT_STATE_NO_PSB,
  58         INTEL_PT_STATE_NO_IP,
  59         INTEL_PT_STATE_ERR_RESYNC,
  60         INTEL_PT_STATE_IN_SYNC,
  61         INTEL_PT_STATE_TNT,
  62         INTEL_PT_STATE_TIP,
  63         INTEL_PT_STATE_TIP_PGD,
  64         INTEL_PT_STATE_FUP,
  65         INTEL_PT_STATE_FUP_NO_TIP,
  66 };
  67
  68 static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state)
  69 {
  70         switch (pkt_state) {
  71         case INTEL_PT_STATE_NO_PSB:
  72         case INTEL_PT_STATE_NO_IP:
  73         case INTEL_PT_STATE_ERR_RESYNC:
  74         case INTEL_PT_STATE_IN_SYNC:
  75         case INTEL_PT_STATE_TNT:
  76                 return true;
  77         case INTEL_PT_STATE_TIP:
  78         case INTEL_PT_STATE_TIP_PGD:
  79         case INTEL_PT_STATE_FUP:
  80         case INTEL_PT_STATE_FUP_NO_TIP:
  81                 return false;
  82         default:
  83                 return true;
  84         };
  85 }
  86
  87 #ifdef INTEL_PT_STRICT
  88 #define INTEL_PT_STATE_ERR1     INTEL_PT_STATE_NO_PSB
  89 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_PSB
  90 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_NO_PSB
  91 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_NO_PSB
  92 #else
  93 #define INTEL_PT_STATE_ERR1     (decoder->pkt_state)
  94 #define INTEL_PT_STATE_ERR2     INTEL_PT_STATE_NO_IP
  95 #define INTEL_PT_STATE_ERR3     INTEL_PT_STATE_ERR_RESYNC
  96 #define INTEL_PT_STATE_ERR4     INTEL_PT_STATE_IN_SYNC
  97 #endif
  98
  99 struct intel_pt_decoder {
 100         int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
 101         int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
 102                          uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
 103                          uint64_t max_insn_cnt, void *data);
 104         bool (*pgd_ip)(uint64_t ip, void *data);
 105         void *data;
 106         struct intel_pt_state state;
 107         const unsigned char *buf;
 108         size_t len;
 109         bool return_compression;
 110         bool branch_enable;
 111         bool mtc_insn;
 112         bool pge;
 113         bool have_tma;
 114         bool have_cyc;
 115         bool fixup_last_mtc;
 116         bool have_last_ip;
 117         enum intel_pt_param_flags flags;
 118         uint64_t pos;
 119         uint64_t last_ip;
 120         uint64_t ip;
 121         uint64_t cr3;
 122         uint64_t timestamp;
 123         uint64_t tsc_timestamp;
 124         uint64_t ref_timestamp;
 125         uint64_t sample_timestamp;
 126         uint64_t ret_addr;
 127         uint64_t ctc_timestamp;
 128         uint64_t ctc_delta;
 129         uint64_t cycle_cnt;
 130         uint64_t cyc_ref_timestamp;
 131         uint32_t last_mtc;
 132         uint32_t tsc_ctc_ratio_n;
 133         uint32_t tsc_ctc_ratio_d;
 134         uint32_t tsc_ctc_mult;
 135         uint32_t tsc_slip;
 136         uint32_t ctc_rem_mask;
 137         int mtc_shift;
 138         struct intel_pt_stack stack;
 139         enum intel_pt_pkt_state pkt_state;
 140         struct intel_pt_pkt packet;
 141         struct intel_pt_pkt tnt;
 142         int pkt_step;
 143         int pkt_len;
 144         int last_packet_type;
 145         unsigned int cbr;
 146         unsigned int cbr_seen;
 147         unsigned int max_non_turbo_ratio;
 148         double max_non_turbo_ratio_fp;
 149         double cbr_cyc_to_tsc;
 150         double calc_cyc_to_tsc;
 151         bool have_calc_cyc_to_tsc;
 152         int exec_mode;
 153         unsigned int insn_bytes;
 154         uint64_t period;
 155         enum intel_pt_period_type period_type;
 156         uint64_t tot_insn_cnt;
 157         uint64_t period_insn_cnt;
 158         uint64_t period_mask;
 159         uint64_t period_ticks;
 160         uint64_t last_masked_timestamp;
 161         bool continuous_period;
 162         bool overflow;
 163         bool set_fup_tx_flags;
 164         bool set_fup_ptw;
 165         bool set_fup_mwait;
 166         bool set_fup_pwre;
 167         bool set_fup_exstop;
 168         unsigned int fup_tx_flags;
 169         unsigned int tx_flags;
 170         uint64_t fup_ptw_payload;
 171         uint64_t fup_mwait_payload;
 172         uint64_t fup_pwre_payload;
 173         uint64_t cbr_payload;
 174         uint64_t timestamp_insn_cnt;
 175         uint64_t sample_insn_cnt;
 176         uint64_t stuck_ip;
 177         int no_progress;
 178         int stuck_ip_prd;
 179         int stuck_ip_cnt;
 180         const unsigned char *next_buf;
 181         size_t next_len;
 182         unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
 183 };
 184
 185 static uint64_t intel_pt_lower_power_of_2(uint64_t x)
 186 {
 187         int i;
 188
 189         for (i = 0; x != 1; i++)
 190                 x >>= 1;
 191
 192         return x << i;
 193 }
 194
 195 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
 196 {
 197         if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
 198                 uint64_t period;
 199
 200                 period = intel_pt_lower_power_of_2(decoder->period);
 201                 decoder->period_mask  = ~(period - 1);
 202                 decoder->period_ticks = period;
 203         }
 204 }
 205
 206 static uint64_t multdiv(uint64_t t, uint32_t n, uint32_t d)
 207 {
 208         if (!d)
 209                 return 0;
 210         return (t / d) * n + ((t % d) * n) / d;
 211 }
 212
 213 struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
 214 {
 215         struct intel_pt_decoder *decoder;
 216
 217         if (!params->get_trace || !params->walk_insn)
 218                 return NULL;
 219
 220         decoder = zalloc(sizeof(struct intel_pt_decoder));
 221         if (!decoder)
 222                 return NULL;
 223
 224         decoder->get_trace          = params->get_trace;
 225         decoder->walk_insn          = params->walk_insn;
 226         decoder->pgd_ip             = params->pgd_ip;
 227         decoder->data               = params->data;
 228         decoder->return_compression = params->return_compression;
 229         decoder->branch_enable      = params->branch_enable;
 230
 231         decoder->flags              = params->flags;
 232
 233         decoder->period             = params->period;
 234         decoder->period_type        = params->period_type;
 235
 236         decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;
 237         decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;
 238
 239         intel_pt_setup_period(decoder);
 240
 241         decoder->mtc_shift = params->mtc_period;
 242         decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;
 243
 244         decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;
 245         decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;
 246
 247         if (!decoder->tsc_ctc_ratio_n)
 248                 decoder->tsc_ctc_ratio_d = 0;
 249
 250         if (decoder->tsc_ctc_ratio_d) {
 251                 if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))
 252                         decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /
 253                                                 decoder->tsc_ctc_ratio_d;
 254         }
 255
 256         /*
 257          * A TSC packet can slip past MTC packets so that the timestamp appears
 258          * to go backwards. One estimate is that can be up to about 40 CPU
 259          * cycles, which is certainly less than 0x1000 TSC ticks, but accept
 260          * slippage an order of magnitude more to be on the safe side.
 261          */
 262         decoder->tsc_slip = 0x10000;
 263
 264         intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);
 265         intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);
 266         intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);
 267         intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);
 268         intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);
 269
 270         return decoder;
 271 }
 272
 273 static void intel_pt_pop_blk(struct intel_pt_stack *stack)
 274 {
 275         struct intel_pt_blk *blk = stack->blk;
 276
 277         stack->blk = blk->prev;
 278         if (!stack->spare)
 279                 stack->spare = blk;
 280         else
 281                 free(blk);
 282 }
 283
 284 static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
 285 {
 286         if (!stack->pos) {
 287                 if (!stack->blk)
 288                         return 0;
 289                 intel_pt_pop_blk(stack);
 290                 if (!stack->blk)
 291                         return 0;
 292                 stack->pos = INTEL_PT_BLK_SIZE;
 293         }
 294         return stack->blk->ip[--stack->pos];
 295 }
 296
 297 static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
 298 {
 299         struct intel_pt_blk *blk;
 300
 301         if (stack->spare) {
 302                 blk = stack->spare;
 303                 stack->spare = NULL;
 304         } else {
 305                 blk = malloc(sizeof(struct intel_pt_blk));
 306                 if (!blk)
 307                         return -ENOMEM;
 308         }
 309
 310         blk->prev = stack->blk;
 311         stack->blk = blk;
 312         stack->pos = 0;
 313         return 0;
 314 }
 315
 316 static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
 317 {
 318         int err;
 319
 320         if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
 321                 err = intel_pt_alloc_blk(stack);
 322                 if (err)
 323                         return err;
 324         }
 325
 326         stack->blk->ip[stack->pos++] = ip;
 327         return 0;
 328 }
 329
 330 static void intel_pt_clear_stack(struct intel_pt_stack *stack)
 331 {
 332         while (stack->blk)
 333                 intel_pt_pop_blk(stack);
 334         stack->pos = 0;
 335 }
 336
 337 static void intel_pt_free_stack(struct intel_pt_stack *stack)
 338 {
 339         intel_pt_clear_stack(stack);
 340         zfree(&stack->blk);
 341         zfree(&stack->spare);
 342 }
 343
 344 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
 345 {
 346         intel_pt_free_stack(&decoder->stack);
 347         free(decoder);
 348 }
 349
 350 static int intel_pt_ext_err(int code)
 351 {
 352         switch (code) {
 353         case -ENOMEM:
 354                 return INTEL_PT_ERR_NOMEM;
 355         case -ENOSYS:
 356                 return INTEL_PT_ERR_INTERN;
 357         case -EBADMSG:
 358                 return INTEL_PT_ERR_BADPKT;
 359         case -ENODATA:
 360                 return INTEL_PT_ERR_NODATA;
 361         case -EILSEQ:
 362                 return INTEL_PT_ERR_NOINSN;
 363         case -ENOENT:
 364                 return INTEL_PT_ERR_MISMAT;
 365         case -EOVERFLOW:
 366                 return INTEL_PT_ERR_OVR;
 367         case -ENOSPC:
 368                 return INTEL_PT_ERR_LOST;
 369         case -ELOOP:
 370                 return INTEL_PT_ERR_NELOOP;
 371         default:
 372                 return INTEL_PT_ERR_UNK;
 373         }
 374 }
 375
 376 static const char *intel_pt_err_msgs[] = {
 377         [INTEL_PT_ERR_NOMEM]  = "Memory allocation failed",
 378         [INTEL_PT_ERR_INTERN] = "Internal error",
 379         [INTEL_PT_ERR_BADPKT] = "Bad packet",
 380         [INTEL_PT_ERR_NODATA] = "No more data",
 381         [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
 382         [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
 383         [INTEL_PT_ERR_OVR]    = "Overflow packet",
 384         [INTEL_PT_ERR_LOST]   = "Lost trace data",
 385         [INTEL_PT_ERR_UNK]    = "Unknown error!",
 386         [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
 387 };
 388
 389 int intel_pt__strerror(int code, char *buf, size_t buflen)
 390 {
 391         if (code < 1 || code >= INTEL_PT_ERR_MAX)
 392                 code = INTEL_PT_ERR_UNK;
 393         strlcpy(buf, intel_pt_err_msgs[code], buflen);
 394         return 0;
 395 }
 396
 397 static uint64_t intel_pt_calc_ip(const struct intel_pt_pkt *packet,
 398                                  uint64_t last_ip)
 399 {
 400         uint64_t ip;
 401
 402         switch (packet->count) {
 403         case 1:
 404                 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
 405                      packet->payload;
 406                 break;
 407         case 2:
 408                 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
 409                      packet->payload;
 410                 break;
 411         case 3:
 412                 ip = packet->payload;
 413                 /* Sign-extend 6-byte ip */
 414                 if (ip & (uint64_t)0x800000000000ULL)
 415                         ip |= (uint64_t)0xffff000000000000ULL;
 416                 break;
 417         case 4:
 418                 ip = (last_ip & (uint64_t)0xffff000000000000ULL) |
 419                      packet->payload;
 420                 break;
 421         case 6:
 422                 ip = packet->payload;
 423                 break;
 424         default:
 425                 return 0;
 426         }
 427
 428         return ip;
 429 }
 430
 431 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
 432 {
 433         decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);
 434         decoder->have_last_ip = true;
 435 }
 436
 437 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
 438 {
 439         intel_pt_set_last_ip(decoder);
 440         decoder->ip = decoder->last_ip;
 441 }
 442
 443 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
 444 {
 445         intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
 446                             decoder->buf);
 447 }
 448
 449 static int intel_pt_bug(struct intel_pt_decoder *decoder)
 450 {
 451         intel_pt_log("ERROR: Internal error\n");
 452         decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 453         return -ENOSYS;
 454 }
 455
 456 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
 457 {
 458         decoder->tx_flags = 0;
 459 }
 460
 461 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
 462 {
 463         decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
 464 }
 465
 466 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
 467 {
 468         intel_pt_clear_tx_flags(decoder);
 469         decoder->have_tma = false;
 470         decoder->pkt_len = 1;
 471         decoder->pkt_step = 1;
 472         intel_pt_decoder_log_packet(decoder);
 473         if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
 474                 intel_pt_log("ERROR: Bad packet\n");
 475                 decoder->pkt_state = INTEL_PT_STATE_ERR1;
 476         }
 477         return -EBADMSG;
 478 }
 479
 480 static int intel_pt_get_data(struct intel_pt_decoder *decoder)
 481 {
 482         struct intel_pt_buffer buffer = { .buf = 0, };
 483         int ret;
 484
 485         decoder->pkt_step = 0;
 486
 487         intel_pt_log("Getting more data\n");
 488         ret = decoder->get_trace(&buffer, decoder->data);
 489         if (ret)
 490                 return ret;
 491         decoder->buf = buffer.buf;
 492         decoder->len = buffer.len;
 493         if (!decoder->len) {
 494                 intel_pt_log("No more data\n");
 495                 return -ENODATA;
 496         }
 497         if (!buffer.consecutive) {
 498                 decoder->ip = 0;
 499                 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
 500                 decoder->ref_timestamp = buffer.ref_timestamp;
 501                 decoder->timestamp = 0;
 502                 decoder->have_tma = false;
 503                 decoder->state.trace_nr = buffer.trace_nr;
 504                 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
 505                              decoder->ref_timestamp);
 506                 return -ENOLINK;
 507         }
 508
 509         return 0;
 510 }
 511
 512 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
 513 {
 514         if (!decoder->next_buf)
 515                 return intel_pt_get_data(decoder);
 516
 517         decoder->buf = decoder->next_buf;
 518         decoder->len = decoder->next_len;
 519         decoder->next_buf = 0;
 520         decoder->next_len = 0;
 521         return 0;
 522 }
 523
 524 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
 525 {
 526         unsigned char *buf = decoder->temp_buf;
 527         size_t old_len, len, n;
 528         int ret;
 529
 530         old_len = decoder->len;
 531         len = decoder->len;
 532         memcpy(buf, decoder->buf, len);
 533
 534         ret = intel_pt_get_data(decoder);
 535         if (ret) {
 536                 decoder->pos += old_len;
 537                 return ret < 0 ? ret : -EINVAL;
 538         }
 539
 540         n = INTEL_PT_PKT_MAX_SZ - len;
 541         if (n > decoder->len)
 542                 n = decoder->len;
 543         memcpy(buf + len, decoder->buf, n);
 544         len += n;
 545
 546         ret = intel_pt_get_packet(buf, len, &decoder->packet);
 547         if (ret < (int)old_len) {
 548                 decoder->next_buf = decoder->buf;
 549                 decoder->next_len = decoder->len;
 550                 decoder->buf = buf;
 551                 decoder->len = old_len;
 552                 return intel_pt_bad_packet(decoder);
 553         }
 554
 555         decoder->next_buf = decoder->buf + (ret - old_len);
 556         decoder->next_len = decoder->len - (ret - old_len);
 557
 558         decoder->buf = buf;
 559         decoder->len = ret;
 560
 561         return ret;
 562 }
 563
 564 struct intel_pt_pkt_info {
 565         struct intel_pt_decoder   *decoder;
 566         struct intel_pt_pkt       packet;
 567         uint64_t                  pos;
 568         int                       pkt_len;
 569         int                       last_packet_type;
 570         void                      *data;
 571 };
 572
 573 typedef int (*intel_pt_pkt_cb_t)(struct intel_pt_pkt_info *pkt_info);
 574
 575 /* Lookahead packets in current buffer */
 576 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,
 577                                   intel_pt_pkt_cb_t cb, void *data)
 578 {
 579         struct intel_pt_pkt_info pkt_info;
 580         const unsigned char *buf = decoder->buf;
 581         size_t len = decoder->len;
 582         int ret;
 583
 584         pkt_info.decoder          = decoder;
 585         pkt_info.pos              = decoder->pos;
 586         pkt_info.pkt_len          = decoder->pkt_step;
 587         pkt_info.last_packet_type = decoder->last_packet_type;
 588         pkt_info.data             = data;
 589
 590         while (1) {
 591                 do {
 592                         pkt_info.pos += pkt_info.pkt_len;
 593                         buf          += pkt_info.pkt_len;
 594                         len          -= pkt_info.pkt_len;
 595
 596                         if (!len)
 597                                 return INTEL_PT_NEED_MORE_BYTES;
 598
 599                         ret = intel_pt_get_packet(buf, len, &pkt_info.packet);
 600                         if (!ret)
 601                                 return INTEL_PT_NEED_MORE_BYTES;
 602                         if (ret < 0)
 603                                 return ret;
 604
 605                         pkt_info.pkt_len = ret;
 606                 } while (pkt_info.packet.type == INTEL_PT_PAD);
 607
 608                 ret = cb(&pkt_info);
 609                 if (ret)
 610                         return 0;
 611
 612                 pkt_info.last_packet_type = pkt_info.packet.type;
 613         }
 614 }
 615
 616 struct intel_pt_calc_cyc_to_tsc_info {
 617         uint64_t        cycle_cnt;
 618         unsigned int    cbr;
 619         uint32_t        last_mtc;
 620         uint64_t        ctc_timestamp;
 621         uint64_t        ctc_delta;
 622         uint64_t        tsc_timestamp;
 623         uint64_t        timestamp;
 624         bool            have_tma;
 625         bool            fixup_last_mtc;
 626         bool            from_mtc;
 627         double          cbr_cyc_to_tsc;
 628 };
 629
 630 /*
 631  * MTC provides a 8-bit slice of CTC but the TMA packet only provides the lower
 632  * 16 bits of CTC. If mtc_shift > 8 then some of the MTC bits are not in the CTC
 633  * provided by the TMA packet. Fix-up the last_mtc calculated from the TMA
 634  * packet by copying the missing bits from the current MTC assuming the least
 635  * difference between the two, and that the current MTC comes after last_mtc.
 636  */
 637 static void intel_pt_fixup_last_mtc(uint32_t mtc, int mtc_shift,
 638                                     uint32_t *last_mtc)
 639 {
 640         uint32_t first_missing_bit = 1U << (16 - mtc_shift);
 641         uint32_t mask = ~(first_missing_bit - 1);
 642
 643         *last_mtc |= mtc & mask;
 644         if (*last_mtc >= mtc) {
 645                 *last_mtc -= first_missing_bit;
 646                 *last_mtc &= 0xff;
 647         }
 648 }
 649
 650 static int intel_pt_calc_cyc_cb(struct intel_pt_pkt_info *pkt_info)
 651 {
 652         struct intel_pt_decoder *decoder = pkt_info->decoder;
 653         struct intel_pt_calc_cyc_to_tsc_info *data = pkt_info->data;
 654         uint64_t timestamp;
 655         double cyc_to_tsc;
 656         unsigned int cbr;
 657         uint32_t mtc, mtc_delta, ctc, fc, ctc_rem;
 658
 659         switch (pkt_info->packet.type) {
 660         case INTEL_PT_TNT:
 661         case INTEL_PT_TIP_PGE:
 662         case INTEL_PT_TIP:
 663         case INTEL_PT_FUP:
 664         case INTEL_PT_PSB:
 665         case INTEL_PT_PIP:
 666         case INTEL_PT_MODE_EXEC:
 667         case INTEL_PT_MODE_TSX:
 668         case INTEL_PT_PSBEND:
 669         case INTEL_PT_PAD:
 670         case INTEL_PT_VMCS:
 671         case INTEL_PT_MNT:
 672         case INTEL_PT_PTWRITE:
 673         case INTEL_PT_PTWRITE_IP:
 674                 return 0;
 675
 676         case INTEL_PT_MTC:
 677                 if (!data->have_tma)
 678                         return 0;
 679
 680                 mtc = pkt_info->packet.payload;
 681                 if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {
 682                         data->fixup_last_mtc = false;
 683                         intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
 684                                                 &data->last_mtc);
 685                 }
 686                 if (mtc > data->last_mtc)
 687                         mtc_delta = mtc - data->last_mtc;
 688                 else
 689                         mtc_delta = mtc + 256 - data->last_mtc;
 690                 data->ctc_delta += mtc_delta << decoder->mtc_shift;
 691                 data->last_mtc = mtc;
 692
 693                 if (decoder->tsc_ctc_mult) {
 694                         timestamp = data->ctc_timestamp +
 695                                 data->ctc_delta * decoder->tsc_ctc_mult;
 696                 } else {
 697                         timestamp = data->ctc_timestamp +
 698                                 multdiv(data->ctc_delta,
 699                                         decoder->tsc_ctc_ratio_n,
 700                                         decoder->tsc_ctc_ratio_d);
 701                 }
 702
 703                 if (timestamp < data->timestamp)
 704                         return 1;
 705
 706                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 707                         data->timestamp = timestamp;
 708                         return 0;
 709                 }
 710
 711                 break;
 712
 713         case INTEL_PT_TSC:
 714                 /*
 715                  * For now, do not support using TSC packets - refer
 716                  * intel_pt_calc_cyc_to_tsc().
 717                  */
 718                 if (data->from_mtc)
 719                         return 1;
 720                 timestamp = pkt_info->packet.payload |
 721                             (data->timestamp & (0xffULL << 56));
 722                 if (data->from_mtc && timestamp < data->timestamp &&
 723                     data->timestamp - timestamp < decoder->tsc_slip)
 724                         return 1;
 725                 if (timestamp < data->timestamp)
 726                         timestamp += (1ULL << 56);
 727                 if (pkt_info->last_packet_type != INTEL_PT_CYC) {
 728                         if (data->from_mtc)
 729                                 return 1;
 730                         data->tsc_timestamp = timestamp;
 731                         data->timestamp = timestamp;
 732                         return 0;
 733                 }
 734                 break;
 735
 736         case INTEL_PT_TMA:
 737                 if (data->from_mtc)
 738                         return 1;
 739
 740                 if (!decoder->tsc_ctc_ratio_d)
 741                         return 0;
 742
 743                 ctc = pkt_info->packet.payload;
 744                 fc = pkt_info->packet.count;
 745                 ctc_rem = ctc & decoder->ctc_rem_mask;
 746
 747                 data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
 748
 749                 data->ctc_timestamp = data->tsc_timestamp - fc;
 750                 if (decoder->tsc_ctc_mult) {
 751                         data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
 752                 } else {
 753                         data->ctc_timestamp -=
 754                                 multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,
 755                                         decoder->tsc_ctc_ratio_d);
 756                 }
 757
 758                 data->ctc_delta = 0;
 759                 data->have_tma = true;
 760                 data->fixup_last_mtc = true;
 761
 762                 return 0;
 763
 764         case INTEL_PT_CYC:
 765                 data->cycle_cnt += pkt_info->packet.payload;
 766                 return 0;
 767
 768         case INTEL_PT_CBR:
 769                 cbr = pkt_info->packet.payload;
 770                 if (data->cbr && data->cbr != cbr)
 771                         return 1;
 772                 data->cbr = cbr;
 773                 data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
 774                 return 0;
 775
 776         case INTEL_PT_TIP_PGD:
 777         case INTEL_PT_TRACESTOP:
 778         case INTEL_PT_EXSTOP:
 779         case INTEL_PT_EXSTOP_IP:
 780         case INTEL_PT_MWAIT:
 781         case INTEL_PT_PWRE:
 782         case INTEL_PT_PWRX:
 783         case INTEL_PT_OVF:
 784         case INTEL_PT_BAD: /* Does not happen */
 785         default:
 786                 return 1;
 787         }
 788
 789         if (!data->cbr && decoder->cbr) {
 790                 data->cbr = decoder->cbr;
 791                 data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;
 792         }
 793
 794         if (!data->cycle_cnt)
 795                 return 1;
 796
 797         cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;
 798
 799         if (data->cbr && cyc_to_tsc > data->cbr_cyc_to_tsc &&
 800             cyc_to_tsc / data->cbr_cyc_to_tsc > 1.25) {
 801                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle too big (c.f. CBR-based value %g), pos " x64_fmt "\n",
 802                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 803                 return 1;
 804         }
 805
 806         decoder->calc_cyc_to_tsc = cyc_to_tsc;
 807         decoder->have_calc_cyc_to_tsc = true;
 808
 809         if (data->cbr) {
 810                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. CBR-based value %g, pos " x64_fmt "\n",
 811                              cyc_to_tsc, data->cbr_cyc_to_tsc, pkt_info->pos);
 812         } else {
 813                 intel_pt_log("Timestamp: calculated %g TSC ticks per cycle c.f. unknown CBR-based value, pos " x64_fmt "\n",
 814                              cyc_to_tsc, pkt_info->pos);
 815         }
 816
 817         return 1;
 818 }
 819
 820 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,
 821                                      bool from_mtc)
 822 {
 823         struct intel_pt_calc_cyc_to_tsc_info data = {
 824                 .cycle_cnt      = 0,
 825                 .cbr            = 0,
 826                 .last_mtc       = decoder->last_mtc,
 827                 .ctc_timestamp  = decoder->ctc_timestamp,
 828                 .ctc_delta      = decoder->ctc_delta,
 829                 .tsc_timestamp  = decoder->tsc_timestamp,
 830                 .timestamp      = decoder->timestamp,
 831                 .have_tma       = decoder->have_tma,
 832                 .fixup_last_mtc = decoder->fixup_last_mtc,
 833                 .from_mtc       = from_mtc,
 834                 .cbr_cyc_to_tsc = 0,
 835         };
 836
 837         /*
 838          * For now, do not support using TSC packets for at least the reasons:
 839          * 1) timing might have stopped
 840          * 2) TSC packets within PSB+ can slip against CYC packets
 841          */
 842         if (!from_mtc)
 843                 return;
 844
 845         intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);
 846 }
 847
 848 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
 849 {
 850         int ret;
 851
 852         decoder->last_packet_type = decoder->packet.type;
 853
 854         do {
 855                 decoder->pos += decoder->pkt_step;
 856                 decoder->buf += decoder->pkt_step;
 857                 decoder->len -= decoder->pkt_step;
 858
 859                 if (!decoder->len) {
 860                         ret = intel_pt_get_next_data(decoder);
 861                         if (ret)
 862                                 return ret;
 863                 }
 864
 865                 ret = intel_pt_get_packet(decoder->buf, decoder->len,
 866                                           &decoder->packet);
 867                 if (ret == INTEL_PT_NEED_MORE_BYTES && BITS_PER_LONG == 32 &&
 868                     decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
 869                         ret = intel_pt_get_split_packet(decoder);
 870                         if (ret < 0)
 871                                 return ret;
 872                 }
 873                 if (ret <= 0)
 874                         return intel_pt_bad_packet(decoder);
 875
 876                 decoder->pkt_len = ret;
 877                 decoder->pkt_step = ret;
 878                 intel_pt_decoder_log_packet(decoder);
 879         } while (decoder->packet.type == INTEL_PT_PAD);
 880
 881         return 0;
 882 }
 883
 884 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
 885 {
 886         uint64_t timestamp, masked_timestamp;
 887
 888         timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 889         masked_timestamp = timestamp & decoder->period_mask;
 890         if (decoder->continuous_period) {
 891                 if (masked_timestamp > decoder->last_masked_timestamp)
 892                         return 1;
 893         } else {
 894                 timestamp += 1;
 895                 masked_timestamp = timestamp & decoder->period_mask;
 896                 if (masked_timestamp > decoder->last_masked_timestamp) {
 897                         decoder->last_masked_timestamp = masked_timestamp;
 898                         decoder->continuous_period = true;
 899                 }
 900         }
 901
 902         if (masked_timestamp < decoder->last_masked_timestamp)
 903                 return decoder->period_ticks;
 904
 905         return decoder->period_ticks - (timestamp - masked_timestamp);
 906 }
 907
 908 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
 909 {
 910         switch (decoder->period_type) {
 911         case INTEL_PT_PERIOD_INSTRUCTIONS:
 912                 return decoder->period - decoder->period_insn_cnt;
 913         case INTEL_PT_PERIOD_TICKS:
 914                 return intel_pt_next_period(decoder);
 915         case INTEL_PT_PERIOD_NONE:
 916         case INTEL_PT_PERIOD_MTC:
 917         default:
 918                 return 0;
 919         }
 920 }
 921
 922 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
 923 {
 924         uint64_t timestamp, masked_timestamp;
 925
 926         switch (decoder->period_type) {
 927         case INTEL_PT_PERIOD_INSTRUCTIONS:
 928                 decoder->period_insn_cnt = 0;
 929                 break;
 930         case INTEL_PT_PERIOD_TICKS:
 931                 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
 932                 masked_timestamp = timestamp & decoder->period_mask;
 933                 if (masked_timestamp > decoder->last_masked_timestamp)
 934                         decoder->last_masked_timestamp = masked_timestamp;
 935                 else
 936                         decoder->last_masked_timestamp += decoder->period_ticks;
 937                 break;
 938         case INTEL_PT_PERIOD_NONE:
 939         case INTEL_PT_PERIOD_MTC:
 940         default:
 941                 break;
 942         }
 943
 944         decoder->state.type |= INTEL_PT_INSTRUCTION;
 945 }
 946
 947 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
 948                               struct intel_pt_insn *intel_pt_insn, uint64_t ip)
 949 {
 950         uint64_t max_insn_cnt, insn_cnt = 0;
 951         int err;
 952
 953         if (!decoder->mtc_insn)
 954                 decoder->mtc_insn = true;
 955
 956         max_insn_cnt = intel_pt_next_sample(decoder);
 957
 958         err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
 959                                  max_insn_cnt, decoder->data);
 960
 961         decoder->tot_insn_cnt += insn_cnt;
 962         decoder->timestamp_insn_cnt += insn_cnt;
 963         decoder->sample_insn_cnt += insn_cnt;
 964         decoder->period_insn_cnt += insn_cnt;
 965
 966         if (err) {
 967                 decoder->no_progress = 0;
 968                 decoder->pkt_state = INTEL_PT_STATE_ERR2;
 969                 intel_pt_log_at("ERROR: Failed to get instruction",
 970                                 decoder->ip);
 971                 if (err == -ENOENT)
 972                         return -ENOLINK;
 973                 return -EILSEQ;
 974         }
 975
 976         if (ip && decoder->ip == ip) {
 977                 err = -EAGAIN;
 978                 goto out;
 979         }
 980
 981         if (max_insn_cnt && insn_cnt >= max_insn_cnt)
 982                 intel_pt_sample_insn(decoder);
 983
 984         if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
 985                 decoder->state.type = INTEL_PT_INSTRUCTION;
 986                 decoder->state.from_ip = decoder->ip;
 987                 decoder->state.to_ip = 0;
 988                 decoder->ip += intel_pt_insn->length;
 989                 err = INTEL_PT_RETURN;
 990                 goto out;
 991         }
 992
 993         if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
 994                 /* Zero-length calls are excluded */
 995                 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
 996                     intel_pt_insn->rel) {
 997                         err = intel_pt_push(&decoder->stack, decoder->ip +
 998                                             intel_pt_insn->length);
 999                         if (err)
1000                                 goto out;
1001                 }
1002         } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
1003                 decoder->ret_addr = intel_pt_pop(&decoder->stack);
1004         }
1005
1006         if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
1007                 int cnt = decoder->no_progress++;
1008
1009                 decoder->state.from_ip = decoder->ip;
1010                 decoder->ip += intel_pt_insn->length +
1011                                 intel_pt_insn->rel;
1012                 decoder->state.to_ip = decoder->ip;
1013                 err = INTEL_PT_RETURN;
1014
1015                 /*
1016                  * Check for being stuck in a loop.  This can happen if a
1017                  * decoder error results in the decoder erroneously setting the
1018                  * ip to an address that is itself in an infinite loop that
1019                  * consumes no packets.  When that happens, there must be an
1020                  * unconditional branch.
1021                  */
1022                 if (cnt) {
1023                         if (cnt == 1) {
1024                                 decoder->stuck_ip = decoder->state.to_ip;
1025                                 decoder->stuck_ip_prd = 1;
1026                                 decoder->stuck_ip_cnt = 1;
1027                         } else if (cnt > INTEL_PT_MAX_LOOPS ||
1028                                    decoder->state.to_ip == decoder->stuck_ip) {
1029                                 intel_pt_log_at("ERROR: Never-ending loop",
1030                                                 decoder->state.to_ip);
1031                                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1032                                 err = -ELOOP;
1033                                 goto out;
1034                         } else if (!--decoder->stuck_ip_cnt) {
1035                                 decoder->stuck_ip_prd += 1;
1036                                 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
1037                                 decoder->stuck_ip = decoder->state.to_ip;
1038                         }
1039                 }
1040                 goto out_no_progress;
1041         }
1042 out:
1043         decoder->no_progress = 0;
1044 out_no_progress:
1045         decoder->state.insn_op = intel_pt_insn->op;
1046         decoder->state.insn_len = intel_pt_insn->length;
1047         memcpy(decoder->state.insn, intel_pt_insn->buf,
1048                INTEL_PT_INSN_BUF_SZ);
1049
1050         if (decoder->tx_flags & INTEL_PT_IN_TX)
1051                 decoder->state.flags |= INTEL_PT_IN_TX;
1052
1053         return err;
1054 }
1055
1056 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)
1057 {
1058         bool ret = false;
1059
1060         if (decoder->set_fup_tx_flags) {
1061                 decoder->set_fup_tx_flags = false;
1062                 decoder->tx_flags = decoder->fup_tx_flags;
1063                 decoder->state.type = INTEL_PT_TRANSACTION;
1064                 decoder->state.from_ip = decoder->ip;
1065                 decoder->state.to_ip = 0;
1066                 decoder->state.flags = decoder->fup_tx_flags;
1067                 return true;
1068         }
1069         if (decoder->set_fup_ptw) {
1070                 decoder->set_fup_ptw = false;
1071                 decoder->state.type = INTEL_PT_PTW;
1072                 decoder->state.flags |= INTEL_PT_FUP_IP;
1073                 decoder->state.from_ip = decoder->ip;
1074                 decoder->state.to_ip = 0;
1075                 decoder->state.ptw_payload = decoder->fup_ptw_payload;
1076                 return true;
1077         }
1078         if (decoder->set_fup_mwait) {
1079                 decoder->set_fup_mwait = false;
1080                 decoder->state.type = INTEL_PT_MWAIT_OP;
1081                 decoder->state.from_ip = decoder->ip;
1082                 decoder->state.to_ip = 0;
1083                 decoder->state.mwait_payload = decoder->fup_mwait_payload;
1084                 ret = true;
1085         }
1086         if (decoder->set_fup_pwre) {
1087                 decoder->set_fup_pwre = false;
1088                 decoder->state.type |= INTEL_PT_PWR_ENTRY;
1089                 decoder->state.type &= ~INTEL_PT_BRANCH;
1090                 decoder->state.from_ip = decoder->ip;
1091                 decoder->state.to_ip = 0;
1092                 decoder->state.pwre_payload = decoder->fup_pwre_payload;
1093                 ret = true;
1094         }
1095         if (decoder->set_fup_exstop) {
1096                 decoder->set_fup_exstop = false;
1097                 decoder->state.type |= INTEL_PT_EX_STOP;
1098                 decoder->state.type &= ~INTEL_PT_BRANCH;
1099                 decoder->state.flags |= INTEL_PT_FUP_IP;
1100                 decoder->state.from_ip = decoder->ip;
1101                 decoder->state.to_ip = 0;
1102                 ret = true;
1103         }
1104         return ret;
1105 }
1106
1107 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,
1108                                           struct intel_pt_insn *intel_pt_insn,
1109                                           uint64_t ip, int err)
1110 {
1111         return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&
1112                intel_pt_insn->branch == INTEL_PT_BR_INDIRECT &&
1113                ip == decoder->ip + intel_pt_insn->length;
1114 }
1115
1116 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
1117 {
1118         struct intel_pt_insn intel_pt_insn;
1119         uint64_t ip;
1120         int err;
1121
1122         ip = decoder->last_ip;
1123
1124         while (1) {
1125                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
1126                 if (err == INTEL_PT_RETURN)
1127                         return 0;
1128                 if (err == -EAGAIN ||
1129                     intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {
1130                         if (intel_pt_fup_event(decoder))
1131                                 return 0;
1132                         return -EAGAIN;
1133                 }
1134                 decoder->set_fup_tx_flags = false;
1135                 if (err)
1136                         return err;
1137
1138                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1139                         intel_pt_log_at("ERROR: Unexpected indirect branch",
1140                                         decoder->ip);
1141                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1142                         return -ENOENT;
1143                 }
1144
1145                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1146                         intel_pt_log_at("ERROR: Unexpected conditional branch",
1147                                         decoder->ip);
1148                         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1149                         return -ENOENT;
1150                 }
1151
1152                 intel_pt_bug(decoder);
1153         }
1154 }
1155
1156 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
1157 {
1158         struct intel_pt_insn intel_pt_insn;
1159         int err;
1160
1161         err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1162         if (err == INTEL_PT_RETURN &&
1163             decoder->pgd_ip &&
1164             decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1165             (decoder->state.type & INTEL_PT_BRANCH) &&
1166             decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {
1167                 /* Unconditional branch leaving filter region */
1168                 decoder->no_progress = 0;
1169                 decoder->pge = false;
1170                 decoder->continuous_period = false;
1171                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1172                 decoder->state.type |= INTEL_PT_TRACE_END;
1173                 return 0;
1174         }
1175         if (err == INTEL_PT_RETURN)
1176                 return 0;
1177         if (err)
1178                 return err;
1179
1180         if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1181                 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
1182                         decoder->pge = false;
1183                         decoder->continuous_period = false;
1184                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1185                         decoder->state.from_ip = decoder->ip;
1186                         if (decoder->packet.count == 0) {
1187                                 decoder->state.to_ip = 0;
1188                         } else {
1189                                 decoder->state.to_ip = decoder->last_ip;
1190                                 decoder->ip = decoder->last_ip;
1191                         }
1192                         decoder->state.type |= INTEL_PT_TRACE_END;
1193                 } else {
1194                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1195                         decoder->state.from_ip = decoder->ip;
1196                         if (decoder->packet.count == 0) {
1197                                 decoder->state.to_ip = 0;
1198                         } else {
1199                                 decoder->state.to_ip = decoder->last_ip;
1200                                 decoder->ip = decoder->last_ip;
1201                         }
1202                 }
1203                 return 0;
1204         }
1205
1206         if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1207                 uint64_t to_ip = decoder->ip + intel_pt_insn.length +
1208                                  intel_pt_insn.rel;
1209
1210                 if (decoder->pgd_ip &&
1211                     decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&
1212                     decoder->pgd_ip(to_ip, decoder->data)) {
1213                         /* Conditional branch leaving filter region */
1214                         decoder->pge = false;
1215                         decoder->continuous_period = false;
1216                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1217                         decoder->ip = to_ip;
1218                         decoder->state.from_ip = decoder->ip;
1219                         decoder->state.to_ip = to_ip;
1220                         decoder->state.type |= INTEL_PT_TRACE_END;
1221                         return 0;
1222                 }
1223                 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
1224                                 decoder->ip);
1225                 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1226                 return -ENOENT;
1227         }
1228
1229         return intel_pt_bug(decoder);
1230 }
1231
1232 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
1233 {
1234         struct intel_pt_insn intel_pt_insn;
1235         int err;
1236
1237         while (1) {
1238                 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
1239                 if (err == INTEL_PT_RETURN)
1240                         return 0;
1241                 if (err)
1242                         return err;
1243
1244                 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
1245                         if (!decoder->return_compression) {
1246                                 intel_pt_log_at("ERROR: RET when expecting conditional branch",
1247                                                 decoder->ip);
1248                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1249                                 return -ENOENT;
1250                         }
1251                         if (!decoder->ret_addr) {
1252                                 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
1253                                                 decoder->ip);
1254                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1255                                 return -ENOENT;
1256                         }
1257                         if (!(decoder->tnt.payload & BIT63)) {
1258                                 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
1259                                                 decoder->ip);
1260                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1261                                 return -ENOENT;
1262                         }
1263                         decoder->tnt.count -= 1;
1264                         if (!decoder->tnt.count)
1265                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1266                         decoder->tnt.payload <<= 1;
1267                         decoder->state.from_ip = decoder->ip;
1268                         decoder->ip = decoder->ret_addr;
1269                         decoder->state.to_ip = decoder->ip;
1270                         return 0;
1271                 }
1272
1273                 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
1274                         /* Handle deferred TIPs */
1275                         err = intel_pt_get_next_packet(decoder);
1276                         if (err)
1277                                 return err;
1278                         if (decoder->packet.type != INTEL_PT_TIP ||
1279                             decoder->packet.count == 0) {
1280                                 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
1281                                                 decoder->ip);
1282                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1283                                 decoder->pkt_step = 0;
1284                                 return -ENOENT;
1285                         }
1286                         intel_pt_set_last_ip(decoder);
1287                         decoder->state.from_ip = decoder->ip;
1288                         decoder->state.to_ip = decoder->last_ip;
1289                         decoder->ip = decoder->last_ip;
1290                         return 0;
1291                 }
1292
1293                 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
1294                         decoder->tnt.count -= 1;
1295                         if (!decoder->tnt.count)
1296                                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1297                         if (decoder->tnt.payload & BIT63) {
1298                                 decoder->tnt.payload <<= 1;
1299                                 decoder->state.from_ip = decoder->ip;
1300                                 decoder->ip += intel_pt_insn.length +
1301                                                intel_pt_insn.rel;
1302                                 decoder->state.to_ip = decoder->ip;
1303                                 return 0;
1304                         }
1305                         /* Instruction sample for a non-taken branch */
1306                         if (decoder->state.type & INTEL_PT_INSTRUCTION) {
1307                                 decoder->tnt.payload <<= 1;
1308                                 decoder->state.type = INTEL_PT_INSTRUCTION;
1309                                 decoder->state.from_ip = decoder->ip;
1310                                 decoder->state.to_ip = 0;
1311                                 decoder->ip += intel_pt_insn.length;
1312                                 return 0;
1313                         }
1314                         decoder->ip += intel_pt_insn.length;
1315                         if (!decoder->tnt.count)
1316                                 return -EAGAIN;
1317                         decoder->tnt.payload <<= 1;
1318                         continue;
1319                 }
1320
1321                 return intel_pt_bug(decoder);
1322         }
1323 }
1324
1325 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
1326 {
1327         unsigned int fup_tx_flags;
1328         int err;
1329
1330         fup_tx_flags = decoder->packet.payload &
1331                        (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
1332         err = intel_pt_get_next_packet(decoder);
1333         if (err)
1334                 return err;
1335         if (decoder->packet.type == INTEL_PT_FUP) {
1336                 decoder->fup_tx_flags = fup_tx_flags;
1337                 decoder->set_fup_tx_flags = true;
1338                 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
1339                         *no_tip = true;
1340         } else {
1341                 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
1342                                 decoder->pos);
1343                 intel_pt_update_in_tx(decoder);
1344         }
1345         return 0;
1346 }
1347
1348 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
1349 {
1350         uint64_t timestamp;
1351
1352         decoder->have_tma = false;
1353
1354         if (decoder->ref_timestamp) {
1355                 timestamp = decoder->packet.payload |
1356                             (decoder->ref_timestamp & (0xffULL << 56));
1357                 if (timestamp < decoder->ref_timestamp) {
1358                         if (decoder->ref_timestamp - timestamp > (1ULL << 55))
1359                                 timestamp += (1ULL << 56);
1360                 } else {
1361                         if (timestamp - decoder->ref_timestamp > (1ULL << 55))
1362                                 timestamp -= (1ULL << 56);
1363                 }
1364                 decoder->tsc_timestamp = timestamp;
1365                 decoder->timestamp = timestamp;
1366                 decoder->ref_timestamp = 0;
1367                 decoder->timestamp_insn_cnt = 0;
1368         } else if (decoder->timestamp) {
1369                 timestamp = decoder->packet.payload |
1370                             (decoder->timestamp & (0xffULL << 56));
1371                 decoder->tsc_timestamp = timestamp;
1372                 if (timestamp < decoder->timestamp &&
1373                     decoder->timestamp - timestamp < decoder->tsc_slip) {
1374                         intel_pt_log_to("Suppressing backwards timestamp",
1375                                         timestamp);
1376                         timestamp = decoder->timestamp;
1377                 }
1378                 if (timestamp < decoder->timestamp) {
1379                         intel_pt_log_to("Wraparound timestamp", timestamp);
1380                         timestamp += (1ULL << 56);
1381                         decoder->tsc_timestamp = timestamp;
1382                 }
1383                 decoder->timestamp = timestamp;
1384                 decoder->timestamp_insn_cnt = 0;
1385         }
1386
1387         if (decoder->last_packet_type == INTEL_PT_CYC) {
1388                 decoder->cyc_ref_timestamp = decoder->timestamp;
1389                 decoder->cycle_cnt = 0;
1390                 decoder->have_calc_cyc_to_tsc = false;
1391                 intel_pt_calc_cyc_to_tsc(decoder, false);
1392         }
1393
1394         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1395 }
1396
1397 static int intel_pt_overflow(struct intel_pt_decoder *decoder)
1398 {
1399         intel_pt_log("ERROR: Buffer overflow\n");
1400         intel_pt_clear_tx_flags(decoder);
1401         decoder->timestamp_insn_cnt = 0;
1402         decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
1403         decoder->overflow = true;
1404         return -EOVERFLOW;
1405 }
1406
1407 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)
1408 {
1409         uint32_t ctc = decoder->packet.payload;
1410         uint32_t fc = decoder->packet.count;
1411         uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;
1412
1413         if (!decoder->tsc_ctc_ratio_d)
1414                 return;
1415
1416         decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;
1417         decoder->ctc_timestamp = decoder->tsc_timestamp - fc;
1418         if (decoder->tsc_ctc_mult) {
1419                 decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;
1420         } else {
1421                 decoder->ctc_timestamp -= multdiv(ctc_rem,
1422                                                   decoder->tsc_ctc_ratio_n,
1423                                                   decoder->tsc_ctc_ratio_d);
1424         }
1425         decoder->ctc_delta = 0;
1426         decoder->have_tma = true;
1427         decoder->fixup_last_mtc = true;
1428         intel_pt_log("CTC timestamp " x64_fmt " last MTC %#x  CTC rem %#x\n",
1429                      decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);
1430 }
1431
1432 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)
1433 {
1434         uint64_t timestamp;
1435         uint32_t mtc, mtc_delta;
1436
1437         if (!decoder->have_tma)
1438                 return;
1439
1440         mtc = decoder->packet.payload;
1441
1442         if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {
1443                 decoder->fixup_last_mtc = false;
1444                 intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,
1445                                         &decoder->last_mtc);
1446         }
1447
1448         if (mtc > decoder->last_mtc)
1449                 mtc_delta = mtc - decoder->last_mtc;
1450         else
1451                 mtc_delta = mtc + 256 - decoder->last_mtc;
1452
1453         decoder->ctc_delta += mtc_delta << decoder->mtc_shift;
1454
1455         if (decoder->tsc_ctc_mult) {
1456                 timestamp = decoder->ctc_timestamp +
1457                             decoder->ctc_delta * decoder->tsc_ctc_mult;
1458         } else {
1459                 timestamp = decoder->ctc_timestamp +
1460                             multdiv(decoder->ctc_delta,
1461                                     decoder->tsc_ctc_ratio_n,
1462                                     decoder->tsc_ctc_ratio_d);
1463         }
1464
1465         if (timestamp < decoder->timestamp)
1466                 intel_pt_log("Suppressing MTC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1467                              timestamp, decoder->timestamp);
1468         else
1469                 decoder->timestamp = timestamp;
1470
1471         decoder->timestamp_insn_cnt = 0;
1472         decoder->last_mtc = mtc;
1473
1474         if (decoder->last_packet_type == INTEL_PT_CYC) {
1475                 decoder->cyc_ref_timestamp = decoder->timestamp;
1476                 decoder->cycle_cnt = 0;
1477                 decoder->have_calc_cyc_to_tsc = false;
1478                 intel_pt_calc_cyc_to_tsc(decoder, true);
1479         }
1480
1481         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1482 }
1483
1484 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)
1485 {
1486         unsigned int cbr = decoder->packet.payload & 0xff;
1487
1488         decoder->cbr_payload = decoder->packet.payload;
1489
1490         if (decoder->cbr == cbr)
1491                 return;
1492
1493         decoder->cbr = cbr;
1494         decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;
1495 }
1496
1497 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)
1498 {
1499         uint64_t timestamp = decoder->cyc_ref_timestamp;
1500
1501         decoder->have_cyc = true;
1502
1503         decoder->cycle_cnt += decoder->packet.payload;
1504
1505         if (!decoder->cyc_ref_timestamp)
1506                 return;
1507
1508         if (decoder->have_calc_cyc_to_tsc)
1509                 timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;
1510         else if (decoder->cbr)
1511                 timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;
1512         else
1513                 return;
1514
1515         if (timestamp < decoder->timestamp)
1516                 intel_pt_log("Suppressing CYC timestamp " x64_fmt " less than current timestamp " x64_fmt "\n",
1517                              timestamp, decoder->timestamp);
1518         else
1519                 decoder->timestamp = timestamp;
1520
1521         decoder->timestamp_insn_cnt = 0;
1522
1523         intel_pt_log_to("Setting timestamp", decoder->timestamp);
1524 }
1525
1526 /* Walk PSB+ packets when already in sync. */
1527 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
1528 {
1529         int err;
1530
1531         while (1) {
1532                 err = intel_pt_get_next_packet(decoder);
1533                 if (err)
1534                         return err;
1535
1536                 switch (decoder->packet.type) {
1537                 case INTEL_PT_PSBEND:
1538                         return 0;
1539
1540                 case INTEL_PT_TIP_PGD:
1541                 case INTEL_PT_TIP_PGE:
1542                 case INTEL_PT_TIP:
1543                 case INTEL_PT_TNT:
1544                 case INTEL_PT_TRACESTOP:
1545                 case INTEL_PT_BAD:
1546                 case INTEL_PT_PSB:
1547                 case INTEL_PT_PTWRITE:
1548                 case INTEL_PT_PTWRITE_IP:
1549                 case INTEL_PT_EXSTOP:
1550                 case INTEL_PT_EXSTOP_IP:
1551                 case INTEL_PT_MWAIT:
1552                 case INTEL_PT_PWRE:
1553                 case INTEL_PT_PWRX:
1554                         decoder->have_tma = false;
1555                         intel_pt_log("ERROR: Unexpected packet\n");
1556                         return -EAGAIN;
1557
1558                 case INTEL_PT_OVF:
1559                         return intel_pt_overflow(decoder);
1560
1561                 case INTEL_PT_TSC:
1562                         intel_pt_calc_tsc_timestamp(decoder);
1563                         break;
1564
1565                 case INTEL_PT_TMA:
1566                         intel_pt_calc_tma(decoder);
1567                         break;
1568
1569                 case INTEL_PT_CBR:
1570                         intel_pt_calc_cbr(decoder);
1571                         break;
1572
1573                 case INTEL_PT_MODE_EXEC:
1574                         decoder->exec_mode = decoder->packet.payload;
1575                         break;
1576
1577                 case INTEL_PT_PIP:
1578                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1579                         break;
1580
1581                 case INTEL_PT_FUP:
1582                         decoder->pge = true;
1583                         if (decoder->packet.count)
1584                                 intel_pt_set_last_ip(decoder);
1585                         break;
1586
1587                 case INTEL_PT_MODE_TSX:
1588                         intel_pt_update_in_tx(decoder);
1589                         break;
1590
1591                 case INTEL_PT_MTC:
1592                         intel_pt_calc_mtc_timestamp(decoder);
1593                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1594                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1595                         break;
1596
1597                 case INTEL_PT_CYC:
1598                 case INTEL_PT_VMCS:
1599                 case INTEL_PT_MNT:
1600                 case INTEL_PT_PAD:
1601                 default:
1602                         break;
1603                 }
1604         }
1605 }
1606
1607 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
1608 {
1609         int err;
1610
1611         if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
1612                 decoder->tx_flags = 0;
1613                 decoder->state.flags &= ~INTEL_PT_IN_TX;
1614                 decoder->state.flags |= INTEL_PT_ABORT_TX;
1615         } else {
1616                 decoder->state.flags |= INTEL_PT_ASYNC;
1617         }
1618
1619         while (1) {
1620                 err = intel_pt_get_next_packet(decoder);
1621                 if (err)
1622                         return err;
1623
1624                 switch (decoder->packet.type) {
1625                 case INTEL_PT_TNT:
1626                 case INTEL_PT_FUP:
1627                 case INTEL_PT_TRACESTOP:
1628                 case INTEL_PT_PSB:
1629                 case INTEL_PT_TSC:
1630                 case INTEL_PT_TMA:
1631                 case INTEL_PT_MODE_TSX:
1632                 case INTEL_PT_BAD:
1633                 case INTEL_PT_PSBEND:
1634                 case INTEL_PT_PTWRITE:
1635                 case INTEL_PT_PTWRITE_IP:
1636                 case INTEL_PT_EXSTOP:
1637                 case INTEL_PT_EXSTOP_IP:
1638                 case INTEL_PT_MWAIT:
1639                 case INTEL_PT_PWRE:
1640                 case INTEL_PT_PWRX:
1641                         intel_pt_log("ERROR: Missing TIP after FUP\n");
1642                         decoder->pkt_state = INTEL_PT_STATE_ERR3;
1643                         decoder->pkt_step = 0;
1644                         return -ENOENT;
1645
1646                 case INTEL_PT_CBR:
1647                         intel_pt_calc_cbr(decoder);
1648                         break;
1649
1650                 case INTEL_PT_OVF:
1651                         return intel_pt_overflow(decoder);
1652
1653                 case INTEL_PT_TIP_PGD:
1654                         decoder->state.from_ip = decoder->ip;
1655                         if (decoder->packet.count == 0) {
1656                                 decoder->state.to_ip = 0;
1657                         } else {
1658                                 intel_pt_set_ip(decoder);
1659                                 decoder->state.to_ip = decoder->ip;
1660                         }
1661                         decoder->pge = false;
1662                         decoder->continuous_period = false;
1663                         decoder->state.type |= INTEL_PT_TRACE_END;
1664                         return 0;
1665
1666                 case INTEL_PT_TIP_PGE:
1667                         decoder->pge = true;
1668                         intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1669                                      decoder->ip);
1670                         decoder->state.from_ip = 0;
1671                         if (decoder->packet.count == 0) {
1672                                 decoder->state.to_ip = 0;
1673                         } else {
1674                                 intel_pt_set_ip(decoder);
1675                                 decoder->state.to_ip = decoder->ip;
1676                         }
1677                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1678                         return 0;
1679
1680                 case INTEL_PT_TIP:
1681                         decoder->state.from_ip = decoder->ip;
1682                         if (decoder->packet.count == 0) {
1683                                 decoder->state.to_ip = 0;
1684                         } else {
1685                                 intel_pt_set_ip(decoder);
1686                                 decoder->state.to_ip = decoder->ip;
1687                         }
1688                         return 0;
1689
1690                 case INTEL_PT_PIP:
1691                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1692                         break;
1693
1694                 case INTEL_PT_MTC:
1695                         intel_pt_calc_mtc_timestamp(decoder);
1696                         if (decoder->period_type == INTEL_PT_PERIOD_MTC)
1697                                 decoder->state.type |= INTEL_PT_INSTRUCTION;
1698                         break;
1699
1700                 case INTEL_PT_CYC:
1701                         intel_pt_calc_cyc_timestamp(decoder);
1702                         break;
1703
1704                 case INTEL_PT_MODE_EXEC:
1705                         decoder->exec_mode = decoder->packet.payload;
1706                         break;
1707
1708                 case INTEL_PT_VMCS:
1709                 case INTEL_PT_MNT:
1710                 case INTEL_PT_PAD:
1711                         break;
1712
1713                 default:
1714                         return intel_pt_bug(decoder);
1715                 }
1716         }
1717 }
1718
1719 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1720 {
1721         bool no_tip = false;
1722         int err;
1723
1724         while (1) {
1725                 err = intel_pt_get_next_packet(decoder);
1726                 if (err)
1727                         return err;
1728 next:
1729                 switch (decoder->packet.type) {
1730                 case INTEL_PT_TNT:
1731                         if (!decoder->packet.count)
1732                                 break;
1733                         decoder->tnt = decoder->packet;
1734                         decoder->pkt_state = INTEL_PT_STATE_TNT;
1735                         err = intel_pt_walk_tnt(decoder);
1736                         if (err == -EAGAIN)
1737                                 break;
1738                         return err;
1739
1740                 case INTEL_PT_TIP_PGD:
1741                         if (decoder->packet.count != 0)
1742                                 intel_pt_set_last_ip(decoder);
1743                         decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1744                         return intel_pt_walk_tip(decoder);
1745
1746                 case INTEL_PT_TIP_PGE: {
1747                         decoder->pge = true;
1748                         if (decoder->packet.count == 0) {
1749                                 intel_pt_log_at("Skipping zero TIP.PGE",
1750                                                 decoder->pos);
1751                                 break;
1752                         }
1753                         intel_pt_set_ip(decoder);
1754                         decoder->state.from_ip = 0;
1755                         decoder->state.to_ip = decoder->ip;
1756                         decoder->state.type |= INTEL_PT_TRACE_BEGIN;
1757                         return 0;
1758                 }
1759
1760                 case INTEL_PT_OVF:
1761                         return intel_pt_overflow(decoder);
1762
1763                 case INTEL_PT_TIP:
1764                         if (decoder->packet.count != 0)
1765                                 intel_pt_set_last_ip(decoder);
1766                         decoder->pkt_state = INTEL_PT_STATE_TIP;
1767                         return intel_pt_walk_tip(decoder);
1768
1769                 case INTEL_PT_FUP:
1770                         if (decoder->packet.count == 0) {
1771                                 intel_pt_log_at("Skipping zero FUP",
1772                                                 decoder->pos);
1773                                 no_tip = false;
1774                                 break;
1775                         }
1776                         intel_pt_set_last_ip(decoder);
1777                         if (!decoder->branch_enable) {
1778                                 decoder->ip = decoder->last_ip;
1779                                 if (intel_pt_fup_event(decoder))
1780                                         return 0;
1781                                 no_tip = false;
1782                                 break;
1783                         }
1784                         if (decoder->set_fup_mwait)
1785                                 no_tip = true;
1786                         err = intel_pt_walk_fup(decoder);
1787                         if (err != -EAGAIN) {
1788                                 if (err)
1789                                         return err;
1790                                 if (no_tip)
1791                                         decoder->pkt_state =
1792                                                 INTEL_PT_STATE_FUP_NO_TIP;
1793                                 else
1794                                         decoder->pkt_state = INTEL_PT_STATE_FUP;
1795                                 return 0;
1796                         }
1797                         if (no_tip) {
1798                                 no_tip = false;
1799                                 break;
1800                         }
1801                         return intel_pt_walk_fup_tip(decoder);
1802
1803                 case INTEL_PT_TRACESTOP:
1804                         decoder->pge = false;
1805                         decoder->continuous_period = false;
1806                         intel_pt_clear_tx_flags(decoder);
1807                         decoder->have_tma = false;
1808                         break;
1809
1810                 case INTEL_PT_PSB:
1811                         decoder->last_ip = 0;
1812                         decoder->have_last_ip = true;
1813                         intel_pt_clear_stack(&decoder->stack);
1814                         err = intel_pt_walk_psbend(decoder);
1815                         if (err == -EAGAIN)
1816                                 goto next;
1817                         if (err)
1818                                 return err;
1819                         break;
1820
1821                 case INTEL_PT_PIP:
1822                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
1823                         break;
1824
1825                 case INTEL_PT_MTC:
1826                         intel_pt_calc_mtc_timestamp(decoder);
1827                         if (decoder->period_type != INTEL_PT_PERIOD_MTC)
1828                                 break;
1829                         /*
1830                          * Ensure that there has been an instruction since the
1831                          * last MTC.
1832                          */
1833                         if (!decoder->mtc_insn)
1834                                 break;
1835                         decoder->mtc_insn = false;
1836                         /* Ensure that there is a timestamp */
1837                         if (!decoder->timestamp)
1838                                 break;
1839                         decoder->state.type = INTEL_PT_INSTRUCTION;
1840                         decoder->state.from_ip = decoder->ip;
1841                         decoder->state.to_ip = 0;
1842                         decoder->mtc_insn = false;
1843                         return 0;
1844
1845                 case INTEL_PT_TSC:
1846                         intel_pt_calc_tsc_timestamp(decoder);
1847                         break;
1848
1849                 case INTEL_PT_TMA:
1850                         intel_pt_calc_tma(decoder);
1851                         break;
1852
1853                 case INTEL_PT_CYC:
1854                         intel_pt_calc_cyc_timestamp(decoder);
1855                         break;
1856
1857                 case INTEL_PT_CBR:
1858                         intel_pt_calc_cbr(decoder);
1859                         if (!decoder->branch_enable &&
1860                             decoder->cbr != decoder->cbr_seen) {
1861                                 decoder->cbr_seen = decoder->cbr;
1862                                 decoder->state.type = INTEL_PT_CBR_CHG;
1863                                 decoder->state.from_ip = decoder->ip;
1864                                 decoder->state.to_ip = 0;
1865                                 decoder->state.cbr_payload =
1866                                                         decoder->packet.payload;
1867                                 return 0;
1868                         }
1869                         break;
1870
1871                 case INTEL_PT_MODE_EXEC:
1872                         decoder->exec_mode = decoder->packet.payload;
1873                         break;
1874
1875                 case INTEL_PT_MODE_TSX:
1876                         /* MODE_TSX need not be followed by FUP */
1877                         if (!decoder->pge) {
1878                                 intel_pt_update_in_tx(decoder);
1879                                 break;
1880                         }
1881                         err = intel_pt_mode_tsx(decoder, &no_tip);
1882                         if (err)
1883                                 return err;
1884                         goto next;
1885
1886                 case INTEL_PT_BAD: /* Does not happen */
1887                         return intel_pt_bug(decoder);
1888
1889                 case INTEL_PT_PSBEND:
1890                 case INTEL_PT_VMCS:
1891                 case INTEL_PT_MNT:
1892                 case INTEL_PT_PAD:
1893                         break;
1894
1895                 case INTEL_PT_PTWRITE_IP:
1896                         decoder->fup_ptw_payload = decoder->packet.payload;
1897                         err = intel_pt_get_next_packet(decoder);
1898                         if (err)
1899                                 return err;
1900                         if (decoder->packet.type == INTEL_PT_FUP) {
1901                                 decoder->set_fup_ptw = true;
1902                                 no_tip = true;
1903                         } else {
1904                                 intel_pt_log_at("ERROR: Missing FUP after PTWRITE",
1905                                                 decoder->pos);
1906                         }
1907                         goto next;
1908
1909                 case INTEL_PT_PTWRITE:
1910                         decoder->state.type = INTEL_PT_PTW;
1911                         decoder->state.from_ip = decoder->ip;
1912                         decoder->state.to_ip = 0;
1913                         decoder->state.ptw_payload = decoder->packet.payload;
1914                         return 0;
1915
1916                 case INTEL_PT_MWAIT:
1917                         decoder->fup_mwait_payload = decoder->packet.payload;
1918                         decoder->set_fup_mwait = true;
1919                         break;
1920
1921                 case INTEL_PT_PWRE:
1922                         if (decoder->set_fup_mwait) {
1923                                 decoder->fup_pwre_payload =
1924                                                         decoder->packet.payload;
1925                                 decoder->set_fup_pwre = true;
1926                                 break;
1927                         }
1928                         decoder->state.type = INTEL_PT_PWR_ENTRY;
1929                         decoder->state.from_ip = decoder->ip;
1930                         decoder->state.to_ip = 0;
1931                         decoder->state.pwrx_payload = decoder->packet.payload;
1932                         return 0;
1933
1934                 case INTEL_PT_EXSTOP_IP:
1935                         err = intel_pt_get_next_packet(decoder);
1936                         if (err)
1937                                 return err;
1938                         if (decoder->packet.type == INTEL_PT_FUP) {
1939                                 decoder->set_fup_exstop = true;
1940                                 no_tip = true;
1941                         } else {
1942                                 intel_pt_log_at("ERROR: Missing FUP after EXSTOP",
1943                                                 decoder->pos);
1944                         }
1945                         goto next;
1946
1947                 case INTEL_PT_EXSTOP:
1948                         decoder->state.type = INTEL_PT_EX_STOP;
1949                         decoder->state.from_ip = decoder->ip;
1950                         decoder->state.to_ip = 0;
1951                         return 0;
1952
1953                 case INTEL_PT_PWRX:
1954                         decoder->state.type = INTEL_PT_PWR_EXIT;
1955                         decoder->state.from_ip = decoder->ip;
1956                         decoder->state.to_ip = 0;
1957                         decoder->state.pwrx_payload = decoder->packet.payload;
1958                         return 0;
1959
1960                 default:
1961                         return intel_pt_bug(decoder);
1962                 }
1963         }
1964 }
1965
1966 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)
1967 {
1968         return decoder->packet.count &&
1969                (decoder->have_last_ip || decoder->packet.count == 3 ||
1970                 decoder->packet.count == 6);
1971 }
1972
1973 /* Walk PSB+ packets to get in sync. */
1974 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1975 {
1976         int err;
1977
1978         while (1) {
1979                 err = intel_pt_get_next_packet(decoder);
1980                 if (err)
1981                         return err;
1982
1983                 switch (decoder->packet.type) {
1984                 case INTEL_PT_TIP_PGD:
1985                         decoder->continuous_period = false;
1986                         __fallthrough;
1987                 case INTEL_PT_TIP_PGE:
1988                 case INTEL_PT_TIP:
1989                 case INTEL_PT_PTWRITE:
1990                 case INTEL_PT_PTWRITE_IP:
1991                 case INTEL_PT_EXSTOP:
1992                 case INTEL_PT_EXSTOP_IP:
1993                 case INTEL_PT_MWAIT:
1994                 case INTEL_PT_PWRE:
1995                 case INTEL_PT_PWRX:
1996                         intel_pt_log("ERROR: Unexpected packet\n");
1997                         return -ENOENT;
1998
1999                 case INTEL_PT_FUP:
2000                         decoder->pge = true;
2001                         if (intel_pt_have_ip(decoder)) {
2002                                 uint64_t current_ip = decoder->ip;
2003
2004                                 intel_pt_set_ip(decoder);
2005                                 if (current_ip)
2006                                         intel_pt_log_to("Setting IP",
2007                                                         decoder->ip);
2008                         }
2009                         break;
2010
2011                 case INTEL_PT_MTC:
2012                         intel_pt_calc_mtc_timestamp(decoder);
2013                         break;
2014
2015                 case INTEL_PT_TSC:
2016                         intel_pt_calc_tsc_timestamp(decoder);
2017                         break;
2018
2019                 case INTEL_PT_TMA:
2020                         intel_pt_calc_tma(decoder);
2021                         break;
2022
2023                 case INTEL_PT_CYC:
2024                         intel_pt_calc_cyc_timestamp(decoder);
2025                         break;
2026
2027                 case INTEL_PT_CBR:
2028                         intel_pt_calc_cbr(decoder);
2029                         break;
2030
2031                 case INTEL_PT_PIP:
2032                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2033                         break;
2034
2035                 case INTEL_PT_MODE_EXEC:
2036                         decoder->exec_mode = decoder->packet.payload;
2037                         break;
2038
2039                 case INTEL_PT_MODE_TSX:
2040                         intel_pt_update_in_tx(decoder);
2041                         break;
2042
2043                 case INTEL_PT_TRACESTOP:
2044                         decoder->pge = false;
2045                         decoder->continuous_period = false;
2046                         intel_pt_clear_tx_flags(decoder);
2047                         __fallthrough;
2048
2049                 case INTEL_PT_TNT:
2050                         decoder->have_tma = false;
2051                         intel_pt_log("ERROR: Unexpected packet\n");
2052                         if (decoder->ip)
2053                                 decoder->pkt_state = INTEL_PT_STATE_ERR4;
2054                         else
2055                                 decoder->pkt_state = INTEL_PT_STATE_ERR3;
2056                         return -ENOENT;
2057
2058                 case INTEL_PT_BAD: /* Does not happen */
2059                         return intel_pt_bug(decoder);
2060
2061                 case INTEL_PT_OVF:
2062                         return intel_pt_overflow(decoder);
2063
2064                 case INTEL_PT_PSBEND:
2065                         return 0;
2066
2067                 case INTEL_PT_PSB:
2068                 case INTEL_PT_VMCS:
2069                 case INTEL_PT_MNT:
2070                 case INTEL_PT_PAD:
2071                 default:
2072                         break;
2073                 }
2074         }
2075 }
2076
2077 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
2078 {
2079         int err;
2080
2081         while (1) {
2082                 err = intel_pt_get_next_packet(decoder);
2083                 if (err)
2084                         return err;
2085
2086                 switch (decoder->packet.type) {
2087                 case INTEL_PT_TIP_PGD:
2088                         decoder->continuous_period = false;
2089                         __fallthrough;
2090                 case INTEL_PT_TIP_PGE:
2091                 case INTEL_PT_TIP:
2092                         decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
2093                         if (intel_pt_have_ip(decoder))
2094                                 intel_pt_set_ip(decoder);
2095                         if (!decoder->ip)
2096                                 break;
2097                         if (decoder->packet.type == INTEL_PT_TIP_PGE)
2098                                 decoder->state.type |= INTEL_PT_TRACE_BEGIN;
2099                         if (decoder->packet.type == INTEL_PT_TIP_PGD)
2100                                 decoder->state.type |= INTEL_PT_TRACE_END;
2101                         return 0;
2102
2103                 case INTEL_PT_FUP:
2104                         if (intel_pt_have_ip(decoder))
2105                                 intel_pt_set_ip(decoder);
2106                         if (decoder->ip)
2107                                 return 0;
2108                         break;
2109
2110                 case INTEL_PT_MTC:
2111                         intel_pt_calc_mtc_timestamp(decoder);
2112                         break;
2113
2114                 case INTEL_PT_TSC:
2115                         intel_pt_calc_tsc_timestamp(decoder);
2116                         break;
2117
2118                 case INTEL_PT_TMA:
2119                         intel_pt_calc_tma(decoder);
2120                         break;
2121
2122                 case INTEL_PT_CYC:
2123                         intel_pt_calc_cyc_timestamp(decoder);
2124                         break;
2125
2126                 case INTEL_PT_CBR:
2127                         intel_pt_calc_cbr(decoder);
2128                         break;
2129
2130                 case INTEL_PT_PIP:
2131                         decoder->cr3 = decoder->packet.payload & (BIT63 - 1);
2132                         break;
2133
2134                 case INTEL_PT_MODE_EXEC:
2135                         decoder->exec_mode = decoder->packet.payload;
2136                         break;
2137
2138                 case INTEL_PT_MODE_TSX:
2139                         intel_pt_update_in_tx(decoder);
2140                         break;
2141
2142                 case INTEL_PT_OVF:
2143                         return intel_pt_overflow(decoder);
2144
2145                 case INTEL_PT_BAD: /* Does not happen */
2146                         return intel_pt_bug(decoder);
2147
2148                 case INTEL_PT_TRACESTOP:
2149                         decoder->pge = false;
2150                         decoder->continuous_period = false;
2151                         intel_pt_clear_tx_flags(decoder);
2152                         decoder->have_tma = false;
2153                         break;
2154
2155                 case INTEL_PT_PSB:
2156                         decoder->last_ip = 0;
2157                         decoder->have_last_ip = true;
2158                         intel_pt_clear_stack(&decoder->stack);
2159                         err = intel_pt_walk_psb(decoder);
2160                         if (err)
2161                                 return err;
2162                         if (decoder->ip) {
2163                                 /* Do not have a sample */
2164                                 decoder->state.type = 0;
2165                                 return 0;
2166                         }
2167                         break;
2168
2169                 case INTEL_PT_TNT:
2170                 case INTEL_PT_PSBEND:
2171                 case INTEL_PT_VMCS:
2172                 case INTEL_PT_MNT:
2173                 case INTEL_PT_PAD:
2174                 case INTEL_PT_PTWRITE:
2175                 case INTEL_PT_PTWRITE_IP:
2176                 case INTEL_PT_EXSTOP:
2177                 case INTEL_PT_EXSTOP_IP:
2178                 case INTEL_PT_MWAIT:
2179                 case INTEL_PT_PWRE:
2180                 case INTEL_PT_PWRX:
2181                 default:
2182                         break;
2183                 }
2184         }
2185 }
2186
2187 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
2188 {
2189         int err;
2190
2191         decoder->set_fup_tx_flags = false;
2192         decoder->set_fup_ptw = false;
2193         decoder->set_fup_mwait = false;
2194         decoder->set_fup_pwre = false;
2195         decoder->set_fup_exstop = false;
2196
2197         if (!decoder->branch_enable) {
2198                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2199                 decoder->overflow = false;
2200                 decoder->state.type = 0; /* Do not have a sample */
2201                 return 0;
2202         }
2203
2204         intel_pt_log("Scanning for full IP\n");
2205         err = intel_pt_walk_to_ip(decoder);
2206         if (err)
2207                 return err;
2208
2209         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2210         decoder->overflow = false;
2211
2212         decoder->state.from_ip = 0;
2213         decoder->state.to_ip = decoder->ip;
2214         intel_pt_log_to("Setting IP", decoder->ip);
2215
2216         return 0;
2217 }
2218
2219 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
2220 {
2221         const unsigned char *end = decoder->buf + decoder->len;
2222         size_t i;
2223
2224         for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
2225                 if (i > decoder->len)
2226                         continue;
2227                 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
2228                         return i;
2229         }
2230         return 0;
2231 }
2232
2233 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
2234 {
2235         size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
2236         const char *psb = INTEL_PT_PSB_STR;
2237
2238         if (rest_psb > decoder->len ||
2239             memcmp(decoder->buf, psb + part_psb, rest_psb))
2240                 return 0;
2241
2242         return rest_psb;
2243 }
2244
2245 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
2246                                   int part_psb)
2247 {
2248         int rest_psb, ret;
2249
2250         decoder->pos += decoder->len;
2251         decoder->len = 0;
2252
2253         ret = intel_pt_get_next_data(decoder);
2254         if (ret)
2255                 return ret;
2256
2257         rest_psb = intel_pt_rest_psb(decoder, part_psb);
2258         if (!rest_psb)
2259                 return 0;
2260
2261         decoder->pos -= part_psb;
2262         decoder->next_buf = decoder->buf + rest_psb;
2263         decoder->next_len = decoder->len - rest_psb;
2264         memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2265         decoder->buf = decoder->temp_buf;
2266         decoder->len = INTEL_PT_PSB_LEN;
2267
2268         return 0;
2269 }
2270
2271 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
2272 {
2273         unsigned char *next;
2274         int ret;
2275
2276         intel_pt_log("Scanning for PSB\n");
2277         while (1) {
2278                 if (!decoder->len) {
2279                         ret = intel_pt_get_next_data(decoder);
2280                         if (ret)
2281                                 return ret;
2282                 }
2283
2284                 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
2285                               INTEL_PT_PSB_LEN);
2286                 if (!next) {
2287                         int part_psb;
2288
2289                         part_psb = intel_pt_part_psb(decoder);
2290                         if (part_psb) {
2291                                 ret = intel_pt_get_split_psb(decoder, part_psb);
2292                                 if (ret)
2293                                         return ret;
2294                         } else {
2295                                 decoder->pos += decoder->len;
2296                                 decoder->len = 0;
2297                         }
2298                         continue;
2299                 }
2300
2301                 decoder->pkt_step = next - decoder->buf;
2302                 return intel_pt_get_next_packet(decoder);
2303         }
2304 }
2305
2306 static int intel_pt_sync(struct intel_pt_decoder *decoder)
2307 {
2308         int err;
2309
2310         decoder->pge = false;
2311         decoder->continuous_period = false;
2312         decoder->have_last_ip = false;
2313         decoder->last_ip = 0;
2314         decoder->ip = 0;
2315         intel_pt_clear_stack(&decoder->stack);
2316
2317         err = intel_pt_scan_for_psb(decoder);
2318         if (err)
2319                 return err;
2320
2321         decoder->have_last_ip = true;
2322         decoder->pkt_state = INTEL_PT_STATE_NO_IP;
2323
2324         err = intel_pt_walk_psb(decoder);
2325         if (err)
2326                 return err;
2327
2328         if (decoder->ip) {
2329                 decoder->state.type = 0; /* Do not have a sample */
2330                 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2331         } else {
2332                 return intel_pt_sync_ip(decoder);
2333         }
2334
2335         return 0;
2336 }
2337
2338 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
2339 {
2340         uint64_t est = decoder->sample_insn_cnt << 1;
2341
2342         if (!decoder->cbr || !decoder->max_non_turbo_ratio)
2343                 goto out;
2344
2345         est *= decoder->max_non_turbo_ratio;
2346         est /= decoder->cbr;
2347 out:
2348         return decoder->sample_timestamp + est;
2349 }
2350
2351 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
2352 {
2353         int err;
2354
2355         do {
2356                 decoder->state.type = INTEL_PT_BRANCH;
2357                 decoder->state.flags = 0;
2358
2359                 switch (decoder->pkt_state) {
2360                 case INTEL_PT_STATE_NO_PSB:
2361                         err = intel_pt_sync(decoder);
2362                         break;
2363                 case INTEL_PT_STATE_NO_IP:
2364                         decoder->have_last_ip = false;
2365                         decoder->last_ip = 0;
2366                         decoder->ip = 0;
2367                         __fallthrough;
2368                 case INTEL_PT_STATE_ERR_RESYNC:
2369                         err = intel_pt_sync_ip(decoder);
2370                         break;
2371                 case INTEL_PT_STATE_IN_SYNC:
2372                         err = intel_pt_walk_trace(decoder);
2373                         break;
2374                 case INTEL_PT_STATE_TNT:
2375                         err = intel_pt_walk_tnt(decoder);
2376                         if (err == -EAGAIN)
2377                                 err = intel_pt_walk_trace(decoder);
2378                         break;
2379                 case INTEL_PT_STATE_TIP:
2380                 case INTEL_PT_STATE_TIP_PGD:
2381                         err = intel_pt_walk_tip(decoder);
2382                         break;
2383                 case INTEL_PT_STATE_FUP:
2384                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2385                         err = intel_pt_walk_fup(decoder);
2386                         if (err == -EAGAIN)
2387                                 err = intel_pt_walk_fup_tip(decoder);
2388                         else if (!err)
2389                                 decoder->pkt_state = INTEL_PT_STATE_FUP;
2390                         break;
2391                 case INTEL_PT_STATE_FUP_NO_TIP:
2392                         decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
2393                         err = intel_pt_walk_fup(decoder);
2394                         if (err == -EAGAIN)
2395                                 err = intel_pt_walk_trace(decoder);
2396                         break;
2397                 default:
2398                         err = intel_pt_bug(decoder);
2399                         break;
2400                 }
2401         } while (err == -ENOLINK);
2402
2403         if (err) {
2404                 decoder->state.err = intel_pt_ext_err(err);
2405                 decoder->state.from_ip = decoder->ip;
2406                 decoder->sample_timestamp = decoder->timestamp;
2407                 decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2408         } else {
2409                 decoder->state.err = 0;
2410                 if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {
2411                         decoder->cbr_seen = decoder->cbr;
2412                         decoder->state.type |= INTEL_PT_CBR_CHG;
2413                         decoder->state.cbr_payload = decoder->cbr_payload;
2414                 }
2415                 if (intel_pt_sample_time(decoder->pkt_state)) {
2416                         decoder->sample_timestamp = decoder->timestamp;
2417                         decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;
2418                 }
2419         }
2420
2421         decoder->state.timestamp = decoder->sample_timestamp;
2422         decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
2423         decoder->state.cr3 = decoder->cr3;
2424         decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;
2425
2426         return &decoder->state;
2427 }
2428
2429 /**
2430  * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
2431  * @buf: pointer to buffer pointer
2432  * @len: size of buffer
2433  *
2434  * Updates the buffer pointer to point to the start of the next PSB packet if
2435  * there is one, otherwise the buffer pointer is unchanged.  If @buf is updated,
2436  * @len is adjusted accordingly.
2437  *
2438  * Return: %true if a PSB packet is found, %false otherwise.
2439  */
2440 static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
2441 {
2442         unsigned char *next;
2443
2444         next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2445         if (next) {
2446                 *len -= next - *buf;
2447                 *buf = next;
2448                 return true;
2449         }
2450         return false;
2451 }
2452
2453 /**
2454  * intel_pt_step_psb - move buffer pointer to the start of the following PSB
2455  *                     packet.
2456  * @buf: pointer to buffer pointer
2457  * @len: size of buffer
2458  *
2459  * Updates the buffer pointer to point to the start of the following PSB packet
2460  * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
2461  * pointer is unchanged.  If @buf is updated, @len is adjusted accordingly.
2462  *
2463  * Return: %true if a PSB packet is found, %false otherwise.
2464  */
2465 static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
2466 {
2467         unsigned char *next;
2468
2469         if (!*len)
2470                 return false;
2471
2472         next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
2473         if (next) {
2474                 *len -= next - *buf;
2475                 *buf = next;
2476                 return true;
2477         }
2478         return false;
2479 }
2480
2481 /**
2482  * intel_pt_last_psb - find the last PSB packet in a buffer.
2483  * @buf: buffer
2484  * @len: size of buffer
2485  *
2486  * This function finds the last PSB in a buffer.
2487  *
2488  * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
2489  */
2490 static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
2491 {
2492         const char *n = INTEL_PT_PSB_STR;
2493         unsigned char *p;
2494         size_t k;
2495
2496         if (len < INTEL_PT_PSB_LEN)
2497                 return NULL;
2498
2499         k = len - INTEL_PT_PSB_LEN + 1;
2500         while (1) {
2501                 p = memrchr(buf, n[0], k);
2502                 if (!p)
2503                         return NULL;
2504                 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
2505                         return p;
2506                 k = p - buf;
2507                 if (!k)
2508                         return NULL;
2509         }
2510 }
2511
2512 /**
2513  * intel_pt_next_tsc - find and return next TSC.
2514  * @buf: buffer
2515  * @len: size of buffer
2516  * @tsc: TSC value returned
2517  * @rem: returns remaining size when TSC is found
2518  *
2519  * Find a TSC packet in @buf and return the TSC value.  This function assumes
2520  * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
2521  * PSBEND packet is found.
2522  *
2523  * Return: %true if TSC is found, false otherwise.
2524  */
2525 static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc,
2526                               size_t *rem)
2527 {
2528         struct intel_pt_pkt packet;
2529         int ret;
2530
2531         while (len) {
2532                 ret = intel_pt_get_packet(buf, len, &packet);
2533                 if (ret <= 0)
2534                         return false;
2535                 if (packet.type == INTEL_PT_TSC) {
2536                         *tsc = packet.payload;
2537                         *rem = len;
2538                         return true;
2539                 }
2540                 if (packet.type == INTEL_PT_PSBEND)
2541                         return false;
2542                 buf += ret;
2543                 len -= ret;
2544         }
2545         return false;
2546 }
2547
2548 /**
2549  * intel_pt_tsc_cmp - compare 7-byte TSCs.
2550  * @tsc1: first TSC to compare
2551  * @tsc2: second TSC to compare
2552  *
2553  * This function compares 7-byte TSC values allowing for the possibility that
2554  * TSC wrapped around.  Generally it is not possible to know if TSC has wrapped
2555  * around so for that purpose this function assumes the absolute difference is
2556  * less than half the maximum difference.
2557  *
2558  * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
2559  * after @tsc2.
2560  */
2561 static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
2562 {
2563         const uint64_t halfway = (1ULL << 55);
2564
2565         if (tsc1 == tsc2)
2566                 return 0;
2567
2568         if (tsc1 < tsc2) {
2569                 if (tsc2 - tsc1 < halfway)
2570                         return -1;
2571                 else
2572                         return 1;
2573         } else {
2574                 if (tsc1 - tsc2 < halfway)
2575                         return 1;
2576                 else
2577                         return -1;
2578         }
2579 }
2580
2581 #define MAX_PADDING (PERF_AUXTRACE_RECORD_ALIGNMENT - 1)
2582
2583 /**
2584  * adj_for_padding - adjust overlap to account for padding.
2585  * @buf_b: second buffer
2586  * @buf_a: first buffer
2587  * @len_a: size of first buffer
2588  *
2589  * @buf_a might have up to 7 bytes of padding appended. Adjust the overlap
2590  * accordingly.
2591  *
2592  * Return: A pointer into @buf_b from where non-overlapped data starts
2593  */
2594 static unsigned char *adj_for_padding(unsigned char *buf_b,
2595                                       unsigned char *buf_a, size_t len_a)
2596 {
2597         unsigned char *p = buf_b - MAX_PADDING;
2598         unsigned char *q = buf_a + len_a - MAX_PADDING;
2599         int i;
2600
2601         for (i = MAX_PADDING; i; i--, p++, q++) {
2602                 if (*p != *q)
2603                         break;
2604         }
2605
2606         return p;
2607 }
2608
2609 /**
2610  * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
2611  *                             using TSC.
2612  * @buf_a: first buffer
2613  * @len_a: size of first buffer
2614  * @buf_b: second buffer
2615  * @len_b: size of second buffer
2616  * @consecutive: returns true if there is data in buf_b that is consecutive
2617  *               to buf_a
2618  *
2619  * If the trace contains TSC we can look at the last TSC of @buf_a and the
2620  * first TSC of @buf_b in order to determine if the buffers overlap, and then
2621  * walk forward in @buf_b until a later TSC is found.  A precondition is that
2622  * @buf_a and @buf_b are positioned at a PSB.
2623  *
2624  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2625  * @buf_b + @len_b if there is no non-overlapped data.
2626  */
2627 static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
2628                                                 size_t len_a,
2629                                                 unsigned char *buf_b,
2630                                                 size_t len_b, bool *consecutive)
2631 {
2632         uint64_t tsc_a, tsc_b;
2633         unsigned char *p;
2634         size_t len, rem_a, rem_b;
2635
2636         p = intel_pt_last_psb(buf_a, len_a);
2637         if (!p)
2638                 return buf_b; /* No PSB in buf_a => no overlap */
2639
2640         len = len_a - (p - buf_a);
2641         if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a)) {
2642                 /* The last PSB+ in buf_a is incomplete, so go back one more */
2643                 len_a -= len;
2644                 p = intel_pt_last_psb(buf_a, len_a);
2645                 if (!p)
2646                         return buf_b; /* No full PSB+ => assume no overlap */
2647                 len = len_a - (p - buf_a);
2648                 if (!intel_pt_next_tsc(p, len, &tsc_a, &rem_a))
2649                         return buf_b; /* No TSC in buf_a => assume no overlap */
2650         }
2651
2652         while (1) {
2653                 /* Ignore PSB+ with no TSC */
2654                 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b, &rem_b)) {
2655                         int cmp = intel_pt_tsc_cmp(tsc_a, tsc_b);
2656
2657                         /* Same TSC, so buffers are consecutive */
2658                         if (!cmp && rem_b >= rem_a) {
2659                                 unsigned char *start;
2660
2661                                 *consecutive = true;
2662                                 start = buf_b + len_b - (rem_b - rem_a);
2663                                 return adj_for_padding(start, buf_a, len_a);
2664                         }
2665                         if (cmp < 0)
2666                                 return buf_b; /* tsc_a < tsc_b => no overlap */
2667                 }
2668
2669                 if (!intel_pt_step_psb(&buf_b, &len_b))
2670                         return buf_b + len_b; /* No PSB in buf_b => no data */
2671         }
2672 }
2673
2674 /**
2675  * intel_pt_find_overlap - determine start of non-overlapped trace data.
2676  * @buf_a: first buffer
2677  * @len_a: size of first buffer
2678  * @buf_b: second buffer
2679  * @len_b: size of second buffer
2680  * @have_tsc: can use TSC packets to detect overlap
2681  * @consecutive: returns true if there is data in buf_b that is consecutive
2682  *               to buf_a
2683  *
2684  * When trace samples or snapshots are recorded there is the possibility that
2685  * the data overlaps.  Note that, for the purposes of decoding, data is only
2686  * useful if it begins with a PSB packet.
2687  *
2688  * Return: A pointer into @buf_b from where non-overlapped data starts, or
2689  * @buf_b + @len_b if there is no non-overlapped data.
2690  */
2691 unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
2692                                      unsigned char *buf_b, size_t len_b,
2693                                      bool have_tsc, bool *consecutive)
2694 {
2695         unsigned char *found;
2696
2697         /* Buffer 'b' must start at PSB so throw away everything before that */
2698         if (!intel_pt_next_psb(&buf_b, &len_b))
2699                 return buf_b + len_b; /* No PSB */
2700
2701         if (!intel_pt_next_psb(&buf_a, &len_a))
2702                 return buf_b; /* No overlap */
2703
2704         if (have_tsc) {
2705                 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b,
2706                                                   consecutive);
2707                 if (found)
2708                         return found;
2709         }
2710
2711         /*
2712          * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
2713          * we can ignore the first part of buffer 'a'.
2714          */
2715         while (len_b < len_a) {
2716                 if (!intel_pt_step_psb(&buf_a, &len_a))
2717                         return buf_b; /* No overlap */
2718         }
2719
2720         /* Now len_b >= len_a */
2721         while (1) {
2722                 /* Potential overlap so check the bytes */
2723                 found = memmem(buf_a, len_a, buf_b, len_a);
2724                 if (found) {
2725                         *consecutive = true;
2726                         return adj_for_padding(buf_b + len_a, buf_a, len_a);
2727                 }
2728
2729                 /* Try again at next PSB in buffer 'a' */
2730                 if (!intel_pt_step_psb(&buf_a, &len_a))
2731                         return buf_b; /* No overlap */
2732         }
2733 }