arch/x86/events/amd/core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/perf_event.h>
   3 #include <linux/jump_label.h>
   4 #include <linux/export.h>
   5 #include <linux/types.h>
   6 #include <linux/init.h>
   7 #include <linux/slab.h>
   8 #include <linux/delay.h>
   9 #include <linux/jiffies.h>
  10 #include <asm/apicdef.h>
  11 #include <asm/apic.h>
  12 #include <asm/nmi.h>
  13
  14 #include "../perf_event.h"
  15
  16 static DEFINE_PER_CPU(unsigned long, perf_nmi_tstamp);
  17 static unsigned long perf_nmi_window;
  18
  19 /* AMD Event 0xFFF: Merge.  Used with Large Increment per Cycle events */
  20 #define AMD_MERGE_EVENT ((0xFULL << 32) | 0xFFULL)
  21 #define AMD_MERGE_EVENT_ENABLE (AMD_MERGE_EVENT | ARCH_PERFMON_EVENTSEL_ENABLE)
  22
  23 /* PMC Enable and Overflow bits for PerfCntrGlobal* registers */
  24 static u64 amd_pmu_global_cntr_mask __read_mostly;
  25
  26 static __initconst const u64 amd_hw_cache_event_ids
  27                                 [PERF_COUNT_HW_CACHE_MAX]
  28                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  29                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  30 {
  31  [ C(L1D) ] = {
  32         [ C(OP_READ) ] = {
  33                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
  34                 [ C(RESULT_MISS)   ] = 0x0141, /* Data Cache Misses          */
  35         },
  36         [ C(OP_WRITE) ] = {
  37                 [ C(RESULT_ACCESS) ] = 0,
  38                 [ C(RESULT_MISS)   ] = 0,
  39         },
  40         [ C(OP_PREFETCH) ] = {
  41                 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
  42                 [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
  43         },
  44  },
  45  [ C(L1I ) ] = {
  46         [ C(OP_READ) ] = {
  47                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
  48                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
  49         },
  50         [ C(OP_WRITE) ] = {
  51                 [ C(RESULT_ACCESS) ] = -1,
  52                 [ C(RESULT_MISS)   ] = -1,
  53         },
  54         [ C(OP_PREFETCH) ] = {
  55                 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
  56                 [ C(RESULT_MISS)   ] = 0,
  57         },
  58  },
  59  [ C(LL  ) ] = {
  60         [ C(OP_READ) ] = {
  61                 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
  62                 [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
  63         },
  64         [ C(OP_WRITE) ] = {
  65                 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
  66                 [ C(RESULT_MISS)   ] = 0,
  67         },
  68         [ C(OP_PREFETCH) ] = {
  69                 [ C(RESULT_ACCESS) ] = 0,
  70                 [ C(RESULT_MISS)   ] = 0,
  71         },
  72  },
  73  [ C(DTLB) ] = {
  74         [ C(OP_READ) ] = {
  75                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
  76                 [ C(RESULT_MISS)   ] = 0x0746, /* L1_DTLB_AND_L2_DLTB_MISS.ALL */
  77         },
  78         [ C(OP_WRITE) ] = {
  79                 [ C(RESULT_ACCESS) ] = 0,
  80                 [ C(RESULT_MISS)   ] = 0,
  81         },
  82         [ C(OP_PREFETCH) ] = {
  83                 [ C(RESULT_ACCESS) ] = 0,
  84                 [ C(RESULT_MISS)   ] = 0,
  85         },
  86  },
  87  [ C(ITLB) ] = {
  88         [ C(OP_READ) ] = {
  89                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
  90                 [ C(RESULT_MISS)   ] = 0x0385, /* L1_ITLB_AND_L2_ITLB_MISS.ALL */
  91         },
  92         [ C(OP_WRITE) ] = {
  93                 [ C(RESULT_ACCESS) ] = -1,
  94                 [ C(RESULT_MISS)   ] = -1,
  95         },
  96         [ C(OP_PREFETCH) ] = {
  97                 [ C(RESULT_ACCESS) ] = -1,
  98                 [ C(RESULT_MISS)   ] = -1,
  99         },
 100  },
 101  [ C(BPU ) ] = {
 102         [ C(OP_READ) ] = {
 103                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 104                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 105         },
 106         [ C(OP_WRITE) ] = {
 107                 [ C(RESULT_ACCESS) ] = -1,
 108                 [ C(RESULT_MISS)   ] = -1,
 109         },
 110         [ C(OP_PREFETCH) ] = {
 111                 [ C(RESULT_ACCESS) ] = -1,
 112                 [ C(RESULT_MISS)   ] = -1,
 113         },
 114  },
 115  [ C(NODE) ] = {
 116         [ C(OP_READ) ] = {
 117                 [ C(RESULT_ACCESS) ] = 0xb8e9, /* CPU Request to Memory, l+r */
 118                 [ C(RESULT_MISS)   ] = 0x98e9, /* CPU Request to Memory, r   */
 119         },
 120         [ C(OP_WRITE) ] = {
 121                 [ C(RESULT_ACCESS) ] = -1,
 122                 [ C(RESULT_MISS)   ] = -1,
 123         },
 124         [ C(OP_PREFETCH) ] = {
 125                 [ C(RESULT_ACCESS) ] = -1,
 126                 [ C(RESULT_MISS)   ] = -1,
 127         },
 128  },
 129 };
 130
 131 static __initconst const u64 amd_hw_cache_event_ids_f17h
 132                                 [PERF_COUNT_HW_CACHE_MAX]
 133                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 134                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 135 [C(L1D)] = {
 136         [C(OP_READ)] = {
 137                 [C(RESULT_ACCESS)] = 0x0040, /* Data Cache Accesses */
 138                 [C(RESULT_MISS)]   = 0xc860, /* L2$ access from DC Miss */
 139         },
 140         [C(OP_WRITE)] = {
 141                 [C(RESULT_ACCESS)] = 0,
 142                 [C(RESULT_MISS)]   = 0,
 143         },
 144         [C(OP_PREFETCH)] = {
 145                 [C(RESULT_ACCESS)] = 0xff5a, /* h/w prefetch DC Fills */
 146                 [C(RESULT_MISS)]   = 0,
 147         },
 148 },
 149 [C(L1I)] = {
 150         [C(OP_READ)] = {
 151                 [C(RESULT_ACCESS)] = 0x0080, /* Instruction cache fetches  */
 152                 [C(RESULT_MISS)]   = 0x0081, /* Instruction cache misses   */
 153         },
 154         [C(OP_WRITE)] = {
 155                 [C(RESULT_ACCESS)] = -1,
 156                 [C(RESULT_MISS)]   = -1,
 157         },
 158         [C(OP_PREFETCH)] = {
 159                 [C(RESULT_ACCESS)] = 0,
 160                 [C(RESULT_MISS)]   = 0,
 161         },
 162 },
 163 [C(LL)] = {
 164         [C(OP_READ)] = {
 165                 [C(RESULT_ACCESS)] = 0,
 166                 [C(RESULT_MISS)]   = 0,
 167         },
 168         [C(OP_WRITE)] = {
 169                 [C(RESULT_ACCESS)] = 0,
 170                 [C(RESULT_MISS)]   = 0,
 171         },
 172         [C(OP_PREFETCH)] = {
 173                 [C(RESULT_ACCESS)] = 0,
 174                 [C(RESULT_MISS)]   = 0,
 175         },
 176 },
 177 [C(DTLB)] = {
 178         [C(OP_READ)] = {
 179                 [C(RESULT_ACCESS)] = 0xff45, /* All L2 DTLB accesses */
 180                 [C(RESULT_MISS)]   = 0xf045, /* L2 DTLB misses (PT walks) */
 181         },
 182         [C(OP_WRITE)] = {
 183                 [C(RESULT_ACCESS)] = 0,
 184                 [C(RESULT_MISS)]   = 0,
 185         },
 186         [C(OP_PREFETCH)] = {
 187                 [C(RESULT_ACCESS)] = 0,
 188                 [C(RESULT_MISS)]   = 0,
 189         },
 190 },
 191 [C(ITLB)] = {
 192         [C(OP_READ)] = {
 193                 [C(RESULT_ACCESS)] = 0x0084, /* L1 ITLB misses, L2 ITLB hits */
 194                 [C(RESULT_MISS)]   = 0xff85, /* L1 ITLB misses, L2 misses */
 195         },
 196         [C(OP_WRITE)] = {
 197                 [C(RESULT_ACCESS)] = -1,
 198                 [C(RESULT_MISS)]   = -1,
 199         },
 200         [C(OP_PREFETCH)] = {
 201                 [C(RESULT_ACCESS)] = -1,
 202                 [C(RESULT_MISS)]   = -1,
 203         },
 204 },
 205 [C(BPU)] = {
 206         [C(OP_READ)] = {
 207                 [C(RESULT_ACCESS)] = 0x00c2, /* Retired Branch Instr.      */
 208                 [C(RESULT_MISS)]   = 0x00c3, /* Retired Mispredicted BI    */
 209         },
 210         [C(OP_WRITE)] = {
 211                 [C(RESULT_ACCESS)] = -1,
 212                 [C(RESULT_MISS)]   = -1,
 213         },
 214         [C(OP_PREFETCH)] = {
 215                 [C(RESULT_ACCESS)] = -1,
 216                 [C(RESULT_MISS)]   = -1,
 217         },
 218 },
 219 [C(NODE)] = {
 220         [C(OP_READ)] = {
 221                 [C(RESULT_ACCESS)] = 0,
 222                 [C(RESULT_MISS)]   = 0,
 223         },
 224         [C(OP_WRITE)] = {
 225                 [C(RESULT_ACCESS)] = -1,
 226                 [C(RESULT_MISS)]   = -1,
 227         },
 228         [C(OP_PREFETCH)] = {
 229                 [C(RESULT_ACCESS)] = -1,
 230                 [C(RESULT_MISS)]   = -1,
 231         },
 232 },
 233 };
 234
 235 /*
 236  * AMD Performance Monitor K7 and later, up to and including Family 16h:
 237  */
 238 static const u64 amd_perfmon_event_map[PERF_COUNT_HW_MAX] =
 239 {
 240         [PERF_COUNT_HW_CPU_CYCLES]              = 0x0076,
 241         [PERF_COUNT_HW_INSTRUCTIONS]            = 0x00c0,
 242         [PERF_COUNT_HW_CACHE_REFERENCES]        = 0x077d,
 243         [PERF_COUNT_HW_CACHE_MISSES]            = 0x077e,
 244         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = 0x00c2,
 245         [PERF_COUNT_HW_BRANCH_MISSES]           = 0x00c3,
 246         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */
 247         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = 0x00d1, /* "Dispatch stalls" event */
 248 };
 249
 250 /*
 251  * AMD Performance Monitor Family 17h and later:
 252  */
 253 static const u64 amd_zen1_perfmon_event_map[PERF_COUNT_HW_MAX] =
 254 {
 255         [PERF_COUNT_HW_CPU_CYCLES]              = 0x0076,
 256         [PERF_COUNT_HW_INSTRUCTIONS]            = 0x00c0,
 257         [PERF_COUNT_HW_CACHE_REFERENCES]        = 0xff60,
 258         [PERF_COUNT_HW_CACHE_MISSES]            = 0x0964,
 259         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = 0x00c2,
 260         [PERF_COUNT_HW_BRANCH_MISSES]           = 0x00c3,
 261         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x0287,
 262         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = 0x0187,
 263 };
 264
 265 static const u64 amd_zen2_perfmon_event_map[PERF_COUNT_HW_MAX] =
 266 {
 267         [PERF_COUNT_HW_CPU_CYCLES]              = 0x0076,
 268         [PERF_COUNT_HW_INSTRUCTIONS]            = 0x00c0,
 269         [PERF_COUNT_HW_CACHE_REFERENCES]        = 0xff60,
 270         [PERF_COUNT_HW_CACHE_MISSES]            = 0x0964,
 271         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = 0x00c2,
 272         [PERF_COUNT_HW_BRANCH_MISSES]           = 0x00c3,
 273         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00a9,
 274 };
 275
 276 static const u64 amd_zen4_perfmon_event_map[PERF_COUNT_HW_MAX] =
 277 {
 278         [PERF_COUNT_HW_CPU_CYCLES]              = 0x0076,
 279         [PERF_COUNT_HW_INSTRUCTIONS]            = 0x00c0,
 280         [PERF_COUNT_HW_CACHE_REFERENCES]        = 0xff60,
 281         [PERF_COUNT_HW_CACHE_MISSES]            = 0x0964,
 282         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = 0x00c2,
 283         [PERF_COUNT_HW_BRANCH_MISSES]           = 0x00c3,
 284         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00a9,
 285         [PERF_COUNT_HW_REF_CPU_CYCLES]          = 0x100000120,
 286 };
 287
 288 static u64 amd_pmu_event_map(int hw_event)
 289 {
 290         if (cpu_feature_enabled(X86_FEATURE_ZEN4) || boot_cpu_data.x86 >= 0x1a)
 291                 return amd_zen4_perfmon_event_map[hw_event];
 292
 293         if (cpu_feature_enabled(X86_FEATURE_ZEN2) || boot_cpu_data.x86 >= 0x19)
 294                 return amd_zen2_perfmon_event_map[hw_event];
 295
 296         if (cpu_feature_enabled(X86_FEATURE_ZEN1))
 297                 return amd_zen1_perfmon_event_map[hw_event];
 298
 299         return amd_perfmon_event_map[hw_event];
 300 }
 301
 302 /*
 303  * Previously calculated offsets
 304  */
 305 static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly;
 306 static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly;
 307
 308 /*
 309  * Legacy CPUs:
 310  *   4 counters starting at 0xc0010000 each offset by 1
 311  *
 312  * CPUs with core performance counter extensions:
 313  *   6 counters starting at 0xc0010200 each offset by 2
 314  */
 315 static inline int amd_pmu_addr_offset(int index, bool eventsel)
 316 {
 317         int offset;
 318
 319         if (!index)
 320                 return index;
 321
 322         if (eventsel)
 323                 offset = event_offsets[index];
 324         else
 325                 offset = count_offsets[index];
 326
 327         if (offset)
 328                 return offset;
 329
 330         if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
 331                 offset = index;
 332         else
 333                 offset = index << 1;
 334
 335         if (eventsel)
 336                 event_offsets[index] = offset;
 337         else
 338                 count_offsets[index] = offset;
 339
 340         return offset;
 341 }
 342
 343 /*
 344  * AMD64 events are detected based on their event codes.
 345  */
 346 static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc)
 347 {
 348         return ((hwc->config >> 24) & 0x0f00) | (hwc->config & 0x00ff);
 349 }
 350
 351 static inline bool amd_is_pair_event_code(struct hw_perf_event *hwc)
 352 {
 353         if (!(x86_pmu.flags & PMU_FL_PAIR))
 354                 return false;
 355
 356         switch (amd_get_event_code(hwc)) {
 357         case 0x003:     return true;    /* Retired SSE/AVX FLOPs */
 358         default:        return false;
 359         }
 360 }
 361
 362 DEFINE_STATIC_CALL_RET0(amd_pmu_branch_hw_config, *x86_pmu.hw_config);
 363
 364 static int amd_core_hw_config(struct perf_event *event)
 365 {
 366         if (event->attr.exclude_host && event->attr.exclude_guest)
 367                 /*
 368                  * When HO == GO == 1 the hardware treats that as GO == HO == 0
 369                  * and will count in both modes. We don't want to count in that
 370                  * case so we emulate no-counting by setting US = OS = 0.
 371                  */
 372                 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
 373                                       ARCH_PERFMON_EVENTSEL_OS);
 374         else if (event->attr.exclude_host)
 375                 event->hw.config |= AMD64_EVENTSEL_GUESTONLY;
 376         else if (event->attr.exclude_guest)
 377                 event->hw.config |= AMD64_EVENTSEL_HOSTONLY;
 378
 379         if ((x86_pmu.flags & PMU_FL_PAIR) && amd_is_pair_event_code(&event->hw))
 380                 event->hw.flags |= PERF_X86_EVENT_PAIR;
 381
 382         if (has_branch_stack(event))
 383                 return static_call(amd_pmu_branch_hw_config)(event);
 384
 385         return 0;
 386 }
 387
 388 static inline int amd_is_nb_event(struct hw_perf_event *hwc)
 389 {
 390         return (hwc->config & 0xe0) == 0xe0;
 391 }
 392
 393 static inline int amd_has_nb(struct cpu_hw_events *cpuc)
 394 {
 395         struct amd_nb *nb = cpuc->amd_nb;
 396
 397         return nb && nb->nb_id != -1;
 398 }
 399
 400 static int amd_pmu_hw_config(struct perf_event *event)
 401 {
 402         int ret;
 403
 404         /* pass precise event sampling to ibs: */
 405         if (event->attr.precise_ip && get_ibs_caps())
 406                 return forward_event_to_ibs(event);
 407
 408         if (has_branch_stack(event) && !x86_pmu.lbr_nr)
 409                 return -EOPNOTSUPP;
 410
 411         ret = x86_pmu_hw_config(event);
 412         if (ret)
 413                 return ret;
 414
 415         if (event->attr.type == PERF_TYPE_RAW)
 416                 event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
 417
 418         return amd_core_hw_config(event);
 419 }
 420
 421 static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc,
 422                                            struct perf_event *event)
 423 {
 424         struct amd_nb *nb = cpuc->amd_nb;
 425         int i;
 426
 427         /*
 428          * need to scan whole list because event may not have
 429          * been assigned during scheduling
 430          *
 431          * no race condition possible because event can only
 432          * be removed on one CPU at a time AND PMU is disabled
 433          * when we come here
 434          */
 435         for (i = 0; i < x86_pmu.num_counters; i++) {
 436                 if (cmpxchg(nb->owners + i, event, NULL) == event)
 437                         break;
 438         }
 439 }
 440
 441  /*
 442   * AMD64 NorthBridge events need special treatment because
 443   * counter access needs to be synchronized across all cores
 444   * of a package. Refer to BKDG section 3.12
 445   *
 446   * NB events are events measuring L3 cache, Hypertransport
 447   * traffic. They are identified by an event code >= 0xe00.
 448   * They measure events on the NorthBride which is shared
 449   * by all cores on a package. NB events are counted on a
 450   * shared set of counters. When a NB event is programmed
 451   * in a counter, the data actually comes from a shared
 452   * counter. Thus, access to those counters needs to be
 453   * synchronized.
 454   *
 455   * We implement the synchronization such that no two cores
 456   * can be measuring NB events using the same counters. Thus,
 457   * we maintain a per-NB allocation table. The available slot
 458   * is propagated using the event_constraint structure.
 459   *
 460   * We provide only one choice for each NB event based on
 461   * the fact that only NB events have restrictions. Consequently,
 462   * if a counter is available, there is a guarantee the NB event
 463   * will be assigned to it. If no slot is available, an empty
 464   * constraint is returned and scheduling will eventually fail
 465   * for this event.
 466   *
 467   * Note that all cores attached the same NB compete for the same
 468   * counters to host NB events, this is why we use atomic ops. Some
 469   * multi-chip CPUs may have more than one NB.
 470   *
 471   * Given that resources are allocated (cmpxchg), they must be
 472   * eventually freed for others to use. This is accomplished by
 473   * calling __amd_put_nb_event_constraints()
 474   *
 475   * Non NB events are not impacted by this restriction.
 476   */
 477 static struct event_constraint *
 478 __amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
 479                                struct event_constraint *c)
 480 {
 481         struct hw_perf_event *hwc = &event->hw;
 482         struct amd_nb *nb = cpuc->amd_nb;
 483         struct perf_event *old;
 484         int idx, new = -1;
 485
 486         if (!c)
 487                 c = &unconstrained;
 488
 489         if (cpuc->is_fake)
 490                 return c;
 491
 492         /*
 493          * detect if already present, if so reuse
 494          *
 495          * cannot merge with actual allocation
 496          * because of possible holes
 497          *
 498          * event can already be present yet not assigned (in hwc->idx)
 499          * because of successive calls to x86_schedule_events() from
 500          * hw_perf_group_sched_in() without hw_perf_enable()
 501          */
 502         for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) {
 503                 if (new == -1 || hwc->idx == idx)
 504                         /* assign free slot, prefer hwc->idx */
 505                         old = cmpxchg(nb->owners + idx, NULL, event);
 506                 else if (nb->owners[idx] == event)
 507                         /* event already present */
 508                         old = event;
 509                 else
 510                         continue;
 511
 512                 if (old && old != event)
 513                         continue;
 514
 515                 /* reassign to this slot */
 516                 if (new != -1)
 517                         cmpxchg(nb->owners + new, event, NULL);
 518                 new = idx;
 519
 520                 /* already present, reuse */
 521                 if (old == event)
 522                         break;
 523         }
 524
 525         if (new == -1)
 526                 return &emptyconstraint;
 527
 528         return &nb->event_constraints[new];
 529 }
 530
 531 static struct amd_nb *amd_alloc_nb(int cpu)
 532 {
 533         struct amd_nb *nb;
 534         int i;
 535
 536         nb = kzalloc_node(sizeof(struct amd_nb), GFP_KERNEL, cpu_to_node(cpu));
 537         if (!nb)
 538                 return NULL;
 539
 540         nb->nb_id = -1;
 541
 542         /*
 543          * initialize all possible NB constraints
 544          */
 545         for (i = 0; i < x86_pmu.num_counters; i++) {
 546                 __set_bit(i, nb->event_constraints[i].idxmsk);
 547                 nb->event_constraints[i].weight = 1;
 548         }
 549         return nb;
 550 }
 551
 552 typedef void (amd_pmu_branch_reset_t)(void);
 553 DEFINE_STATIC_CALL_NULL(amd_pmu_branch_reset, amd_pmu_branch_reset_t);
 554
 555 static void amd_pmu_cpu_reset(int cpu)
 556 {
 557         if (x86_pmu.lbr_nr)
 558                 static_call(amd_pmu_branch_reset)();
 559
 560         if (x86_pmu.version < 2)
 561                 return;
 562
 563         /* Clear enable bits i.e. PerfCntrGlobalCtl.PerfCntrEn */
 564         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 0);
 565
 566         /*
 567          * Clear freeze and overflow bits i.e. PerfCntrGLobalStatus.LbrFreeze
 568          * and PerfCntrGLobalStatus.PerfCntrOvfl
 569          */
 570         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
 571                GLOBAL_STATUS_LBRS_FROZEN | amd_pmu_global_cntr_mask);
 572 }
 573
 574 static int amd_pmu_cpu_prepare(int cpu)
 575 {
 576         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
 577
 578         cpuc->lbr_sel = kzalloc_node(sizeof(struct er_account), GFP_KERNEL,
 579                                      cpu_to_node(cpu));
 580         if (!cpuc->lbr_sel)
 581                 return -ENOMEM;
 582
 583         WARN_ON_ONCE(cpuc->amd_nb);
 584
 585         if (!x86_pmu.amd_nb_constraints)
 586                 return 0;
 587
 588         cpuc->amd_nb = amd_alloc_nb(cpu);
 589         if (cpuc->amd_nb)
 590                 return 0;
 591
 592         kfree(cpuc->lbr_sel);
 593         cpuc->lbr_sel = NULL;
 594
 595         return -ENOMEM;
 596 }
 597
 598 static void amd_pmu_cpu_starting(int cpu)
 599 {
 600         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
 601         void **onln = &cpuc->kfree_on_online[X86_PERF_KFREE_SHARED];
 602         struct amd_nb *nb;
 603         int i, nb_id;
 604
 605         cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
 606         amd_pmu_cpu_reset(cpu);
 607
 608         if (!x86_pmu.amd_nb_constraints)
 609                 return;
 610
 611         nb_id = topology_amd_node_id(cpu);
 612         WARN_ON_ONCE(nb_id == BAD_APICID);
 613
 614         for_each_online_cpu(i) {
 615                 nb = per_cpu(cpu_hw_events, i).amd_nb;
 616                 if (WARN_ON_ONCE(!nb))
 617                         continue;
 618
 619                 if (nb->nb_id == nb_id) {
 620                         *onln = cpuc->amd_nb;
 621                         cpuc->amd_nb = nb;
 622                         break;
 623                 }
 624         }
 625
 626         cpuc->amd_nb->nb_id = nb_id;
 627         cpuc->amd_nb->refcnt++;
 628 }
 629
 630 static void amd_pmu_cpu_dead(int cpu)
 631 {
 632         struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
 633
 634         kfree(cpuhw->lbr_sel);
 635         cpuhw->lbr_sel = NULL;
 636
 637         if (!x86_pmu.amd_nb_constraints)
 638                 return;
 639
 640         if (cpuhw->amd_nb) {
 641                 struct amd_nb *nb = cpuhw->amd_nb;
 642
 643                 if (nb->nb_id == -1 || --nb->refcnt == 0)
 644                         kfree(nb);
 645
 646                 cpuhw->amd_nb = NULL;
 647         }
 648 }
 649
 650 static inline void amd_pmu_set_global_ctl(u64 ctl)
 651 {
 652         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl);
 653 }
 654
 655 static inline u64 amd_pmu_get_global_status(void)
 656 {
 657         u64 status;
 658
 659         /* PerfCntrGlobalStatus is read-only */
 660         rdmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status);
 661
 662         return status;
 663 }
 664
 665 static inline void amd_pmu_ack_global_status(u64 status)
 666 {
 667         /*
 668          * PerfCntrGlobalStatus is read-only but an overflow acknowledgment
 669          * mechanism exists; writing 1 to a bit in PerfCntrGlobalStatusClr
 670          * clears the same bit in PerfCntrGlobalStatus
 671          */
 672
 673         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, status);
 674 }
 675
 676 static bool amd_pmu_test_overflow_topbit(int idx)
 677 {
 678         u64 counter;
 679
 680         rdmsrl(x86_pmu_event_addr(idx), counter);
 681
 682         return !(counter & BIT_ULL(x86_pmu.cntval_bits - 1));
 683 }
 684
 685 static bool amd_pmu_test_overflow_status(int idx)
 686 {
 687         return amd_pmu_get_global_status() & BIT_ULL(idx);
 688 }
 689
 690 DEFINE_STATIC_CALL(amd_pmu_test_overflow, amd_pmu_test_overflow_topbit);
 691
 692 /*
 693  * When a PMC counter overflows, an NMI is used to process the event and
 694  * reset the counter. NMI latency can result in the counter being updated
 695  * before the NMI can run, which can result in what appear to be spurious
 696  * NMIs. This function is intended to wait for the NMI to run and reset
 697  * the counter to avoid possible unhandled NMI messages.
 698  */
 699 #define OVERFLOW_WAIT_COUNT     50
 700
 701 static void amd_pmu_wait_on_overflow(int idx)
 702 {
 703         unsigned int i;
 704
 705         /*
 706          * Wait for the counter to be reset if it has overflowed. This loop
 707          * should exit very, very quickly, but just in case, don't wait
 708          * forever...
 709          */
 710         for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) {
 711                 if (!static_call(amd_pmu_test_overflow)(idx))
 712                         break;
 713
 714                 /* Might be in IRQ context, so can't sleep */
 715                 udelay(1);
 716         }
 717 }
 718
 719 static void amd_pmu_check_overflow(void)
 720 {
 721         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 722         int idx;
 723
 724         /*
 725          * This shouldn't be called from NMI context, but add a safeguard here
 726          * to return, since if we're in NMI context we can't wait for an NMI
 727          * to reset an overflowed counter value.
 728          */
 729         if (in_nmi())
 730                 return;
 731
 732         /*
 733          * Check each counter for overflow and wait for it to be reset by the
 734          * NMI if it has overflowed. This relies on the fact that all active
 735          * counters are always enabled when this function is called and
 736          * ARCH_PERFMON_EVENTSEL_INT is always set.
 737          */
 738         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 739                 if (!test_bit(idx, cpuc->active_mask))
 740                         continue;
 741
 742                 amd_pmu_wait_on_overflow(idx);
 743         }
 744 }
 745
 746 static void amd_pmu_enable_event(struct perf_event *event)
 747 {
 748         x86_pmu_enable_event(event);
 749 }
 750
 751 static void amd_pmu_enable_all(int added)
 752 {
 753         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 754         int idx;
 755
 756         amd_brs_enable_all();
 757
 758         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 759                 /* only activate events which are marked as active */
 760                 if (!test_bit(idx, cpuc->active_mask))
 761                         continue;
 762
 763                 amd_pmu_enable_event(cpuc->events[idx]);
 764         }
 765 }
 766
 767 static void amd_pmu_v2_enable_event(struct perf_event *event)
 768 {
 769         struct hw_perf_event *hwc = &event->hw;
 770
 771         /*
 772          * Testing cpu_hw_events.enabled should be skipped in this case unlike
 773          * in x86_pmu_enable_event().
 774          *
 775          * Since cpu_hw_events.enabled is set only after returning from
 776          * x86_pmu_start(), the PMCs must be programmed and kept ready.
 777          * Counting starts only after x86_pmu_enable_all() is called.
 778          */
 779         __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
 780 }
 781
 782 static __always_inline void amd_pmu_core_enable_all(void)
 783 {
 784         amd_pmu_set_global_ctl(amd_pmu_global_cntr_mask);
 785 }
 786
 787 static void amd_pmu_v2_enable_all(int added)
 788 {
 789         amd_pmu_lbr_enable_all();
 790         amd_pmu_core_enable_all();
 791 }
 792
 793 static void amd_pmu_disable_event(struct perf_event *event)
 794 {
 795         x86_pmu_disable_event(event);
 796
 797         /*
 798          * This can be called from NMI context (via x86_pmu_stop). The counter
 799          * may have overflowed, but either way, we'll never see it get reset
 800          * by the NMI if we're already in the NMI. And the NMI latency support
 801          * below will take care of any pending NMI that might have been
 802          * generated by the overflow.
 803          */
 804         if (in_nmi())
 805                 return;
 806
 807         amd_pmu_wait_on_overflow(event->hw.idx);
 808 }
 809
 810 static void amd_pmu_disable_all(void)
 811 {
 812         amd_brs_disable_all();
 813         x86_pmu_disable_all();
 814         amd_pmu_check_overflow();
 815 }
 816
 817 static __always_inline void amd_pmu_core_disable_all(void)
 818 {
 819         amd_pmu_set_global_ctl(0);
 820 }
 821
 822 static void amd_pmu_v2_disable_all(void)
 823 {
 824         amd_pmu_core_disable_all();
 825         amd_pmu_lbr_disable_all();
 826         amd_pmu_check_overflow();
 827 }
 828
 829 DEFINE_STATIC_CALL_NULL(amd_pmu_branch_add, *x86_pmu.add);
 830
 831 static void amd_pmu_add_event(struct perf_event *event)
 832 {
 833         if (needs_branch_stack(event))
 834                 static_call(amd_pmu_branch_add)(event);
 835 }
 836
 837 DEFINE_STATIC_CALL_NULL(amd_pmu_branch_del, *x86_pmu.del);
 838
 839 static void amd_pmu_del_event(struct perf_event *event)
 840 {
 841         if (needs_branch_stack(event))
 842                 static_call(amd_pmu_branch_del)(event);
 843 }
 844
 845 /*
 846  * Because of NMI latency, if multiple PMC counters are active or other sources
 847  * of NMIs are received, the perf NMI handler can handle one or more overflowed
 848  * PMC counters outside of the NMI associated with the PMC overflow. If the NMI
 849  * doesn't arrive at the LAPIC in time to become a pending NMI, then the kernel
 850  * back-to-back NMI support won't be active. This PMC handler needs to take into
 851  * account that this can occur, otherwise this could result in unknown NMI
 852  * messages being issued. Examples of this is PMC overflow while in the NMI
 853  * handler when multiple PMCs are active or PMC overflow while handling some
 854  * other source of an NMI.
 855  *
 856  * Attempt to mitigate this by creating an NMI window in which un-handled NMIs
 857  * received during this window will be claimed. This prevents extending the
 858  * window past when it is possible that latent NMIs should be received. The
 859  * per-CPU perf_nmi_tstamp will be set to the window end time whenever perf has
 860  * handled a counter. When an un-handled NMI is received, it will be claimed
 861  * only if arriving within that window.
 862  */
 863 static inline int amd_pmu_adjust_nmi_window(int handled)
 864 {
 865         /*
 866          * If a counter was handled, record a timestamp such that un-handled
 867          * NMIs will be claimed if arriving within that window.
 868          */
 869         if (handled) {
 870                 this_cpu_write(perf_nmi_tstamp, jiffies + perf_nmi_window);
 871
 872                 return handled;
 873         }
 874
 875         if (time_after(jiffies, this_cpu_read(perf_nmi_tstamp)))
 876                 return NMI_DONE;
 877
 878         return NMI_HANDLED;
 879 }
 880
 881 static int amd_pmu_handle_irq(struct pt_regs *regs)
 882 {
 883         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 884         int handled;
 885         int pmu_enabled;
 886
 887         /*
 888          * Save the PMU state.
 889          * It needs to be restored when leaving the handler.
 890          */
 891         pmu_enabled = cpuc->enabled;
 892         cpuc->enabled = 0;
 893
 894         amd_brs_disable_all();
 895
 896         /* Drain BRS is in use (could be inactive) */
 897         if (cpuc->lbr_users)
 898                 amd_brs_drain();
 899
 900         /* Process any counter overflows */
 901         handled = x86_pmu_handle_irq(regs);
 902
 903         cpuc->enabled = pmu_enabled;
 904         if (pmu_enabled)
 905                 amd_brs_enable_all();
 906
 907         return amd_pmu_adjust_nmi_window(handled);
 908 }
 909
 910 static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 911 {
 912         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 913         struct perf_sample_data data;
 914         struct hw_perf_event *hwc;
 915         struct perf_event *event;
 916         int handled = 0, idx;
 917         u64 reserved, status, mask;
 918         bool pmu_enabled;
 919
 920         /*
 921          * Save the PMU state as it needs to be restored when leaving the
 922          * handler
 923          */
 924         pmu_enabled = cpuc->enabled;
 925         cpuc->enabled = 0;
 926
 927         /* Stop counting but do not disable LBR */
 928         amd_pmu_core_disable_all();
 929
 930         status = amd_pmu_get_global_status();
 931
 932         /* Check if any overflows are pending */
 933         if (!status)
 934                 goto done;
 935
 936         /* Read branch records */
 937         if (x86_pmu.lbr_nr) {
 938                 amd_pmu_lbr_read();
 939                 status &= ~GLOBAL_STATUS_LBRS_FROZEN;
 940         }
 941
 942         reserved = status & ~amd_pmu_global_cntr_mask;
 943         if (reserved)
 944                 pr_warn_once("Reserved PerfCntrGlobalStatus bits are set (0x%llx), please consider updating microcode\n",
 945                              reserved);
 946
 947         /* Clear any reserved bits set by buggy microcode */
 948         status &= amd_pmu_global_cntr_mask;
 949
 950         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 951                 if (!test_bit(idx, cpuc->active_mask))
 952                         continue;
 953
 954                 event = cpuc->events[idx];
 955                 hwc = &event->hw;
 956                 x86_perf_event_update(event);
 957                 mask = BIT_ULL(idx);
 958
 959                 if (!(status & mask))
 960                         continue;
 961
 962                 /* Event overflow */
 963                 handled++;
 964                 status &= ~mask;
 965                 perf_sample_data_init(&data, 0, hwc->last_period);
 966
 967                 if (!x86_perf_event_set_period(event))
 968                         continue;
 969
 970                 if (has_branch_stack(event))
 971                         perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
 972
 973                 if (perf_event_overflow(event, &data, regs))
 974                         x86_pmu_stop(event, 0);
 975         }
 976
 977         /*
 978          * It should never be the case that some overflows are not handled as
 979          * the corresponding PMCs are expected to be inactive according to the
 980          * active_mask
 981          */
 982         WARN_ON(status > 0);
 983
 984         /* Clear overflow and freeze bits */
 985         amd_pmu_ack_global_status(~status);
 986
 987         /*
 988          * Unmasking the LVTPC is not required as the Mask (M) bit of the LVT
 989          * PMI entry is not set by the local APIC when a PMC overflow occurs
 990          */
 991         inc_irq_stat(apic_perf_irqs);
 992
 993 done:
 994         cpuc->enabled = pmu_enabled;
 995
 996         /* Resume counting only if PMU is active */
 997         if (pmu_enabled)
 998                 amd_pmu_core_enable_all();
 999
1000         return amd_pmu_adjust_nmi_window(handled);
1001 }
1002
1003 static struct event_constraint *
1004 amd_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
1005                           struct perf_event *event)
1006 {
1007         /*
1008          * if not NB event or no NB, then no constraints
1009          */
1010         if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)))
1011                 return &unconstrained;
1012
1013         return __amd_get_nb_event_constraints(cpuc, event, NULL);
1014 }
1015
1016 static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
1017                                       struct perf_event *event)
1018 {
1019         if (amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))
1020                 __amd_put_nb_event_constraints(cpuc, event);
1021 }
1022
1023 PMU_FORMAT_ATTR(event,  "config:0-7,32-35");
1024 PMU_FORMAT_ATTR(umask,  "config:8-15"   );
1025 PMU_FORMAT_ATTR(edge,   "config:18"     );
1026 PMU_FORMAT_ATTR(inv,    "config:23"     );
1027 PMU_FORMAT_ATTR(cmask,  "config:24-31"  );
1028
1029 static struct attribute *amd_format_attr[] = {
1030         &format_attr_event.attr,
1031         &format_attr_umask.attr,
1032         &format_attr_edge.attr,
1033         &format_attr_inv.attr,
1034         &format_attr_cmask.attr,
1035         NULL,
1036 };
1037
1038 /* AMD Family 15h */
1039
1040 #define AMD_EVENT_TYPE_MASK     0x000000F0ULL
1041
1042 #define AMD_EVENT_FP            0x00000000ULL ... 0x00000010ULL
1043 #define AMD_EVENT_LS            0x00000020ULL ... 0x00000030ULL
1044 #define AMD_EVENT_DC            0x00000040ULL ... 0x00000050ULL
1045 #define AMD_EVENT_CU            0x00000060ULL ... 0x00000070ULL
1046 #define AMD_EVENT_IC_DE         0x00000080ULL ... 0x00000090ULL
1047 #define AMD_EVENT_EX_LS         0x000000C0ULL
1048 #define AMD_EVENT_DE            0x000000D0ULL
1049 #define AMD_EVENT_NB            0x000000E0ULL ... 0x000000F0ULL
1050
1051 /*
1052  * AMD family 15h event code/PMC mappings:
1053  *
1054  * type = event_code & 0x0F0:
1055  *
1056  * 0x000        FP      PERF_CTL[5:3]
1057  * 0x010        FP      PERF_CTL[5:3]
1058  * 0x020        LS      PERF_CTL[5:0]
1059  * 0x030        LS      PERF_CTL[5:0]
1060  * 0x040        DC      PERF_CTL[5:0]
1061  * 0x050        DC      PERF_CTL[5:0]
1062  * 0x060        CU      PERF_CTL[2:0]
1063  * 0x070        CU      PERF_CTL[2:0]
1064  * 0x080        IC/DE   PERF_CTL[2:0]
1065  * 0x090        IC/DE   PERF_CTL[2:0]
1066  * 0x0A0        ---
1067  * 0x0B0        ---
1068  * 0x0C0        EX/LS   PERF_CTL[5:0]
1069  * 0x0D0        DE      PERF_CTL[2:0]
1070  * 0x0E0        NB      NB_PERF_CTL[3:0]
1071  * 0x0F0        NB      NB_PERF_CTL[3:0]
1072  *
1073  * Exceptions:
1074  *
1075  * 0x000        FP      PERF_CTL[3], PERF_CTL[5:3] (*)
1076  * 0x003        FP      PERF_CTL[3]
1077  * 0x004        FP      PERF_CTL[3], PERF_CTL[5:3] (*)
1078  * 0x00B        FP      PERF_CTL[3]
1079  * 0x00D        FP      PERF_CTL[3]
1080  * 0x023        DE      PERF_CTL[2:0]
1081  * 0x02D        LS      PERF_CTL[3]
1082  * 0x02E        LS      PERF_CTL[3,0]
1083  * 0x031        LS      PERF_CTL[2:0] (**)
1084  * 0x043        CU      PERF_CTL[2:0]
1085  * 0x045        CU      PERF_CTL[2:0]
1086  * 0x046        CU      PERF_CTL[2:0]
1087  * 0x054        CU      PERF_CTL[2:0]
1088  * 0x055        CU      PERF_CTL[2:0]
1089  * 0x08F        IC      PERF_CTL[0]
1090  * 0x187        DE      PERF_CTL[0]
1091  * 0x188        DE      PERF_CTL[0]
1092  * 0x0DB        EX      PERF_CTL[5:0]
1093  * 0x0DC        LS      PERF_CTL[5:0]
1094  * 0x0DD        LS      PERF_CTL[5:0]
1095  * 0x0DE        LS      PERF_CTL[5:0]
1096  * 0x0DF        LS      PERF_CTL[5:0]
1097  * 0x1C0        EX      PERF_CTL[5:3]
1098  * 0x1D6        EX      PERF_CTL[5:0]
1099  * 0x1D8        EX      PERF_CTL[5:0]
1100  *
1101  * (*)  depending on the umask all FPU counters may be used
1102  * (**) only one unitmask enabled at a time
1103  */
1104
1105 static struct event_constraint amd_f15_PMC0  = EVENT_CONSTRAINT(0, 0x01, 0);
1106 static struct event_constraint amd_f15_PMC20 = EVENT_CONSTRAINT(0, 0x07, 0);
1107 static struct event_constraint amd_f15_PMC3  = EVENT_CONSTRAINT(0, 0x08, 0);
1108 static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
1109 static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0);
1110 static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
1111
1112 static struct event_constraint *
1113 amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, int idx,
1114                                struct perf_event *event)
1115 {
1116         struct hw_perf_event *hwc = &event->hw;
1117         unsigned int event_code = amd_get_event_code(hwc);
1118
1119         switch (event_code & AMD_EVENT_TYPE_MASK) {
1120         case AMD_EVENT_FP:
1121                 switch (event_code) {
1122                 case 0x000:
1123                         if (!(hwc->config & 0x0000F000ULL))
1124                                 break;
1125                         if (!(hwc->config & 0x00000F00ULL))
1126                                 break;
1127                         return &amd_f15_PMC3;
1128                 case 0x004:
1129                         if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
1130                                 break;
1131                         return &amd_f15_PMC3;
1132                 case 0x003:
1133                 case 0x00B:
1134                 case 0x00D:
1135                         return &amd_f15_PMC3;
1136                 }
1137                 return &amd_f15_PMC53;
1138         case AMD_EVENT_LS:
1139         case AMD_EVENT_DC:
1140         case AMD_EVENT_EX_LS:
1141                 switch (event_code) {
1142                 case 0x023:
1143                 case 0x043:
1144                 case 0x045:
1145                 case 0x046:
1146                 case 0x054:
1147                 case 0x055:
1148                         return &amd_f15_PMC20;
1149                 case 0x02D:
1150                         return &amd_f15_PMC3;
1151                 case 0x02E:
1152                         return &amd_f15_PMC30;
1153                 case 0x031:
1154                         if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
1155                                 return &amd_f15_PMC20;
1156                         return &emptyconstraint;
1157                 case 0x1C0:
1158                         return &amd_f15_PMC53;
1159                 default:
1160                         return &amd_f15_PMC50;
1161                 }
1162         case AMD_EVENT_CU:
1163         case AMD_EVENT_IC_DE:
1164         case AMD_EVENT_DE:
1165                 switch (event_code) {
1166                 case 0x08F:
1167                 case 0x187:
1168                 case 0x188:
1169                         return &amd_f15_PMC0;
1170                 case 0x0DB ... 0x0DF:
1171                 case 0x1D6:
1172                 case 0x1D8:
1173                         return &amd_f15_PMC50;
1174                 default:
1175                         return &amd_f15_PMC20;
1176                 }
1177         case AMD_EVENT_NB:
1178                 /* moved to uncore.c */
1179                 return &emptyconstraint;
1180         default:
1181                 return &emptyconstraint;
1182         }
1183 }
1184
1185 static struct event_constraint pair_constraint;
1186
1187 static struct event_constraint *
1188 amd_get_event_constraints_f17h(struct cpu_hw_events *cpuc, int idx,
1189                                struct perf_event *event)
1190 {
1191         struct hw_perf_event *hwc = &event->hw;
1192
1193         if (amd_is_pair_event_code(hwc))
1194                 return &pair_constraint;
1195
1196         return &unconstrained;
1197 }
1198
1199 static void amd_put_event_constraints_f17h(struct cpu_hw_events *cpuc,
1200                                            struct perf_event *event)
1201 {
1202         struct hw_perf_event *hwc = &event->hw;
1203
1204         if (is_counter_pair(hwc))
1205                 --cpuc->n_pair;
1206 }
1207
1208 /*
1209  * Because of the way BRS operates with an inactive and active phases, and
1210  * the link to one counter, it is not possible to have two events using BRS
1211  * scheduled at the same time. There would be an issue with enforcing the
1212  * period of each one and given that the BRS saturates, it would not be possible
1213  * to guarantee correlated content for all events. Therefore, in situations
1214  * where multiple events want to use BRS, the kernel enforces mutual exclusion.
1215  * Exclusion is enforced by choosing only one counter for events using BRS.
1216  * The event scheduling logic will then automatically multiplex the
1217  * events and ensure that at most one event is actively using BRS.
1218  *
1219  * The BRS counter could be any counter, but there is no constraint on Fam19h,
1220  * therefore all counters are equal and thus we pick the first one: PMC0
1221  */
1222 static struct event_constraint amd_fam19h_brs_cntr0_constraint =
1223         EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK);
1224
1225 static struct event_constraint amd_fam19h_brs_pair_cntr0_constraint =
1226         __EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK, 1, 0, PERF_X86_EVENT_PAIR);
1227
1228 static struct event_constraint *
1229 amd_get_event_constraints_f19h(struct cpu_hw_events *cpuc, int idx,
1230                           struct perf_event *event)
1231 {
1232         struct hw_perf_event *hwc = &event->hw;
1233         bool has_brs = has_amd_brs(hwc);
1234
1235         /*
1236          * In case BRS is used with an event requiring a counter pair,
1237          * the kernel allows it but only on counter 0 & 1 to enforce
1238          * multiplexing requiring to protect BRS in case of multiple
1239          * BRS users
1240          */
1241         if (amd_is_pair_event_code(hwc)) {
1242                 return has_brs ? &amd_fam19h_brs_pair_cntr0_constraint
1243                                : &pair_constraint;
1244         }
1245
1246         if (has_brs)
1247                 return &amd_fam19h_brs_cntr0_constraint;
1248
1249         return &unconstrained;
1250 }
1251
1252
1253 static ssize_t amd_event_sysfs_show(char *page, u64 config)
1254 {
1255         u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT) |
1256                     (config & AMD64_EVENTSEL_EVENT) >> 24;
1257
1258         return x86_event_sysfs_show(page, config, event);
1259 }
1260
1261 static void amd_pmu_limit_period(struct perf_event *event, s64 *left)
1262 {
1263         /*
1264          * Decrease period by the depth of the BRS feature to get the last N
1265          * taken branches and approximate the desired period
1266          */
1267         if (has_branch_stack(event) && *left > x86_pmu.lbr_nr)
1268                 *left -= x86_pmu.lbr_nr;
1269 }
1270
1271 static __initconst const struct x86_pmu amd_pmu = {
1272         .name                   = "AMD",
1273         .handle_irq             = amd_pmu_handle_irq,
1274         .disable_all            = amd_pmu_disable_all,
1275         .enable_all             = amd_pmu_enable_all,
1276         .enable                 = amd_pmu_enable_event,
1277         .disable                = amd_pmu_disable_event,
1278         .hw_config              = amd_pmu_hw_config,
1279         .schedule_events        = x86_schedule_events,
1280         .eventsel               = MSR_K7_EVNTSEL0,
1281         .perfctr                = MSR_K7_PERFCTR0,
1282         .addr_offset            = amd_pmu_addr_offset,
1283         .event_map              = amd_pmu_event_map,
1284         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
1285         .num_counters           = AMD64_NUM_COUNTERS,
1286         .add                    = amd_pmu_add_event,
1287         .del                    = amd_pmu_del_event,
1288         .cntval_bits            = 48,
1289         .cntval_mask            = (1ULL << 48) - 1,
1290         .apic                   = 1,
1291         /* use highest bit to detect overflow */
1292         .max_period             = (1ULL << 47) - 1,
1293         .get_event_constraints  = amd_get_event_constraints,
1294         .put_event_constraints  = amd_put_event_constraints,
1295
1296         .format_attrs           = amd_format_attr,
1297         .events_sysfs_show      = amd_event_sysfs_show,
1298
1299         .cpu_prepare            = amd_pmu_cpu_prepare,
1300         .cpu_starting           = amd_pmu_cpu_starting,
1301         .cpu_dead               = amd_pmu_cpu_dead,
1302
1303         .amd_nb_constraints     = 1,
1304 };
1305
1306 static ssize_t branches_show(struct device *cdev,
1307                               struct device_attribute *attr,
1308                               char *buf)
1309 {
1310         return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu.lbr_nr);
1311 }
1312
1313 static DEVICE_ATTR_RO(branches);
1314
1315 static struct attribute *amd_pmu_branches_attrs[] = {
1316         &dev_attr_branches.attr,
1317         NULL,
1318 };
1319
1320 static umode_t
1321 amd_branches_is_visible(struct kobject *kobj, struct attribute *attr, int i)
1322 {
1323         return x86_pmu.lbr_nr ? attr->mode : 0;
1324 }
1325
1326 static struct attribute_group group_caps_amd_branches = {
1327         .name  = "caps",
1328         .attrs = amd_pmu_branches_attrs,
1329         .is_visible = amd_branches_is_visible,
1330 };
1331
1332 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
1333
1334 EVENT_ATTR_STR(branch-brs, amd_branch_brs,
1335                "event=" __stringify(AMD_FAM19H_BRS_EVENT)"\n");
1336
1337 static struct attribute *amd_brs_events_attrs[] = {
1338         EVENT_PTR(amd_branch_brs),
1339         NULL,
1340 };
1341
1342 static umode_t
1343 amd_brs_is_visible(struct kobject *kobj, struct attribute *attr, int i)
1344 {
1345         return static_cpu_has(X86_FEATURE_BRS) && x86_pmu.lbr_nr ?
1346                attr->mode : 0;
1347 }
1348
1349 static struct attribute_group group_events_amd_brs = {
1350         .name       = "events",
1351         .attrs      = amd_brs_events_attrs,
1352         .is_visible = amd_brs_is_visible,
1353 };
1354
1355 #endif  /* CONFIG_PERF_EVENTS_AMD_BRS */
1356
1357 static const struct attribute_group *amd_attr_update[] = {
1358         &group_caps_amd_branches,
1359 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
1360         &group_events_amd_brs,
1361 #endif
1362         NULL,
1363 };
1364
1365 static int __init amd_core_pmu_init(void)
1366 {
1367         union cpuid_0x80000022_ebx ebx;
1368         u64 even_ctr_mask = 0ULL;
1369         int i;
1370
1371         if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
1372                 return 0;
1373
1374         /* Avoid calculating the value each time in the NMI handler */
1375         perf_nmi_window = msecs_to_jiffies(100);
1376
1377         /*
1378          * If core performance counter extensions exists, we must use
1379          * MSR_F15H_PERF_CTL/MSR_F15H_PERF_CTR msrs. See also
1380          * amd_pmu_addr_offset().
1381          */
1382         x86_pmu.eventsel        = MSR_F15H_PERF_CTL;
1383         x86_pmu.perfctr         = MSR_F15H_PERF_CTR;
1384         x86_pmu.num_counters    = AMD64_NUM_COUNTERS_CORE;
1385
1386         /* Check for Performance Monitoring v2 support */
1387         if (boot_cpu_has(X86_FEATURE_PERFMON_V2)) {
1388                 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
1389
1390                 /* Update PMU version for later usage */
1391                 x86_pmu.version = 2;
1392
1393                 /* Find the number of available Core PMCs */
1394                 x86_pmu.num_counters = ebx.split.num_core_pmc;
1395
1396                 amd_pmu_global_cntr_mask = (1ULL << x86_pmu.num_counters) - 1;
1397
1398                 /* Update PMC handling functions */
1399                 x86_pmu.enable_all = amd_pmu_v2_enable_all;
1400                 x86_pmu.disable_all = amd_pmu_v2_disable_all;
1401                 x86_pmu.enable = amd_pmu_v2_enable_event;
1402                 x86_pmu.handle_irq = amd_pmu_v2_handle_irq;
1403                 static_call_update(amd_pmu_test_overflow, amd_pmu_test_overflow_status);
1404         }
1405
1406         /*
1407          * AMD Core perfctr has separate MSRs for the NB events, see
1408          * the amd/uncore.c driver.
1409          */
1410         x86_pmu.amd_nb_constraints = 0;
1411
1412         if (boot_cpu_data.x86 == 0x15) {
1413                 pr_cont("Fam15h ");
1414                 x86_pmu.get_event_constraints = amd_get_event_constraints_f15h;
1415         }
1416         if (boot_cpu_data.x86 >= 0x17) {
1417                 pr_cont("Fam17h+ ");
1418                 /*
1419                  * Family 17h and compatibles have constraints for Large
1420                  * Increment per Cycle events: they may only be assigned an
1421                  * even numbered counter that has a consecutive adjacent odd
1422                  * numbered counter following it.
1423                  */
1424                 for (i = 0; i < x86_pmu.num_counters - 1; i += 2)
1425                         even_ctr_mask |= BIT_ULL(i);
1426
1427                 pair_constraint = (struct event_constraint)
1428                                     __EVENT_CONSTRAINT(0, even_ctr_mask, 0,
1429                                     x86_pmu.num_counters / 2, 0,
1430                                     PERF_X86_EVENT_PAIR);
1431
1432                 x86_pmu.get_event_constraints = amd_get_event_constraints_f17h;
1433                 x86_pmu.put_event_constraints = amd_put_event_constraints_f17h;
1434                 x86_pmu.perf_ctr_pair_en = AMD_MERGE_EVENT_ENABLE;
1435                 x86_pmu.flags |= PMU_FL_PAIR;
1436         }
1437
1438         /* LBR and BRS are mutually exclusive features */
1439         if (!amd_pmu_lbr_init()) {
1440                 /* LBR requires flushing on context switch */
1441                 x86_pmu.sched_task = amd_pmu_lbr_sched_task;
1442                 static_call_update(amd_pmu_branch_hw_config, amd_pmu_lbr_hw_config);
1443                 static_call_update(amd_pmu_branch_reset, amd_pmu_lbr_reset);
1444                 static_call_update(amd_pmu_branch_add, amd_pmu_lbr_add);
1445                 static_call_update(amd_pmu_branch_del, amd_pmu_lbr_del);
1446         } else if (!amd_brs_init()) {
1447                 /*
1448                  * BRS requires special event constraints and flushing on ctxsw.
1449                  */
1450                 x86_pmu.get_event_constraints = amd_get_event_constraints_f19h;
1451                 x86_pmu.sched_task = amd_pmu_brs_sched_task;
1452                 x86_pmu.limit_period = amd_pmu_limit_period;
1453
1454                 static_call_update(amd_pmu_branch_hw_config, amd_brs_hw_config);
1455                 static_call_update(amd_pmu_branch_reset, amd_brs_reset);
1456                 static_call_update(amd_pmu_branch_add, amd_pmu_brs_add);
1457                 static_call_update(amd_pmu_branch_del, amd_pmu_brs_del);
1458
1459                 /*
1460                  * put_event_constraints callback same as Fam17h, set above
1461                  */
1462
1463                 /* branch sampling must be stopped when entering low power */
1464                 amd_brs_lopwr_init();
1465         }
1466
1467         x86_pmu.attr_update = amd_attr_update;
1468
1469         pr_cont("core perfctr, ");
1470         return 0;
1471 }
1472
1473 __init int amd_pmu_init(void)
1474 {
1475         int ret;
1476
1477         /* Performance-monitoring supported from K7 and later: */
1478         if (boot_cpu_data.x86 < 6)
1479                 return -ENODEV;
1480
1481         x86_pmu = amd_pmu;
1482
1483         ret = amd_core_pmu_init();
1484         if (ret)
1485                 return ret;
1486
1487         if (num_possible_cpus() == 1) {
1488                 /*
1489                  * No point in allocating data structures to serialize
1490                  * against other CPUs, when there is only the one CPU.
1491                  */
1492                 x86_pmu.amd_nb_constraints = 0;
1493         }
1494
1495         if (boot_cpu_data.x86 >= 0x17)
1496                 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids_f17h, sizeof(hw_cache_event_ids));
1497         else
1498                 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids, sizeof(hw_cache_event_ids));
1499
1500         return 0;
1501 }
1502
1503 static inline void amd_pmu_reload_virt(void)
1504 {
1505         if (x86_pmu.version >= 2) {
1506                 /*
1507                  * Clear global enable bits, reprogram the PERF_CTL
1508                  * registers with updated perf_ctr_virt_mask and then
1509                  * set global enable bits once again
1510                  */
1511                 amd_pmu_v2_disable_all();
1512                 amd_pmu_enable_all(0);
1513                 amd_pmu_v2_enable_all(0);
1514                 return;
1515         }
1516
1517         amd_pmu_disable_all();
1518         amd_pmu_enable_all(0);
1519 }
1520
1521 void amd_pmu_enable_virt(void)
1522 {
1523         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1524
1525         cpuc->perf_ctr_virt_mask = 0;
1526
1527         /* Reload all events */
1528         amd_pmu_reload_virt();
1529 }
1530 EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
1531
1532 void amd_pmu_disable_virt(void)
1533 {
1534         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1535
1536         /*
1537          * We only mask out the Host-only bit so that host-only counting works
1538          * when SVM is disabled. If someone sets up a guest-only counter when
1539          * SVM is disabled the Guest-only bits still gets set and the counter
1540          * will not count anything.
1541          */
1542         cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
1543
1544         /* Reload all events */
1545         amd_pmu_reload_virt();
1546 }
1547 EXPORT_SYMBOL_GPL(amd_pmu_disable_virt);