arch/x86/events/amd/core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/perf_event.h>
   3 #include <linux/jump_label.h>
   4 #include <linux/export.h>
   5 #include <linux/types.h>
   6 #include <linux/init.h>
   7 #include <linux/slab.h>
   8 #include <linux/delay.h>
   9 #include <linux/jiffies.h>
  10 #include <asm/apicdef.h>
  11 #include <asm/apic.h>
  12 #include <asm/nmi.h>
  13
  14 #include "../perf_event.h"
  15
  16 static DEFINE_PER_CPU(unsigned long, perf_nmi_tstamp);
  17 static unsigned long perf_nmi_window;
  18
  19 /* AMD Event 0xFFF: Merge.  Used with Large Increment per Cycle events */
  20 #define AMD_MERGE_EVENT ((0xFULL << 32) | 0xFFULL)
  21 #define AMD_MERGE_EVENT_ENABLE (AMD_MERGE_EVENT | ARCH_PERFMON_EVENTSEL_ENABLE)
  22
  23 /* PMC Enable and Overflow bits for PerfCntrGlobal* registers */
  24 static u64 amd_pmu_global_cntr_mask __read_mostly;
  25
  26 static __initconst const u64 amd_hw_cache_event_ids
  27                                 [PERF_COUNT_HW_CACHE_MAX]
  28                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  29                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  30 {
  31  [ C(L1D) ] = {
  32         [ C(OP_READ) ] = {
  33                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
  34                 [ C(RESULT_MISS)   ] = 0x0141, /* Data Cache Misses          */
  35         },
  36         [ C(OP_WRITE) ] = {
  37                 [ C(RESULT_ACCESS) ] = 0,
  38                 [ C(RESULT_MISS)   ] = 0,
  39         },
  40         [ C(OP_PREFETCH) ] = {
  41                 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
  42                 [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
  43         },
  44  },
  45  [ C(L1I ) ] = {
  46         [ C(OP_READ) ] = {
  47                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
  48                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
  49         },
  50         [ C(OP_WRITE) ] = {
  51                 [ C(RESULT_ACCESS) ] = -1,
  52                 [ C(RESULT_MISS)   ] = -1,
  53         },
  54         [ C(OP_PREFETCH) ] = {
  55                 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
  56                 [ C(RESULT_MISS)   ] = 0,
  57         },
  58  },
  59  [ C(LL  ) ] = {
  60         [ C(OP_READ) ] = {
  61                 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
  62                 [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
  63         },
  64         [ C(OP_WRITE) ] = {
  65                 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
  66                 [ C(RESULT_MISS)   ] = 0,
  67         },
  68         [ C(OP_PREFETCH) ] = {
  69                 [ C(RESULT_ACCESS) ] = 0,
  70                 [ C(RESULT_MISS)   ] = 0,
  71         },
  72  },
  73  [ C(DTLB) ] = {
  74         [ C(OP_READ) ] = {
  75                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
  76                 [ C(RESULT_MISS)   ] = 0x0746, /* L1_DTLB_AND_L2_DLTB_MISS.ALL */
  77         },
  78         [ C(OP_WRITE) ] = {
  79                 [ C(RESULT_ACCESS) ] = 0,
  80                 [ C(RESULT_MISS)   ] = 0,
  81         },
  82         [ C(OP_PREFETCH) ] = {
  83                 [ C(RESULT_ACCESS) ] = 0,
  84                 [ C(RESULT_MISS)   ] = 0,
  85         },
  86  },
  87  [ C(ITLB) ] = {
  88         [ C(OP_READ) ] = {
  89                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
  90                 [ C(RESULT_MISS)   ] = 0x0385, /* L1_ITLB_AND_L2_ITLB_MISS.ALL */
  91         },
  92         [ C(OP_WRITE) ] = {
  93                 [ C(RESULT_ACCESS) ] = -1,
  94                 [ C(RESULT_MISS)   ] = -1,
  95         },
  96         [ C(OP_PREFETCH) ] = {
  97                 [ C(RESULT_ACCESS) ] = -1,
  98                 [ C(RESULT_MISS)   ] = -1,
  99         },
 100  },
 101  [ C(BPU ) ] = {
 102         [ C(OP_READ) ] = {
 103                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 104                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 105         },
 106         [ C(OP_WRITE) ] = {
 107                 [ C(RESULT_ACCESS) ] = -1,
 108                 [ C(RESULT_MISS)   ] = -1,
 109         },
 110         [ C(OP_PREFETCH) ] = {
 111                 [ C(RESULT_ACCESS) ] = -1,
 112                 [ C(RESULT_MISS)   ] = -1,
 113         },
 114  },
 115  [ C(NODE) ] = {
 116         [ C(OP_READ) ] = {
 117                 [ C(RESULT_ACCESS) ] = 0xb8e9, /* CPU Request to Memory, l+r */
 118                 [ C(RESULT_MISS)   ] = 0x98e9, /* CPU Request to Memory, r   */
 119         },
 120         [ C(OP_WRITE) ] = {
 121                 [ C(RESULT_ACCESS) ] = -1,
 122                 [ C(RESULT_MISS)   ] = -1,
 123         },
 124         [ C(OP_PREFETCH) ] = {
 125                 [ C(RESULT_ACCESS) ] = -1,
 126                 [ C(RESULT_MISS)   ] = -1,
 127         },
 128  },
 129 };
 130
 131 static __initconst const u64 amd_hw_cache_event_ids_f17h
 132                                 [PERF_COUNT_HW_CACHE_MAX]
 133                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 134                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 135 [C(L1D)] = {
 136         [C(OP_READ)] = {
 137                 [C(RESULT_ACCESS)] = 0x0040, /* Data Cache Accesses */
 138                 [C(RESULT_MISS)]   = 0xc860, /* L2$ access from DC Miss */
 139         },
 140         [C(OP_WRITE)] = {
 141                 [C(RESULT_ACCESS)] = 0,
 142                 [C(RESULT_MISS)]   = 0,
 143         },
 144         [C(OP_PREFETCH)] = {
 145                 [C(RESULT_ACCESS)] = 0xff5a, /* h/w prefetch DC Fills */
 146                 [C(RESULT_MISS)]   = 0,
 147         },
 148 },
 149 [C(L1I)] = {
 150         [C(OP_READ)] = {
 151                 [C(RESULT_ACCESS)] = 0x0080, /* Instruction cache fetches  */
 152                 [C(RESULT_MISS)]   = 0x0081, /* Instruction cache misses   */
 153         },
 154         [C(OP_WRITE)] = {
 155                 [C(RESULT_ACCESS)] = -1,
 156                 [C(RESULT_MISS)]   = -1,
 157         },
 158         [C(OP_PREFETCH)] = {
 159                 [C(RESULT_ACCESS)] = 0,
 160                 [C(RESULT_MISS)]   = 0,
 161         },
 162 },
 163 [C(LL)] = {
 164         [C(OP_READ)] = {
 165                 [C(RESULT_ACCESS)] = 0,
 166                 [C(RESULT_MISS)]   = 0,
 167         },
 168         [C(OP_WRITE)] = {
 169                 [C(RESULT_ACCESS)] = 0,
 170                 [C(RESULT_MISS)]   = 0,
 171         },
 172         [C(OP_PREFETCH)] = {
 173                 [C(RESULT_ACCESS)] = 0,
 174                 [C(RESULT_MISS)]   = 0,
 175         },
 176 },
 177 [C(DTLB)] = {
 178         [C(OP_READ)] = {
 179                 [C(RESULT_ACCESS)] = 0xff45, /* All L2 DTLB accesses */
 180                 [C(RESULT_MISS)]   = 0xf045, /* L2 DTLB misses (PT walks) */
 181         },
 182         [C(OP_WRITE)] = {
 183                 [C(RESULT_ACCESS)] = 0,
 184                 [C(RESULT_MISS)]   = 0,
 185         },
 186         [C(OP_PREFETCH)] = {
 187                 [C(RESULT_ACCESS)] = 0,
 188                 [C(RESULT_MISS)]   = 0,
 189         },
 190 },
 191 [C(ITLB)] = {
 192         [C(OP_READ)] = {
 193                 [C(RESULT_ACCESS)] = 0x0084, /* L1 ITLB misses, L2 ITLB hits */
 194                 [C(RESULT_MISS)]   = 0xff85, /* L1 ITLB misses, L2 misses */
 195         },
 196         [C(OP_WRITE)] = {
 197                 [C(RESULT_ACCESS)] = -1,
 198                 [C(RESULT_MISS)]   = -1,
 199         },
 200         [C(OP_PREFETCH)] = {
 201                 [C(RESULT_ACCESS)] = -1,
 202                 [C(RESULT_MISS)]   = -1,
 203         },
 204 },
 205 [C(BPU)] = {
 206         [C(OP_READ)] = {
 207                 [C(RESULT_ACCESS)] = 0x00c2, /* Retired Branch Instr.      */
 208                 [C(RESULT_MISS)]   = 0x00c3, /* Retired Mispredicted BI    */
 209         },
 210         [C(OP_WRITE)] = {
 211                 [C(RESULT_ACCESS)] = -1,
 212                 [C(RESULT_MISS)]   = -1,
 213         },
 214         [C(OP_PREFETCH)] = {
 215                 [C(RESULT_ACCESS)] = -1,
 216                 [C(RESULT_MISS)]   = -1,
 217         },
 218 },
 219 [C(NODE)] = {
 220         [C(OP_READ)] = {
 221                 [C(RESULT_ACCESS)] = 0,
 222                 [C(RESULT_MISS)]   = 0,
 223         },
 224         [C(OP_WRITE)] = {
 225                 [C(RESULT_ACCESS)] = -1,
 226                 [C(RESULT_MISS)]   = -1,
 227         },
 228         [C(OP_PREFETCH)] = {
 229                 [C(RESULT_ACCESS)] = -1,
 230                 [C(RESULT_MISS)]   = -1,
 231         },
 232 },
 233 };
 234
 235 /*
 236  * AMD Performance Monitor K7 and later, up to and including Family 16h:
 237  */
 238 static const u64 amd_perfmon_event_map[PERF_COUNT_HW_MAX] =
 239 {
 240         [PERF_COUNT_HW_CPU_CYCLES]              = 0x0076,
 241         [PERF_COUNT_HW_INSTRUCTIONS]            = 0x00c0,
 242         [PERF_COUNT_HW_CACHE_REFERENCES]        = 0x077d,
 243         [PERF_COUNT_HW_CACHE_MISSES]            = 0x077e,
 244         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = 0x00c2,
 245         [PERF_COUNT_HW_BRANCH_MISSES]           = 0x00c3,
 246         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */
 247         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = 0x00d1, /* "Dispatch stalls" event */
 248 };
 249
 250 /*
 251  * AMD Performance Monitor Family 17h and later:
 252  */
 253 static const u64 amd_f17h_perfmon_event_map[PERF_COUNT_HW_MAX] =
 254 {
 255         [PERF_COUNT_HW_CPU_CYCLES]              = 0x0076,
 256         [PERF_COUNT_HW_INSTRUCTIONS]            = 0x00c0,
 257         [PERF_COUNT_HW_CACHE_REFERENCES]        = 0xff60,
 258         [PERF_COUNT_HW_CACHE_MISSES]            = 0x0964,
 259         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = 0x00c2,
 260         [PERF_COUNT_HW_BRANCH_MISSES]           = 0x00c3,
 261         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x0287,
 262         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = 0x0187,
 263 };
 264
 265 static u64 amd_pmu_event_map(int hw_event)
 266 {
 267         if (boot_cpu_data.x86 >= 0x17)
 268                 return amd_f17h_perfmon_event_map[hw_event];
 269
 270         return amd_perfmon_event_map[hw_event];
 271 }
 272
 273 /*
 274  * Previously calculated offsets
 275  */
 276 static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly;
 277 static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly;
 278
 279 /*
 280  * Legacy CPUs:
 281  *   4 counters starting at 0xc0010000 each offset by 1
 282  *
 283  * CPUs with core performance counter extensions:
 284  *   6 counters starting at 0xc0010200 each offset by 2
 285  */
 286 static inline int amd_pmu_addr_offset(int index, bool eventsel)
 287 {
 288         int offset;
 289
 290         if (!index)
 291                 return index;
 292
 293         if (eventsel)
 294                 offset = event_offsets[index];
 295         else
 296                 offset = count_offsets[index];
 297
 298         if (offset)
 299                 return offset;
 300
 301         if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
 302                 offset = index;
 303         else
 304                 offset = index << 1;
 305
 306         if (eventsel)
 307                 event_offsets[index] = offset;
 308         else
 309                 count_offsets[index] = offset;
 310
 311         return offset;
 312 }
 313
 314 /*
 315  * AMD64 events are detected based on their event codes.
 316  */
 317 static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc)
 318 {
 319         return ((hwc->config >> 24) & 0x0f00) | (hwc->config & 0x00ff);
 320 }
 321
 322 static inline bool amd_is_pair_event_code(struct hw_perf_event *hwc)
 323 {
 324         if (!(x86_pmu.flags & PMU_FL_PAIR))
 325                 return false;
 326
 327         switch (amd_get_event_code(hwc)) {
 328         case 0x003:     return true;    /* Retired SSE/AVX FLOPs */
 329         default:        return false;
 330         }
 331 }
 332
 333 DEFINE_STATIC_CALL_RET0(amd_pmu_branch_hw_config, *x86_pmu.hw_config);
 334
 335 static int amd_core_hw_config(struct perf_event *event)
 336 {
 337         if (event->attr.exclude_host && event->attr.exclude_guest)
 338                 /*
 339                  * When HO == GO == 1 the hardware treats that as GO == HO == 0
 340                  * and will count in both modes. We don't want to count in that
 341                  * case so we emulate no-counting by setting US = OS = 0.
 342                  */
 343                 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
 344                                       ARCH_PERFMON_EVENTSEL_OS);
 345         else if (event->attr.exclude_host)
 346                 event->hw.config |= AMD64_EVENTSEL_GUESTONLY;
 347         else if (event->attr.exclude_guest)
 348                 event->hw.config |= AMD64_EVENTSEL_HOSTONLY;
 349
 350         if ((x86_pmu.flags & PMU_FL_PAIR) && amd_is_pair_event_code(&event->hw))
 351                 event->hw.flags |= PERF_X86_EVENT_PAIR;
 352
 353         if (has_branch_stack(event))
 354                 return static_call(amd_pmu_branch_hw_config)(event);
 355
 356         return 0;
 357 }
 358
 359 static inline int amd_is_nb_event(struct hw_perf_event *hwc)
 360 {
 361         return (hwc->config & 0xe0) == 0xe0;
 362 }
 363
 364 static inline int amd_has_nb(struct cpu_hw_events *cpuc)
 365 {
 366         struct amd_nb *nb = cpuc->amd_nb;
 367
 368         return nb && nb->nb_id != -1;
 369 }
 370
 371 static int amd_pmu_hw_config(struct perf_event *event)
 372 {
 373         int ret;
 374
 375         /* pass precise event sampling to ibs: */
 376         if (event->attr.precise_ip && get_ibs_caps())
 377                 return forward_event_to_ibs(event);
 378
 379         if (has_branch_stack(event) && !x86_pmu.lbr_nr)
 380                 return -EOPNOTSUPP;
 381
 382         ret = x86_pmu_hw_config(event);
 383         if (ret)
 384                 return ret;
 385
 386         if (event->attr.type == PERF_TYPE_RAW)
 387                 event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
 388
 389         return amd_core_hw_config(event);
 390 }
 391
 392 static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc,
 393                                            struct perf_event *event)
 394 {
 395         struct amd_nb *nb = cpuc->amd_nb;
 396         int i;
 397
 398         /*
 399          * need to scan whole list because event may not have
 400          * been assigned during scheduling
 401          *
 402          * no race condition possible because event can only
 403          * be removed on one CPU at a time AND PMU is disabled
 404          * when we come here
 405          */
 406         for (i = 0; i < x86_pmu.num_counters; i++) {
 407                 if (cmpxchg(nb->owners + i, event, NULL) == event)
 408                         break;
 409         }
 410 }
 411
 412  /*
 413   * AMD64 NorthBridge events need special treatment because
 414   * counter access needs to be synchronized across all cores
 415   * of a package. Refer to BKDG section 3.12
 416   *
 417   * NB events are events measuring L3 cache, Hypertransport
 418   * traffic. They are identified by an event code >= 0xe00.
 419   * They measure events on the NorthBride which is shared
 420   * by all cores on a package. NB events are counted on a
 421   * shared set of counters. When a NB event is programmed
 422   * in a counter, the data actually comes from a shared
 423   * counter. Thus, access to those counters needs to be
 424   * synchronized.
 425   *
 426   * We implement the synchronization such that no two cores
 427   * can be measuring NB events using the same counters. Thus,
 428   * we maintain a per-NB allocation table. The available slot
 429   * is propagated using the event_constraint structure.
 430   *
 431   * We provide only one choice for each NB event based on
 432   * the fact that only NB events have restrictions. Consequently,
 433   * if a counter is available, there is a guarantee the NB event
 434   * will be assigned to it. If no slot is available, an empty
 435   * constraint is returned and scheduling will eventually fail
 436   * for this event.
 437   *
 438   * Note that all cores attached the same NB compete for the same
 439   * counters to host NB events, this is why we use atomic ops. Some
 440   * multi-chip CPUs may have more than one NB.
 441   *
 442   * Given that resources are allocated (cmpxchg), they must be
 443   * eventually freed for others to use. This is accomplished by
 444   * calling __amd_put_nb_event_constraints()
 445   *
 446   * Non NB events are not impacted by this restriction.
 447   */
 448 static struct event_constraint *
 449 __amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
 450                                struct event_constraint *c)
 451 {
 452         struct hw_perf_event *hwc = &event->hw;
 453         struct amd_nb *nb = cpuc->amd_nb;
 454         struct perf_event *old;
 455         int idx, new = -1;
 456
 457         if (!c)
 458                 c = &unconstrained;
 459
 460         if (cpuc->is_fake)
 461                 return c;
 462
 463         /*
 464          * detect if already present, if so reuse
 465          *
 466          * cannot merge with actual allocation
 467          * because of possible holes
 468          *
 469          * event can already be present yet not assigned (in hwc->idx)
 470          * because of successive calls to x86_schedule_events() from
 471          * hw_perf_group_sched_in() without hw_perf_enable()
 472          */
 473         for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) {
 474                 if (new == -1 || hwc->idx == idx)
 475                         /* assign free slot, prefer hwc->idx */
 476                         old = cmpxchg(nb->owners + idx, NULL, event);
 477                 else if (nb->owners[idx] == event)
 478                         /* event already present */
 479                         old = event;
 480                 else
 481                         continue;
 482
 483                 if (old && old != event)
 484                         continue;
 485
 486                 /* reassign to this slot */
 487                 if (new != -1)
 488                         cmpxchg(nb->owners + new, event, NULL);
 489                 new = idx;
 490
 491                 /* already present, reuse */
 492                 if (old == event)
 493                         break;
 494         }
 495
 496         if (new == -1)
 497                 return &emptyconstraint;
 498
 499         return &nb->event_constraints[new];
 500 }
 501
 502 static struct amd_nb *amd_alloc_nb(int cpu)
 503 {
 504         struct amd_nb *nb;
 505         int i;
 506
 507         nb = kzalloc_node(sizeof(struct amd_nb), GFP_KERNEL, cpu_to_node(cpu));
 508         if (!nb)
 509                 return NULL;
 510
 511         nb->nb_id = -1;
 512
 513         /*
 514          * initialize all possible NB constraints
 515          */
 516         for (i = 0; i < x86_pmu.num_counters; i++) {
 517                 __set_bit(i, nb->event_constraints[i].idxmsk);
 518                 nb->event_constraints[i].weight = 1;
 519         }
 520         return nb;
 521 }
 522
 523 typedef void (amd_pmu_branch_reset_t)(void);
 524 DEFINE_STATIC_CALL_NULL(amd_pmu_branch_reset, amd_pmu_branch_reset_t);
 525
 526 static void amd_pmu_cpu_reset(int cpu)
 527 {
 528         if (x86_pmu.lbr_nr)
 529                 static_call(amd_pmu_branch_reset)();
 530
 531         if (x86_pmu.version < 2)
 532                 return;
 533
 534         /* Clear enable bits i.e. PerfCntrGlobalCtl.PerfCntrEn */
 535         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 0);
 536
 537         /*
 538          * Clear freeze and overflow bits i.e. PerfCntrGLobalStatus.LbrFreeze
 539          * and PerfCntrGLobalStatus.PerfCntrOvfl
 540          */
 541         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
 542                GLOBAL_STATUS_LBRS_FROZEN | amd_pmu_global_cntr_mask);
 543 }
 544
 545 static int amd_pmu_cpu_prepare(int cpu)
 546 {
 547         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
 548
 549         cpuc->lbr_sel = kzalloc_node(sizeof(struct er_account), GFP_KERNEL,
 550                                      cpu_to_node(cpu));
 551         if (!cpuc->lbr_sel)
 552                 return -ENOMEM;
 553
 554         WARN_ON_ONCE(cpuc->amd_nb);
 555
 556         if (!x86_pmu.amd_nb_constraints)
 557                 return 0;
 558
 559         cpuc->amd_nb = amd_alloc_nb(cpu);
 560         if (cpuc->amd_nb)
 561                 return 0;
 562
 563         kfree(cpuc->lbr_sel);
 564         cpuc->lbr_sel = NULL;
 565
 566         return -ENOMEM;
 567 }
 568
 569 static void amd_pmu_cpu_starting(int cpu)
 570 {
 571         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
 572         void **onln = &cpuc->kfree_on_online[X86_PERF_KFREE_SHARED];
 573         struct amd_nb *nb;
 574         int i, nb_id;
 575
 576         cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
 577         amd_pmu_cpu_reset(cpu);
 578
 579         if (!x86_pmu.amd_nb_constraints)
 580                 return;
 581
 582         nb_id = topology_amd_node_id(cpu);
 583         WARN_ON_ONCE(nb_id == BAD_APICID);
 584
 585         for_each_online_cpu(i) {
 586                 nb = per_cpu(cpu_hw_events, i).amd_nb;
 587                 if (WARN_ON_ONCE(!nb))
 588                         continue;
 589
 590                 if (nb->nb_id == nb_id) {
 591                         *onln = cpuc->amd_nb;
 592                         cpuc->amd_nb = nb;
 593                         break;
 594                 }
 595         }
 596
 597         cpuc->amd_nb->nb_id = nb_id;
 598         cpuc->amd_nb->refcnt++;
 599 }
 600
 601 static void amd_pmu_cpu_dead(int cpu)
 602 {
 603         struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
 604
 605         kfree(cpuhw->lbr_sel);
 606         cpuhw->lbr_sel = NULL;
 607         amd_pmu_cpu_reset(cpu);
 608
 609         if (!x86_pmu.amd_nb_constraints)
 610                 return;
 611
 612         if (cpuhw->amd_nb) {
 613                 struct amd_nb *nb = cpuhw->amd_nb;
 614
 615                 if (nb->nb_id == -1 || --nb->refcnt == 0)
 616                         kfree(nb);
 617
 618                 cpuhw->amd_nb = NULL;
 619         }
 620 }
 621
 622 static inline void amd_pmu_set_global_ctl(u64 ctl)
 623 {
 624         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl);
 625 }
 626
 627 static inline u64 amd_pmu_get_global_status(void)
 628 {
 629         u64 status;
 630
 631         /* PerfCntrGlobalStatus is read-only */
 632         rdmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status);
 633
 634         return status;
 635 }
 636
 637 static inline void amd_pmu_ack_global_status(u64 status)
 638 {
 639         /*
 640          * PerfCntrGlobalStatus is read-only but an overflow acknowledgment
 641          * mechanism exists; writing 1 to a bit in PerfCntrGlobalStatusClr
 642          * clears the same bit in PerfCntrGlobalStatus
 643          */
 644
 645         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, status);
 646 }
 647
 648 static bool amd_pmu_test_overflow_topbit(int idx)
 649 {
 650         u64 counter;
 651
 652         rdmsrl(x86_pmu_event_addr(idx), counter);
 653
 654         return !(counter & BIT_ULL(x86_pmu.cntval_bits - 1));
 655 }
 656
 657 static bool amd_pmu_test_overflow_status(int idx)
 658 {
 659         return amd_pmu_get_global_status() & BIT_ULL(idx);
 660 }
 661
 662 DEFINE_STATIC_CALL(amd_pmu_test_overflow, amd_pmu_test_overflow_topbit);
 663
 664 /*
 665  * When a PMC counter overflows, an NMI is used to process the event and
 666  * reset the counter. NMI latency can result in the counter being updated
 667  * before the NMI can run, which can result in what appear to be spurious
 668  * NMIs. This function is intended to wait for the NMI to run and reset
 669  * the counter to avoid possible unhandled NMI messages.
 670  */
 671 #define OVERFLOW_WAIT_COUNT     50
 672
 673 static void amd_pmu_wait_on_overflow(int idx)
 674 {
 675         unsigned int i;
 676
 677         /*
 678          * Wait for the counter to be reset if it has overflowed. This loop
 679          * should exit very, very quickly, but just in case, don't wait
 680          * forever...
 681          */
 682         for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) {
 683                 if (!static_call(amd_pmu_test_overflow)(idx))
 684                         break;
 685
 686                 /* Might be in IRQ context, so can't sleep */
 687                 udelay(1);
 688         }
 689 }
 690
 691 static void amd_pmu_check_overflow(void)
 692 {
 693         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 694         int idx;
 695
 696         /*
 697          * This shouldn't be called from NMI context, but add a safeguard here
 698          * to return, since if we're in NMI context we can't wait for an NMI
 699          * to reset an overflowed counter value.
 700          */
 701         if (in_nmi())
 702                 return;
 703
 704         /*
 705          * Check each counter for overflow and wait for it to be reset by the
 706          * NMI if it has overflowed. This relies on the fact that all active
 707          * counters are always enabled when this function is called and
 708          * ARCH_PERFMON_EVENTSEL_INT is always set.
 709          */
 710         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 711                 if (!test_bit(idx, cpuc->active_mask))
 712                         continue;
 713
 714                 amd_pmu_wait_on_overflow(idx);
 715         }
 716 }
 717
 718 static void amd_pmu_enable_event(struct perf_event *event)
 719 {
 720         x86_pmu_enable_event(event);
 721 }
 722
 723 static void amd_pmu_enable_all(int added)
 724 {
 725         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 726         int idx;
 727
 728         amd_brs_enable_all();
 729
 730         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 731                 /* only activate events which are marked as active */
 732                 if (!test_bit(idx, cpuc->active_mask))
 733                         continue;
 734
 735                 amd_pmu_enable_event(cpuc->events[idx]);
 736         }
 737 }
 738
 739 static void amd_pmu_v2_enable_event(struct perf_event *event)
 740 {
 741         struct hw_perf_event *hwc = &event->hw;
 742
 743         /*
 744          * Testing cpu_hw_events.enabled should be skipped in this case unlike
 745          * in x86_pmu_enable_event().
 746          *
 747          * Since cpu_hw_events.enabled is set only after returning from
 748          * x86_pmu_start(), the PMCs must be programmed and kept ready.
 749          * Counting starts only after x86_pmu_enable_all() is called.
 750          */
 751         __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
 752 }
 753
 754 static __always_inline void amd_pmu_core_enable_all(void)
 755 {
 756         amd_pmu_set_global_ctl(amd_pmu_global_cntr_mask);
 757 }
 758
 759 static void amd_pmu_v2_enable_all(int added)
 760 {
 761         amd_pmu_lbr_enable_all();
 762         amd_pmu_core_enable_all();
 763 }
 764
 765 static void amd_pmu_disable_event(struct perf_event *event)
 766 {
 767         x86_pmu_disable_event(event);
 768
 769         /*
 770          * This can be called from NMI context (via x86_pmu_stop). The counter
 771          * may have overflowed, but either way, we'll never see it get reset
 772          * by the NMI if we're already in the NMI. And the NMI latency support
 773          * below will take care of any pending NMI that might have been
 774          * generated by the overflow.
 775          */
 776         if (in_nmi())
 777                 return;
 778
 779         amd_pmu_wait_on_overflow(event->hw.idx);
 780 }
 781
 782 static void amd_pmu_disable_all(void)
 783 {
 784         amd_brs_disable_all();
 785         x86_pmu_disable_all();
 786         amd_pmu_check_overflow();
 787 }
 788
 789 static __always_inline void amd_pmu_core_disable_all(void)
 790 {
 791         amd_pmu_set_global_ctl(0);
 792 }
 793
 794 static void amd_pmu_v2_disable_all(void)
 795 {
 796         amd_pmu_core_disable_all();
 797         amd_pmu_lbr_disable_all();
 798         amd_pmu_check_overflow();
 799 }
 800
 801 DEFINE_STATIC_CALL_NULL(amd_pmu_branch_add, *x86_pmu.add);
 802
 803 static void amd_pmu_add_event(struct perf_event *event)
 804 {
 805         if (needs_branch_stack(event))
 806                 static_call(amd_pmu_branch_add)(event);
 807 }
 808
 809 DEFINE_STATIC_CALL_NULL(amd_pmu_branch_del, *x86_pmu.del);
 810
 811 static void amd_pmu_del_event(struct perf_event *event)
 812 {
 813         if (needs_branch_stack(event))
 814                 static_call(amd_pmu_branch_del)(event);
 815 }
 816
 817 /*
 818  * Because of NMI latency, if multiple PMC counters are active or other sources
 819  * of NMIs are received, the perf NMI handler can handle one or more overflowed
 820  * PMC counters outside of the NMI associated with the PMC overflow. If the NMI
 821  * doesn't arrive at the LAPIC in time to become a pending NMI, then the kernel
 822  * back-to-back NMI support won't be active. This PMC handler needs to take into
 823  * account that this can occur, otherwise this could result in unknown NMI
 824  * messages being issued. Examples of this is PMC overflow while in the NMI
 825  * handler when multiple PMCs are active or PMC overflow while handling some
 826  * other source of an NMI.
 827  *
 828  * Attempt to mitigate this by creating an NMI window in which un-handled NMIs
 829  * received during this window will be claimed. This prevents extending the
 830  * window past when it is possible that latent NMIs should be received. The
 831  * per-CPU perf_nmi_tstamp will be set to the window end time whenever perf has
 832  * handled a counter. When an un-handled NMI is received, it will be claimed
 833  * only if arriving within that window.
 834  */
 835 static inline int amd_pmu_adjust_nmi_window(int handled)
 836 {
 837         /*
 838          * If a counter was handled, record a timestamp such that un-handled
 839          * NMIs will be claimed if arriving within that window.
 840          */
 841         if (handled) {
 842                 this_cpu_write(perf_nmi_tstamp, jiffies + perf_nmi_window);
 843
 844                 return handled;
 845         }
 846
 847         if (time_after(jiffies, this_cpu_read(perf_nmi_tstamp)))
 848                 return NMI_DONE;
 849
 850         return NMI_HANDLED;
 851 }
 852
 853 static int amd_pmu_handle_irq(struct pt_regs *regs)
 854 {
 855         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 856         int handled;
 857         int pmu_enabled;
 858
 859         /*
 860          * Save the PMU state.
 861          * It needs to be restored when leaving the handler.
 862          */
 863         pmu_enabled = cpuc->enabled;
 864         cpuc->enabled = 0;
 865
 866         amd_brs_disable_all();
 867
 868         /* Drain BRS is in use (could be inactive) */
 869         if (cpuc->lbr_users)
 870                 amd_brs_drain();
 871
 872         /* Process any counter overflows */
 873         handled = x86_pmu_handle_irq(regs);
 874
 875         cpuc->enabled = pmu_enabled;
 876         if (pmu_enabled)
 877                 amd_brs_enable_all();
 878
 879         return amd_pmu_adjust_nmi_window(handled);
 880 }
 881
 882 static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 883 {
 884         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 885         struct perf_sample_data data;
 886         struct hw_perf_event *hwc;
 887         struct perf_event *event;
 888         int handled = 0, idx;
 889         u64 reserved, status, mask;
 890         bool pmu_enabled;
 891
 892         /*
 893          * Save the PMU state as it needs to be restored when leaving the
 894          * handler
 895          */
 896         pmu_enabled = cpuc->enabled;
 897         cpuc->enabled = 0;
 898
 899         /* Stop counting but do not disable LBR */
 900         amd_pmu_core_disable_all();
 901
 902         status = amd_pmu_get_global_status();
 903
 904         /* Check if any overflows are pending */
 905         if (!status)
 906                 goto done;
 907
 908         /* Read branch records before unfreezing */
 909         if (status & GLOBAL_STATUS_LBRS_FROZEN) {
 910                 amd_pmu_lbr_read();
 911                 status &= ~GLOBAL_STATUS_LBRS_FROZEN;
 912         }
 913
 914         reserved = status & ~amd_pmu_global_cntr_mask;
 915         if (reserved)
 916                 pr_warn_once("Reserved PerfCntrGlobalStatus bits are set (0x%llx), please consider updating microcode\n",
 917                              reserved);
 918
 919         /* Clear any reserved bits set by buggy microcode */
 920         status &= amd_pmu_global_cntr_mask;
 921
 922         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 923                 if (!test_bit(idx, cpuc->active_mask))
 924                         continue;
 925
 926                 event = cpuc->events[idx];
 927                 hwc = &event->hw;
 928                 x86_perf_event_update(event);
 929                 mask = BIT_ULL(idx);
 930
 931                 if (!(status & mask))
 932                         continue;
 933
 934                 /* Event overflow */
 935                 handled++;
 936                 status &= ~mask;
 937                 perf_sample_data_init(&data, 0, hwc->last_period);
 938
 939                 if (!x86_perf_event_set_period(event))
 940                         continue;
 941
 942                 if (has_branch_stack(event))
 943                         perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL);
 944
 945                 if (perf_event_overflow(event, &data, regs))
 946                         x86_pmu_stop(event, 0);
 947         }
 948
 949         /*
 950          * It should never be the case that some overflows are not handled as
 951          * the corresponding PMCs are expected to be inactive according to the
 952          * active_mask
 953          */
 954         WARN_ON(status > 0);
 955
 956         /* Clear overflow and freeze bits */
 957         amd_pmu_ack_global_status(~status);
 958
 959         /*
 960          * Unmasking the LVTPC is not required as the Mask (M) bit of the LVT
 961          * PMI entry is not set by the local APIC when a PMC overflow occurs
 962          */
 963         inc_irq_stat(apic_perf_irqs);
 964
 965 done:
 966         cpuc->enabled = pmu_enabled;
 967
 968         /* Resume counting only if PMU is active */
 969         if (pmu_enabled)
 970                 amd_pmu_core_enable_all();
 971
 972         return amd_pmu_adjust_nmi_window(handled);
 973 }
 974
 975 static struct event_constraint *
 976 amd_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
 977                           struct perf_event *event)
 978 {
 979         /*
 980          * if not NB event or no NB, then no constraints
 981          */
 982         if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)))
 983                 return &unconstrained;
 984
 985         return __amd_get_nb_event_constraints(cpuc, event, NULL);
 986 }
 987
 988 static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
 989                                       struct perf_event *event)
 990 {
 991         if (amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))
 992                 __amd_put_nb_event_constraints(cpuc, event);
 993 }
 994
 995 PMU_FORMAT_ATTR(event,  "config:0-7,32-35");
 996 PMU_FORMAT_ATTR(umask,  "config:8-15"   );
 997 PMU_FORMAT_ATTR(edge,   "config:18"     );
 998 PMU_FORMAT_ATTR(inv,    "config:23"     );
 999 PMU_FORMAT_ATTR(cmask,  "config:24-31"  );
1000
1001 static struct attribute *amd_format_attr[] = {
1002         &format_attr_event.attr,
1003         &format_attr_umask.attr,
1004         &format_attr_edge.attr,
1005         &format_attr_inv.attr,
1006         &format_attr_cmask.attr,
1007         NULL,
1008 };
1009
1010 /* AMD Family 15h */
1011
1012 #define AMD_EVENT_TYPE_MASK     0x000000F0ULL
1013
1014 #define AMD_EVENT_FP            0x00000000ULL ... 0x00000010ULL
1015 #define AMD_EVENT_LS            0x00000020ULL ... 0x00000030ULL
1016 #define AMD_EVENT_DC            0x00000040ULL ... 0x00000050ULL
1017 #define AMD_EVENT_CU            0x00000060ULL ... 0x00000070ULL
1018 #define AMD_EVENT_IC_DE         0x00000080ULL ... 0x00000090ULL
1019 #define AMD_EVENT_EX_LS         0x000000C0ULL
1020 #define AMD_EVENT_DE            0x000000D0ULL
1021 #define AMD_EVENT_NB            0x000000E0ULL ... 0x000000F0ULL
1022
1023 /*
1024  * AMD family 15h event code/PMC mappings:
1025  *
1026  * type = event_code & 0x0F0:
1027  *
1028  * 0x000        FP      PERF_CTL[5:3]
1029  * 0x010        FP      PERF_CTL[5:3]
1030  * 0x020        LS      PERF_CTL[5:0]
1031  * 0x030        LS      PERF_CTL[5:0]
1032  * 0x040        DC      PERF_CTL[5:0]
1033  * 0x050        DC      PERF_CTL[5:0]
1034  * 0x060        CU      PERF_CTL[2:0]
1035  * 0x070        CU      PERF_CTL[2:0]
1036  * 0x080        IC/DE   PERF_CTL[2:0]
1037  * 0x090        IC/DE   PERF_CTL[2:0]
1038  * 0x0A0        ---
1039  * 0x0B0        ---
1040  * 0x0C0        EX/LS   PERF_CTL[5:0]
1041  * 0x0D0        DE      PERF_CTL[2:0]
1042  * 0x0E0        NB      NB_PERF_CTL[3:0]
1043  * 0x0F0        NB      NB_PERF_CTL[3:0]
1044  *
1045  * Exceptions:
1046  *
1047  * 0x000        FP      PERF_CTL[3], PERF_CTL[5:3] (*)
1048  * 0x003        FP      PERF_CTL[3]
1049  * 0x004        FP      PERF_CTL[3], PERF_CTL[5:3] (*)
1050  * 0x00B        FP      PERF_CTL[3]
1051  * 0x00D        FP      PERF_CTL[3]
1052  * 0x023        DE      PERF_CTL[2:0]
1053  * 0x02D        LS      PERF_CTL[3]
1054  * 0x02E        LS      PERF_CTL[3,0]
1055  * 0x031        LS      PERF_CTL[2:0] (**)
1056  * 0x043        CU      PERF_CTL[2:0]
1057  * 0x045        CU      PERF_CTL[2:0]
1058  * 0x046        CU      PERF_CTL[2:0]
1059  * 0x054        CU      PERF_CTL[2:0]
1060  * 0x055        CU      PERF_CTL[2:0]
1061  * 0x08F        IC      PERF_CTL[0]
1062  * 0x187        DE      PERF_CTL[0]
1063  * 0x188        DE      PERF_CTL[0]
1064  * 0x0DB        EX      PERF_CTL[5:0]
1065  * 0x0DC        LS      PERF_CTL[5:0]
1066  * 0x0DD        LS      PERF_CTL[5:0]
1067  * 0x0DE        LS      PERF_CTL[5:0]
1068  * 0x0DF        LS      PERF_CTL[5:0]
1069  * 0x1C0        EX      PERF_CTL[5:3]
1070  * 0x1D6        EX      PERF_CTL[5:0]
1071  * 0x1D8        EX      PERF_CTL[5:0]
1072  *
1073  * (*)  depending on the umask all FPU counters may be used
1074  * (**) only one unitmask enabled at a time
1075  */
1076
1077 static struct event_constraint amd_f15_PMC0  = EVENT_CONSTRAINT(0, 0x01, 0);
1078 static struct event_constraint amd_f15_PMC20 = EVENT_CONSTRAINT(0, 0x07, 0);
1079 static struct event_constraint amd_f15_PMC3  = EVENT_CONSTRAINT(0, 0x08, 0);
1080 static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
1081 static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0);
1082 static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
1083
1084 static struct event_constraint *
1085 amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, int idx,
1086                                struct perf_event *event)
1087 {
1088         struct hw_perf_event *hwc = &event->hw;
1089         unsigned int event_code = amd_get_event_code(hwc);
1090
1091         switch (event_code & AMD_EVENT_TYPE_MASK) {
1092         case AMD_EVENT_FP:
1093                 switch (event_code) {
1094                 case 0x000:
1095                         if (!(hwc->config & 0x0000F000ULL))
1096                                 break;
1097                         if (!(hwc->config & 0x00000F00ULL))
1098                                 break;
1099                         return &amd_f15_PMC3;
1100                 case 0x004:
1101                         if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
1102                                 break;
1103                         return &amd_f15_PMC3;
1104                 case 0x003:
1105                 case 0x00B:
1106                 case 0x00D:
1107                         return &amd_f15_PMC3;
1108                 }
1109                 return &amd_f15_PMC53;
1110         case AMD_EVENT_LS:
1111         case AMD_EVENT_DC:
1112         case AMD_EVENT_EX_LS:
1113                 switch (event_code) {
1114                 case 0x023:
1115                 case 0x043:
1116                 case 0x045:
1117                 case 0x046:
1118                 case 0x054:
1119                 case 0x055:
1120                         return &amd_f15_PMC20;
1121                 case 0x02D:
1122                         return &amd_f15_PMC3;
1123                 case 0x02E:
1124                         return &amd_f15_PMC30;
1125                 case 0x031:
1126                         if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
1127                                 return &amd_f15_PMC20;
1128                         return &emptyconstraint;
1129                 case 0x1C0:
1130                         return &amd_f15_PMC53;
1131                 default:
1132                         return &amd_f15_PMC50;
1133                 }
1134         case AMD_EVENT_CU:
1135         case AMD_EVENT_IC_DE:
1136         case AMD_EVENT_DE:
1137                 switch (event_code) {
1138                 case 0x08F:
1139                 case 0x187:
1140                 case 0x188:
1141                         return &amd_f15_PMC0;
1142                 case 0x0DB ... 0x0DF:
1143                 case 0x1D6:
1144                 case 0x1D8:
1145                         return &amd_f15_PMC50;
1146                 default:
1147                         return &amd_f15_PMC20;
1148                 }
1149         case AMD_EVENT_NB:
1150                 /* moved to uncore.c */
1151                 return &emptyconstraint;
1152         default:
1153                 return &emptyconstraint;
1154         }
1155 }
1156
1157 static struct event_constraint pair_constraint;
1158
1159 static struct event_constraint *
1160 amd_get_event_constraints_f17h(struct cpu_hw_events *cpuc, int idx,
1161                                struct perf_event *event)
1162 {
1163         struct hw_perf_event *hwc = &event->hw;
1164
1165         if (amd_is_pair_event_code(hwc))
1166                 return &pair_constraint;
1167
1168         return &unconstrained;
1169 }
1170
1171 static void amd_put_event_constraints_f17h(struct cpu_hw_events *cpuc,
1172                                            struct perf_event *event)
1173 {
1174         struct hw_perf_event *hwc = &event->hw;
1175
1176         if (is_counter_pair(hwc))
1177                 --cpuc->n_pair;
1178 }
1179
1180 /*
1181  * Because of the way BRS operates with an inactive and active phases, and
1182  * the link to one counter, it is not possible to have two events using BRS
1183  * scheduled at the same time. There would be an issue with enforcing the
1184  * period of each one and given that the BRS saturates, it would not be possible
1185  * to guarantee correlated content for all events. Therefore, in situations
1186  * where multiple events want to use BRS, the kernel enforces mutual exclusion.
1187  * Exclusion is enforced by choosing only one counter for events using BRS.
1188  * The event scheduling logic will then automatically multiplex the
1189  * events and ensure that at most one event is actively using BRS.
1190  *
1191  * The BRS counter could be any counter, but there is no constraint on Fam19h,
1192  * therefore all counters are equal and thus we pick the first one: PMC0
1193  */
1194 static struct event_constraint amd_fam19h_brs_cntr0_constraint =
1195         EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK);
1196
1197 static struct event_constraint amd_fam19h_brs_pair_cntr0_constraint =
1198         __EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK, 1, 0, PERF_X86_EVENT_PAIR);
1199
1200 static struct event_constraint *
1201 amd_get_event_constraints_f19h(struct cpu_hw_events *cpuc, int idx,
1202                           struct perf_event *event)
1203 {
1204         struct hw_perf_event *hwc = &event->hw;
1205         bool has_brs = has_amd_brs(hwc);
1206
1207         /*
1208          * In case BRS is used with an event requiring a counter pair,
1209          * the kernel allows it but only on counter 0 & 1 to enforce
1210          * multiplexing requiring to protect BRS in case of multiple
1211          * BRS users
1212          */
1213         if (amd_is_pair_event_code(hwc)) {
1214                 return has_brs ? &amd_fam19h_brs_pair_cntr0_constraint
1215                                : &pair_constraint;
1216         }
1217
1218         if (has_brs)
1219                 return &amd_fam19h_brs_cntr0_constraint;
1220
1221         return &unconstrained;
1222 }
1223
1224
1225 static ssize_t amd_event_sysfs_show(char *page, u64 config)
1226 {
1227         u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT) |
1228                     (config & AMD64_EVENTSEL_EVENT) >> 24;
1229
1230         return x86_event_sysfs_show(page, config, event);
1231 }
1232
1233 static void amd_pmu_limit_period(struct perf_event *event, s64 *left)
1234 {
1235         /*
1236          * Decrease period by the depth of the BRS feature to get the last N
1237          * taken branches and approximate the desired period
1238          */
1239         if (has_branch_stack(event) && *left > x86_pmu.lbr_nr)
1240                 *left -= x86_pmu.lbr_nr;
1241 }
1242
1243 static __initconst const struct x86_pmu amd_pmu = {
1244         .name                   = "AMD",
1245         .handle_irq             = amd_pmu_handle_irq,
1246         .disable_all            = amd_pmu_disable_all,
1247         .enable_all             = amd_pmu_enable_all,
1248         .enable                 = amd_pmu_enable_event,
1249         .disable                = amd_pmu_disable_event,
1250         .hw_config              = amd_pmu_hw_config,
1251         .schedule_events        = x86_schedule_events,
1252         .eventsel               = MSR_K7_EVNTSEL0,
1253         .perfctr                = MSR_K7_PERFCTR0,
1254         .addr_offset            = amd_pmu_addr_offset,
1255         .event_map              = amd_pmu_event_map,
1256         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
1257         .num_counters           = AMD64_NUM_COUNTERS,
1258         .add                    = amd_pmu_add_event,
1259         .del                    = amd_pmu_del_event,
1260         .cntval_bits            = 48,
1261         .cntval_mask            = (1ULL << 48) - 1,
1262         .apic                   = 1,
1263         /* use highest bit to detect overflow */
1264         .max_period             = (1ULL << 47) - 1,
1265         .get_event_constraints  = amd_get_event_constraints,
1266         .put_event_constraints  = amd_put_event_constraints,
1267
1268         .format_attrs           = amd_format_attr,
1269         .events_sysfs_show      = amd_event_sysfs_show,
1270
1271         .cpu_prepare            = amd_pmu_cpu_prepare,
1272         .cpu_starting           = amd_pmu_cpu_starting,
1273         .cpu_dead               = amd_pmu_cpu_dead,
1274
1275         .amd_nb_constraints     = 1,
1276 };
1277
1278 static ssize_t branches_show(struct device *cdev,
1279                               struct device_attribute *attr,
1280                               char *buf)
1281 {
1282         return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu.lbr_nr);
1283 }
1284
1285 static DEVICE_ATTR_RO(branches);
1286
1287 static struct attribute *amd_pmu_branches_attrs[] = {
1288         &dev_attr_branches.attr,
1289         NULL,
1290 };
1291
1292 static umode_t
1293 amd_branches_is_visible(struct kobject *kobj, struct attribute *attr, int i)
1294 {
1295         return x86_pmu.lbr_nr ? attr->mode : 0;
1296 }
1297
1298 static struct attribute_group group_caps_amd_branches = {
1299         .name  = "caps",
1300         .attrs = amd_pmu_branches_attrs,
1301         .is_visible = amd_branches_is_visible,
1302 };
1303
1304 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
1305
1306 EVENT_ATTR_STR(branch-brs, amd_branch_brs,
1307                "event=" __stringify(AMD_FAM19H_BRS_EVENT)"\n");
1308
1309 static struct attribute *amd_brs_events_attrs[] = {
1310         EVENT_PTR(amd_branch_brs),
1311         NULL,
1312 };
1313
1314 static umode_t
1315 amd_brs_is_visible(struct kobject *kobj, struct attribute *attr, int i)
1316 {
1317         return static_cpu_has(X86_FEATURE_BRS) && x86_pmu.lbr_nr ?
1318                attr->mode : 0;
1319 }
1320
1321 static struct attribute_group group_events_amd_brs = {
1322         .name       = "events",
1323         .attrs      = amd_brs_events_attrs,
1324         .is_visible = amd_brs_is_visible,
1325 };
1326
1327 #endif  /* CONFIG_PERF_EVENTS_AMD_BRS */
1328
1329 static const struct attribute_group *amd_attr_update[] = {
1330         &group_caps_amd_branches,
1331 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
1332         &group_events_amd_brs,
1333 #endif
1334         NULL,
1335 };
1336
1337 static int __init amd_core_pmu_init(void)
1338 {
1339         union cpuid_0x80000022_ebx ebx;
1340         u64 even_ctr_mask = 0ULL;
1341         int i;
1342
1343         if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
1344                 return 0;
1345
1346         /* Avoid calculating the value each time in the NMI handler */
1347         perf_nmi_window = msecs_to_jiffies(100);
1348
1349         /*
1350          * If core performance counter extensions exists, we must use
1351          * MSR_F15H_PERF_CTL/MSR_F15H_PERF_CTR msrs. See also
1352          * amd_pmu_addr_offset().
1353          */
1354         x86_pmu.eventsel        = MSR_F15H_PERF_CTL;
1355         x86_pmu.perfctr         = MSR_F15H_PERF_CTR;
1356         x86_pmu.num_counters    = AMD64_NUM_COUNTERS_CORE;
1357
1358         /* Check for Performance Monitoring v2 support */
1359         if (boot_cpu_has(X86_FEATURE_PERFMON_V2)) {
1360                 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
1361
1362                 /* Update PMU version for later usage */
1363                 x86_pmu.version = 2;
1364
1365                 /* Find the number of available Core PMCs */
1366                 x86_pmu.num_counters = ebx.split.num_core_pmc;
1367
1368                 amd_pmu_global_cntr_mask = (1ULL << x86_pmu.num_counters) - 1;
1369
1370                 /* Update PMC handling functions */
1371                 x86_pmu.enable_all = amd_pmu_v2_enable_all;
1372                 x86_pmu.disable_all = amd_pmu_v2_disable_all;
1373                 x86_pmu.enable = amd_pmu_v2_enable_event;
1374                 x86_pmu.handle_irq = amd_pmu_v2_handle_irq;
1375                 static_call_update(amd_pmu_test_overflow, amd_pmu_test_overflow_status);
1376         }
1377
1378         /*
1379          * AMD Core perfctr has separate MSRs for the NB events, see
1380          * the amd/uncore.c driver.
1381          */
1382         x86_pmu.amd_nb_constraints = 0;
1383
1384         if (boot_cpu_data.x86 == 0x15) {
1385                 pr_cont("Fam15h ");
1386                 x86_pmu.get_event_constraints = amd_get_event_constraints_f15h;
1387         }
1388         if (boot_cpu_data.x86 >= 0x17) {
1389                 pr_cont("Fam17h+ ");
1390                 /*
1391                  * Family 17h and compatibles have constraints for Large
1392                  * Increment per Cycle events: they may only be assigned an
1393                  * even numbered counter that has a consecutive adjacent odd
1394                  * numbered counter following it.
1395                  */
1396                 for (i = 0; i < x86_pmu.num_counters - 1; i += 2)
1397                         even_ctr_mask |= BIT_ULL(i);
1398
1399                 pair_constraint = (struct event_constraint)
1400                                     __EVENT_CONSTRAINT(0, even_ctr_mask, 0,
1401                                     x86_pmu.num_counters / 2, 0,
1402                                     PERF_X86_EVENT_PAIR);
1403
1404                 x86_pmu.get_event_constraints = amd_get_event_constraints_f17h;
1405                 x86_pmu.put_event_constraints = amd_put_event_constraints_f17h;
1406                 x86_pmu.perf_ctr_pair_en = AMD_MERGE_EVENT_ENABLE;
1407                 x86_pmu.flags |= PMU_FL_PAIR;
1408         }
1409
1410         /* LBR and BRS are mutually exclusive features */
1411         if (!amd_pmu_lbr_init()) {
1412                 /* LBR requires flushing on context switch */
1413                 x86_pmu.sched_task = amd_pmu_lbr_sched_task;
1414                 static_call_update(amd_pmu_branch_hw_config, amd_pmu_lbr_hw_config);
1415                 static_call_update(amd_pmu_branch_reset, amd_pmu_lbr_reset);
1416                 static_call_update(amd_pmu_branch_add, amd_pmu_lbr_add);
1417                 static_call_update(amd_pmu_branch_del, amd_pmu_lbr_del);
1418         } else if (!amd_brs_init()) {
1419                 /*
1420                  * BRS requires special event constraints and flushing on ctxsw.
1421                  */
1422                 x86_pmu.get_event_constraints = amd_get_event_constraints_f19h;
1423                 x86_pmu.sched_task = amd_pmu_brs_sched_task;
1424                 x86_pmu.limit_period = amd_pmu_limit_period;
1425
1426                 static_call_update(amd_pmu_branch_hw_config, amd_brs_hw_config);
1427                 static_call_update(amd_pmu_branch_reset, amd_brs_reset);
1428                 static_call_update(amd_pmu_branch_add, amd_pmu_brs_add);
1429                 static_call_update(amd_pmu_branch_del, amd_pmu_brs_del);
1430
1431                 /*
1432                  * put_event_constraints callback same as Fam17h, set above
1433                  */
1434
1435                 /* branch sampling must be stopped when entering low power */
1436                 amd_brs_lopwr_init();
1437         }
1438
1439         x86_pmu.attr_update = amd_attr_update;
1440
1441         pr_cont("core perfctr, ");
1442         return 0;
1443 }
1444
1445 __init int amd_pmu_init(void)
1446 {
1447         int ret;
1448
1449         /* Performance-monitoring supported from K7 and later: */
1450         if (boot_cpu_data.x86 < 6)
1451                 return -ENODEV;
1452
1453         x86_pmu = amd_pmu;
1454
1455         ret = amd_core_pmu_init();
1456         if (ret)
1457                 return ret;
1458
1459         if (num_possible_cpus() == 1) {
1460                 /*
1461                  * No point in allocating data structures to serialize
1462                  * against other CPUs, when there is only the one CPU.
1463                  */
1464                 x86_pmu.amd_nb_constraints = 0;
1465         }
1466
1467         if (boot_cpu_data.x86 >= 0x17)
1468                 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids_f17h, sizeof(hw_cache_event_ids));
1469         else
1470                 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids, sizeof(hw_cache_event_ids));
1471
1472         return 0;
1473 }
1474
1475 static inline void amd_pmu_reload_virt(void)
1476 {
1477         if (x86_pmu.version >= 2) {
1478                 /*
1479                  * Clear global enable bits, reprogram the PERF_CTL
1480                  * registers with updated perf_ctr_virt_mask and then
1481                  * set global enable bits once again
1482                  */
1483                 amd_pmu_v2_disable_all();
1484                 amd_pmu_enable_all(0);
1485                 amd_pmu_v2_enable_all(0);
1486                 return;
1487         }
1488
1489         amd_pmu_disable_all();
1490         amd_pmu_enable_all(0);
1491 }
1492
1493 void amd_pmu_enable_virt(void)
1494 {
1495         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1496
1497         cpuc->perf_ctr_virt_mask = 0;
1498
1499         /* Reload all events */
1500         amd_pmu_reload_virt();
1501 }
1502 EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
1503
1504 void amd_pmu_disable_virt(void)
1505 {
1506         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1507
1508         /*
1509          * We only mask out the Host-only bit so that host-only counting works
1510          * when SVM is disabled. If someone sets up a guest-only counter when
1511          * SVM is disabled the Guest-only bits still gets set and the counter
1512          * will not count anything.
1513          */
1514         cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
1515
1516         /* Reload all events */
1517         amd_pmu_reload_virt();
1518 }
1519 EXPORT_SYMBOL_GPL(amd_pmu_disable_virt);