drivers/gpu/drm/i915/i915_pmu.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  */
  24
  25 #include <linux/perf_event.h>
  26 #include <linux/pm_runtime.h>
  27
  28 #include "i915_drv.h"
  29 #include "i915_pmu.h"
  30 #include "intel_ringbuffer.h"
  31
  32 /* Frequency for the sampling timer for events which need it. */
  33 #define FREQUENCY 200
  34 #define PERIOD max_t(u64, 10000, NSEC_PER_SEC / FREQUENCY)
  35
  36 #define ENGINE_SAMPLE_MASK \
  37         (BIT(I915_SAMPLE_BUSY) | \
  38          BIT(I915_SAMPLE_WAIT) | \
  39          BIT(I915_SAMPLE_SEMA))
  40
  41 #define ENGINE_SAMPLE_BITS (1 << I915_PMU_SAMPLE_BITS)
  42
  43 static cpumask_t i915_pmu_cpumask = CPU_MASK_NONE;
  44
  45 static u8 engine_config_sample(u64 config)
  46 {
  47         return config & I915_PMU_SAMPLE_MASK;
  48 }
  49
  50 static u8 engine_event_sample(struct perf_event *event)
  51 {
  52         return engine_config_sample(event->attr.config);
  53 }
  54
  55 static u8 engine_event_class(struct perf_event *event)
  56 {
  57         return (event->attr.config >> I915_PMU_CLASS_SHIFT) & 0xff;
  58 }
  59
  60 static u8 engine_event_instance(struct perf_event *event)
  61 {
  62         return (event->attr.config >> I915_PMU_SAMPLE_BITS) & 0xff;
  63 }
  64
  65 static bool is_engine_config(u64 config)
  66 {
  67         return config < __I915_PMU_OTHER(0);
  68 }
  69
  70 static unsigned int config_enabled_bit(u64 config)
  71 {
  72         if (is_engine_config(config))
  73                 return engine_config_sample(config);
  74         else
  75                 return ENGINE_SAMPLE_BITS + (config - __I915_PMU_OTHER(0));
  76 }
  77
  78 static u64 config_enabled_mask(u64 config)
  79 {
  80         return BIT_ULL(config_enabled_bit(config));
  81 }
  82
  83 static bool is_engine_event(struct perf_event *event)
  84 {
  85         return is_engine_config(event->attr.config);
  86 }
  87
  88 static unsigned int event_enabled_bit(struct perf_event *event)
  89 {
  90         return config_enabled_bit(event->attr.config);
  91 }
  92
  93 static bool supports_busy_stats(struct drm_i915_private *i915)
  94 {
  95         return INTEL_GEN(i915) >= 8;
  96 }
  97
  98 static bool pmu_needs_timer(struct drm_i915_private *i915, bool gpu_active)
  99 {
 100         u64 enable;
 101
 102         /*
 103          * Only some counters need the sampling timer.
 104          *
 105          * We start with a bitmask of all currently enabled events.
 106          */
 107         enable = i915->pmu.enable;
 108
 109         /*
 110          * Mask out all the ones which do not need the timer, or in
 111          * other words keep all the ones that could need the timer.
 112          */
 113         enable &= config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY) |
 114                   config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY) |
 115                   ENGINE_SAMPLE_MASK;
 116
 117         /*
 118          * When the GPU is idle per-engine counters do not need to be
 119          * running so clear those bits out.
 120          */
 121         if (!gpu_active)
 122                 enable &= ~ENGINE_SAMPLE_MASK;
 123         /*
 124          * Also there is software busyness tracking available we do not
 125          * need the timer for I915_SAMPLE_BUSY counter.
 126          */
 127         else if (supports_busy_stats(i915))
 128                 enable &= ~BIT(I915_SAMPLE_BUSY);
 129
 130         /*
 131          * If some bits remain it means we need the sampling timer running.
 132          */
 133         return enable;
 134 }
 135
 136 void i915_pmu_gt_parked(struct drm_i915_private *i915)
 137 {
 138         if (!i915->pmu.base.event_init)
 139                 return;
 140
 141         spin_lock_irq(&i915->pmu.lock);
 142         /*
 143          * Signal sampling timer to stop if only engine events are enabled and
 144          * GPU went idle.
 145          */
 146         i915->pmu.timer_enabled = pmu_needs_timer(i915, false);
 147         spin_unlock_irq(&i915->pmu.lock);
 148 }
 149
 150 static void __i915_pmu_maybe_start_timer(struct drm_i915_private *i915)
 151 {
 152         if (!i915->pmu.timer_enabled && pmu_needs_timer(i915, true)) {
 153                 i915->pmu.timer_enabled = true;
 154                 hrtimer_start_range_ns(&i915->pmu.timer,
 155                                        ns_to_ktime(PERIOD), 0,
 156                                        HRTIMER_MODE_REL_PINNED);
 157         }
 158 }
 159
 160 void i915_pmu_gt_unparked(struct drm_i915_private *i915)
 161 {
 162         if (!i915->pmu.base.event_init)
 163                 return;
 164
 165         spin_lock_irq(&i915->pmu.lock);
 166         /*
 167          * Re-enable sampling timer when GPU goes active.
 168          */
 169         __i915_pmu_maybe_start_timer(i915);
 170         spin_unlock_irq(&i915->pmu.lock);
 171 }
 172
 173 static bool grab_forcewake(struct drm_i915_private *i915, bool fw)
 174 {
 175         if (!fw)
 176                 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
 177
 178         return true;
 179 }
 180
 181 static void
 182 update_sample(struct i915_pmu_sample *sample, u32 unit, u32 val)
 183 {
 184         /*
 185          * Since we are doing stochastic sampling for these counters,
 186          * average the delta with the previous value for better accuracy.
 187          */
 188         sample->cur += div_u64(mul_u32_u32(sample->prev + val, unit), 2);
 189         sample->prev = val;
 190 }
 191
 192 static void engines_sample(struct drm_i915_private *dev_priv)
 193 {
 194         struct intel_engine_cs *engine;
 195         enum intel_engine_id id;
 196         bool fw = false;
 197
 198         if ((dev_priv->pmu.enable & ENGINE_SAMPLE_MASK) == 0)
 199                 return;
 200
 201         if (!dev_priv->gt.awake)
 202                 return;
 203
 204         if (!intel_runtime_pm_get_if_in_use(dev_priv))
 205                 return;
 206
 207         for_each_engine(engine, dev_priv, id) {
 208                 u32 current_seqno = intel_engine_get_seqno(engine);
 209                 u32 last_seqno = intel_engine_last_submit(engine);
 210                 u32 val;
 211
 212                 val = !i915_seqno_passed(current_seqno, last_seqno);
 213
 214                 update_sample(&engine->pmu.sample[I915_SAMPLE_BUSY],
 215                               PERIOD, val);
 216
 217                 if (val && (engine->pmu.enable &
 218                     (BIT(I915_SAMPLE_WAIT) | BIT(I915_SAMPLE_SEMA)))) {
 219                         fw = grab_forcewake(dev_priv, fw);
 220
 221                         val = I915_READ_FW(RING_CTL(engine->mmio_base));
 222                 } else {
 223                         val = 0;
 224                 }
 225
 226                 update_sample(&engine->pmu.sample[I915_SAMPLE_WAIT],
 227                               PERIOD, !!(val & RING_WAIT));
 228
 229                 update_sample(&engine->pmu.sample[I915_SAMPLE_SEMA],
 230                               PERIOD, !!(val & RING_WAIT_SEMAPHORE));
 231         }
 232
 233         if (fw)
 234                 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 235
 236         intel_runtime_pm_put(dev_priv);
 237 }
 238
 239 static void frequency_sample(struct drm_i915_private *dev_priv)
 240 {
 241         if (dev_priv->pmu.enable &
 242             config_enabled_mask(I915_PMU_ACTUAL_FREQUENCY)) {
 243                 u32 val;
 244
 245                 val = dev_priv->gt_pm.rps.cur_freq;
 246                 if (dev_priv->gt.awake &&
 247                     intel_runtime_pm_get_if_in_use(dev_priv)) {
 248                         val = intel_get_cagf(dev_priv,
 249                                              I915_READ_NOTRACE(GEN6_RPSTAT1));
 250                         intel_runtime_pm_put(dev_priv);
 251                 }
 252
 253                 update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_ACT],
 254                               1, intel_gpu_freq(dev_priv, val));
 255         }
 256
 257         if (dev_priv->pmu.enable &
 258             config_enabled_mask(I915_PMU_REQUESTED_FREQUENCY)) {
 259                 update_sample(&dev_priv->pmu.sample[__I915_SAMPLE_FREQ_REQ], 1,
 260                               intel_gpu_freq(dev_priv,
 261                                              dev_priv->gt_pm.rps.cur_freq));
 262         }
 263 }
 264
 265 static void pmu_init_previous_samples(struct drm_i915_private *i915)
 266 {
 267         struct intel_engine_cs *engine;
 268         enum intel_engine_id id;
 269         unsigned int i;
 270
 271         for_each_engine(engine, i915, id) {
 272                 for (i = 0; i < ARRAY_SIZE(engine->pmu.sample); i++)
 273                         engine->pmu.sample[i].prev = 0;
 274         }
 275
 276         for (i = 0; i < ARRAY_SIZE(i915->pmu.sample); i++)
 277                 i915->pmu.sample[i].prev = i915->gt_pm.rps.idle_freq;
 278 }
 279
 280 static enum hrtimer_restart i915_sample(struct hrtimer *hrtimer)
 281 {
 282         struct drm_i915_private *i915 =
 283                 container_of(hrtimer, struct drm_i915_private, pmu.timer);
 284
 285         if (!READ_ONCE(i915->pmu.timer_enabled)) {
 286                 pmu_init_previous_samples(i915);
 287
 288                 return HRTIMER_NORESTART;
 289         }
 290
 291         engines_sample(i915);
 292         frequency_sample(i915);
 293
 294         hrtimer_forward_now(hrtimer, ns_to_ktime(PERIOD));
 295         return HRTIMER_RESTART;
 296 }
 297
 298 static u64 count_interrupts(struct drm_i915_private *i915)
 299 {
 300         /* open-coded kstat_irqs() */
 301         struct irq_desc *desc = irq_to_desc(i915->drm.pdev->irq);
 302         u64 sum = 0;
 303         int cpu;
 304
 305         if (!desc || !desc->kstat_irqs)
 306                 return 0;
 307
 308         for_each_possible_cpu(cpu)
 309                 sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
 310
 311         return sum;
 312 }
 313
 314 static void i915_pmu_event_destroy(struct perf_event *event)
 315 {
 316         WARN_ON(event->parent);
 317 }
 318
 319 static int engine_event_init(struct perf_event *event)
 320 {
 321         struct drm_i915_private *i915 =
 322                 container_of(event->pmu, typeof(*i915), pmu.base);
 323
 324         if (!intel_engine_lookup_user(i915, engine_event_class(event),
 325                                       engine_event_instance(event)))
 326                 return -ENODEV;
 327
 328         switch (engine_event_sample(event)) {
 329         case I915_SAMPLE_BUSY:
 330         case I915_SAMPLE_WAIT:
 331                 break;
 332         case I915_SAMPLE_SEMA:
 333                 if (INTEL_GEN(i915) < 6)
 334                         return -ENODEV;
 335                 break;
 336         default:
 337                 return -ENOENT;
 338         }
 339
 340         return 0;
 341 }
 342
 343 static int i915_pmu_event_init(struct perf_event *event)
 344 {
 345         struct drm_i915_private *i915 =
 346                 container_of(event->pmu, typeof(*i915), pmu.base);
 347         int cpu, ret;
 348
 349         if (event->attr.type != event->pmu->type)
 350                 return -ENOENT;
 351
 352         /* unsupported modes and filters */
 353         if (event->attr.sample_period) /* no sampling */
 354                 return -EINVAL;
 355
 356         if (has_branch_stack(event))
 357                 return -EOPNOTSUPP;
 358
 359         if (event->cpu < 0)
 360                 return -EINVAL;
 361
 362         cpu = cpumask_any_and(&i915_pmu_cpumask,
 363                               topology_sibling_cpumask(event->cpu));
 364         if (cpu >= nr_cpu_ids)
 365                 return -ENODEV;
 366
 367         if (is_engine_event(event)) {
 368                 ret = engine_event_init(event);
 369         } else {
 370                 ret = 0;
 371                 switch (event->attr.config) {
 372                 case I915_PMU_ACTUAL_FREQUENCY:
 373                         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 374                                  /* Requires a mutex for sampling! */
 375                                 ret = -ENODEV;
 376                 case I915_PMU_REQUESTED_FREQUENCY:
 377                         if (INTEL_GEN(i915) < 6)
 378                                 ret = -ENODEV;
 379                         break;
 380                 case I915_PMU_INTERRUPTS:
 381                         break;
 382                 case I915_PMU_RC6_RESIDENCY:
 383                         if (!HAS_RC6(i915))
 384                                 ret = -ENODEV;
 385                         break;
 386                 case I915_PMU_RC6p_RESIDENCY:
 387                 case I915_PMU_RC6pp_RESIDENCY:
 388                         if (!HAS_RC6p(i915))
 389                                 ret = -ENODEV;
 390                         break;
 391                 default:
 392                         ret = -ENOENT;
 393                         break;
 394                 }
 395         }
 396         if (ret)
 397                 return ret;
 398
 399         event->cpu = cpu;
 400         if (!event->parent)
 401                 event->destroy = i915_pmu_event_destroy;
 402
 403         return 0;
 404 }
 405
 406 static u64 __i915_pmu_event_read(struct perf_event *event)
 407 {
 408         struct drm_i915_private *i915 =
 409                 container_of(event->pmu, typeof(*i915), pmu.base);
 410         u64 val = 0;
 411
 412         if (is_engine_event(event)) {
 413                 u8 sample = engine_event_sample(event);
 414                 struct intel_engine_cs *engine;
 415
 416                 engine = intel_engine_lookup_user(i915,
 417                                                   engine_event_class(event),
 418                                                   engine_event_instance(event));
 419
 420                 if (WARN_ON_ONCE(!engine)) {
 421                         /* Do nothing */
 422                 } else if (sample == I915_SAMPLE_BUSY &&
 423                            engine->pmu.busy_stats) {
 424                         val = ktime_to_ns(intel_engine_get_busy_time(engine));
 425                 } else {
 426                         val = engine->pmu.sample[sample].cur;
 427                 }
 428         } else {
 429                 switch (event->attr.config) {
 430                 case I915_PMU_ACTUAL_FREQUENCY:
 431                         val =
 432                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_ACT].cur,
 433                                    FREQUENCY);
 434                         break;
 435                 case I915_PMU_REQUESTED_FREQUENCY:
 436                         val =
 437                            div_u64(i915->pmu.sample[__I915_SAMPLE_FREQ_REQ].cur,
 438                                    FREQUENCY);
 439                         break;
 440                 case I915_PMU_INTERRUPTS:
 441                         val = count_interrupts(i915);
 442                         break;
 443                 case I915_PMU_RC6_RESIDENCY:
 444                         intel_runtime_pm_get(i915);
 445                         val = intel_rc6_residency_ns(i915,
 446                                                      IS_VALLEYVIEW(i915) ?
 447                                                      VLV_GT_RENDER_RC6 :
 448                                                      GEN6_GT_GFX_RC6);
 449                         intel_runtime_pm_put(i915);
 450                         break;
 451                 case I915_PMU_RC6p_RESIDENCY:
 452                         intel_runtime_pm_get(i915);
 453                         val = intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6p);
 454                         intel_runtime_pm_put(i915);
 455                         break;
 456                 case I915_PMU_RC6pp_RESIDENCY:
 457                         intel_runtime_pm_get(i915);
 458                         val = intel_rc6_residency_ns(i915, GEN6_GT_GFX_RC6pp);
 459                         intel_runtime_pm_put(i915);
 460                         break;
 461                 }
 462         }
 463
 464         return val;
 465 }
 466
 467 static void i915_pmu_event_read(struct perf_event *event)
 468 {
 469         struct hw_perf_event *hwc = &event->hw;
 470         u64 prev, new;
 471
 472 again:
 473         prev = local64_read(&hwc->prev_count);
 474         new = __i915_pmu_event_read(event);
 475
 476         if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev)
 477                 goto again;
 478
 479         local64_add(new - prev, &event->count);
 480 }
 481
 482 static bool engine_needs_busy_stats(struct intel_engine_cs *engine)
 483 {
 484         return supports_busy_stats(engine->i915) &&
 485                (engine->pmu.enable & BIT(I915_SAMPLE_BUSY));
 486 }
 487
 488 static void i915_pmu_enable(struct perf_event *event)
 489 {
 490         struct drm_i915_private *i915 =
 491                 container_of(event->pmu, typeof(*i915), pmu.base);
 492         unsigned int bit = event_enabled_bit(event);
 493         unsigned long flags;
 494
 495         spin_lock_irqsave(&i915->pmu.lock, flags);
 496
 497         /*
 498          * Update the bitmask of enabled events and increment
 499          * the event reference counter.
 500          */
 501         GEM_BUG_ON(bit >= I915_PMU_MASK_BITS);
 502         GEM_BUG_ON(i915->pmu.enable_count[bit] == ~0);
 503         i915->pmu.enable |= BIT_ULL(bit);
 504         i915->pmu.enable_count[bit]++;
 505
 506         /*
 507          * Start the sampling timer if needed and not already enabled.
 508          */
 509         __i915_pmu_maybe_start_timer(i915);
 510
 511         /*
 512          * For per-engine events the bitmask and reference counting
 513          * is stored per engine.
 514          */
 515         if (is_engine_event(event)) {
 516                 u8 sample = engine_event_sample(event);
 517                 struct intel_engine_cs *engine;
 518
 519                 engine = intel_engine_lookup_user(i915,
 520                                                   engine_event_class(event),
 521                                                   engine_event_instance(event));
 522                 GEM_BUG_ON(!engine);
 523                 engine->pmu.enable |= BIT(sample);
 524
 525                 GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS);
 526                 GEM_BUG_ON(engine->pmu.enable_count[sample] == ~0);
 527                 if (engine->pmu.enable_count[sample]++ == 0) {
 528                         /*
 529                          * Enable engine busy stats tracking if needed or
 530                          * alternatively cancel the scheduled disable.
 531                          *
 532                          * If the delayed disable was pending, cancel it and
 533                          * in this case do not enable since it already is.
 534                          */
 535                         if (engine_needs_busy_stats(engine) &&
 536                             !engine->pmu.busy_stats) {
 537                                 engine->pmu.busy_stats = true;
 538                                 if (!cancel_delayed_work(&engine->pmu.disable_busy_stats))
 539                                         intel_enable_engine_stats(engine);
 540                         }
 541                 }
 542         }
 543
 544         /*
 545          * Store the current counter value so we can report the correct delta
 546          * for all listeners. Even when the event was already enabled and has
 547          * an existing non-zero value.
 548          */
 549         local64_set(&event->hw.prev_count, __i915_pmu_event_read(event));
 550
 551         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 552 }
 553
 554 static void __disable_busy_stats(struct work_struct *work)
 555 {
 556         struct intel_engine_cs *engine =
 557                container_of(work, typeof(*engine), pmu.disable_busy_stats.work);
 558
 559         intel_disable_engine_stats(engine);
 560 }
 561
 562 static void i915_pmu_disable(struct perf_event *event)
 563 {
 564         struct drm_i915_private *i915 =
 565                 container_of(event->pmu, typeof(*i915), pmu.base);
 566         unsigned int bit = event_enabled_bit(event);
 567         unsigned long flags;
 568
 569         spin_lock_irqsave(&i915->pmu.lock, flags);
 570
 571         if (is_engine_event(event)) {
 572                 u8 sample = engine_event_sample(event);
 573                 struct intel_engine_cs *engine;
 574
 575                 engine = intel_engine_lookup_user(i915,
 576                                                   engine_event_class(event),
 577                                                   engine_event_instance(event));
 578                 GEM_BUG_ON(!engine);
 579                 GEM_BUG_ON(sample >= I915_PMU_SAMPLE_BITS);
 580                 GEM_BUG_ON(engine->pmu.enable_count[sample] == 0);
 581                 /*
 582                  * Decrement the reference count and clear the enabled
 583                  * bitmask when the last listener on an event goes away.
 584                  */
 585                 if (--engine->pmu.enable_count[sample] == 0) {
 586                         engine->pmu.enable &= ~BIT(sample);
 587                         if (!engine_needs_busy_stats(engine) &&
 588                             engine->pmu.busy_stats) {
 589                                 engine->pmu.busy_stats = false;
 590                                 /*
 591                                  * We request a delayed disable to handle the
 592                                  * rapid on/off cycles on events, which can
 593                                  * happen when tools like perf stat start, in a
 594                                  * nicer way.
 595                                  *
 596                                  * In addition, this also helps with busy stats
 597                                  * accuracy with background CPU offline/online
 598                                  * migration events.
 599                                  */
 600                                 queue_delayed_work(system_wq,
 601                                                    &engine->pmu.disable_busy_stats,
 602                                                    round_jiffies_up_relative(HZ));
 603                         }
 604                 }
 605         }
 606
 607         GEM_BUG_ON(bit >= I915_PMU_MASK_BITS);
 608         GEM_BUG_ON(i915->pmu.enable_count[bit] == 0);
 609         /*
 610          * Decrement the reference count and clear the enabled
 611          * bitmask when the last listener on an event goes away.
 612          */
 613         if (--i915->pmu.enable_count[bit] == 0) {
 614                 i915->pmu.enable &= ~BIT_ULL(bit);
 615                 i915->pmu.timer_enabled &= pmu_needs_timer(i915, true);
 616         }
 617
 618         spin_unlock_irqrestore(&i915->pmu.lock, flags);
 619 }
 620
 621 static void i915_pmu_event_start(struct perf_event *event, int flags)
 622 {
 623         i915_pmu_enable(event);
 624         event->hw.state = 0;
 625 }
 626
 627 static void i915_pmu_event_stop(struct perf_event *event, int flags)
 628 {
 629         if (flags & PERF_EF_UPDATE)
 630                 i915_pmu_event_read(event);
 631         i915_pmu_disable(event);
 632         event->hw.state = PERF_HES_STOPPED;
 633 }
 634
 635 static int i915_pmu_event_add(struct perf_event *event, int flags)
 636 {
 637         if (flags & PERF_EF_START)
 638                 i915_pmu_event_start(event, flags);
 639
 640         return 0;
 641 }
 642
 643 static void i915_pmu_event_del(struct perf_event *event, int flags)
 644 {
 645         i915_pmu_event_stop(event, PERF_EF_UPDATE);
 646 }
 647
 648 static int i915_pmu_event_event_idx(struct perf_event *event)
 649 {
 650         return 0;
 651 }
 652
 653 static ssize_t i915_pmu_format_show(struct device *dev,
 654                                     struct device_attribute *attr, char *buf)
 655 {
 656         struct dev_ext_attribute *eattr;
 657
 658         eattr = container_of(attr, struct dev_ext_attribute, attr);
 659         return sprintf(buf, "%s\n", (char *)eattr->var);
 660 }
 661
 662 #define I915_PMU_FORMAT_ATTR(_name, _config) \
 663         (&((struct dev_ext_attribute[]) { \
 664                 { .attr = __ATTR(_name, 0444, i915_pmu_format_show, NULL), \
 665                   .var = (void *)_config, } \
 666         })[0].attr.attr)
 667
 668 static struct attribute *i915_pmu_format_attrs[] = {
 669         I915_PMU_FORMAT_ATTR(i915_eventid, "config:0-20"),
 670         NULL,
 671 };
 672
 673 static const struct attribute_group i915_pmu_format_attr_group = {
 674         .name = "format",
 675         .attrs = i915_pmu_format_attrs,
 676 };
 677
 678 static ssize_t i915_pmu_event_show(struct device *dev,
 679                                    struct device_attribute *attr, char *buf)
 680 {
 681         struct dev_ext_attribute *eattr;
 682
 683         eattr = container_of(attr, struct dev_ext_attribute, attr);
 684         return sprintf(buf, "config=0x%lx\n", (unsigned long)eattr->var);
 685 }
 686
 687 #define I915_EVENT_ATTR(_name, _config) \
 688         (&((struct dev_ext_attribute[]) { \
 689                 { .attr = __ATTR(_name, 0444, i915_pmu_event_show, NULL), \
 690                   .var = (void *)_config, } \
 691         })[0].attr.attr)
 692
 693 #define I915_EVENT_STR(_name, _str) \
 694         (&((struct perf_pmu_events_attr[]) { \
 695                 { .attr      = __ATTR(_name, 0444, perf_event_sysfs_show, NULL), \
 696                   .id        = 0, \
 697                   .event_str = _str, } \
 698         })[0].attr.attr)
 699
 700 #define I915_EVENT(_name, _config, _unit) \
 701         I915_EVENT_ATTR(_name, _config), \
 702         I915_EVENT_STR(_name.unit, _unit)
 703
 704 #define I915_ENGINE_EVENT(_name, _class, _instance, _sample) \
 705         I915_EVENT_ATTR(_name, __I915_PMU_ENGINE(_class, _instance, _sample)), \
 706         I915_EVENT_STR(_name.unit, "ns")
 707
 708 #define I915_ENGINE_EVENTS(_name, _class, _instance) \
 709         I915_ENGINE_EVENT(_name##_instance-busy, _class, _instance, I915_SAMPLE_BUSY), \
 710         I915_ENGINE_EVENT(_name##_instance-sema, _class, _instance, I915_SAMPLE_SEMA), \
 711         I915_ENGINE_EVENT(_name##_instance-wait, _class, _instance, I915_SAMPLE_WAIT)
 712
 713 static struct attribute *i915_pmu_events_attrs[] = {
 714         I915_ENGINE_EVENTS(rcs, I915_ENGINE_CLASS_RENDER, 0),
 715         I915_ENGINE_EVENTS(bcs, I915_ENGINE_CLASS_COPY, 0),
 716         I915_ENGINE_EVENTS(vcs, I915_ENGINE_CLASS_VIDEO, 0),
 717         I915_ENGINE_EVENTS(vcs, I915_ENGINE_CLASS_VIDEO, 1),
 718         I915_ENGINE_EVENTS(vecs, I915_ENGINE_CLASS_VIDEO_ENHANCE, 0),
 719
 720         I915_EVENT(actual-frequency,    I915_PMU_ACTUAL_FREQUENCY,    "MHz"),
 721         I915_EVENT(requested-frequency, I915_PMU_REQUESTED_FREQUENCY, "MHz"),
 722
 723         I915_EVENT_ATTR(interrupts, I915_PMU_INTERRUPTS),
 724
 725         I915_EVENT(rc6-residency,   I915_PMU_RC6_RESIDENCY,   "ns"),
 726         I915_EVENT(rc6p-residency,  I915_PMU_RC6p_RESIDENCY,  "ns"),
 727         I915_EVENT(rc6pp-residency, I915_PMU_RC6pp_RESIDENCY, "ns"),
 728
 729         NULL,
 730 };
 731
 732 static const struct attribute_group i915_pmu_events_attr_group = {
 733         .name = "events",
 734         .attrs = i915_pmu_events_attrs,
 735 };
 736
 737 static ssize_t
 738 i915_pmu_get_attr_cpumask(struct device *dev,
 739                           struct device_attribute *attr,
 740                           char *buf)
 741 {
 742         return cpumap_print_to_pagebuf(true, buf, &i915_pmu_cpumask);
 743 }
 744
 745 static DEVICE_ATTR(cpumask, 0444, i915_pmu_get_attr_cpumask, NULL);
 746
 747 static struct attribute *i915_cpumask_attrs[] = {
 748         &dev_attr_cpumask.attr,
 749         NULL,
 750 };
 751
 752 static struct attribute_group i915_pmu_cpumask_attr_group = {
 753         .attrs = i915_cpumask_attrs,
 754 };
 755
 756 static const struct attribute_group *i915_pmu_attr_groups[] = {
 757         &i915_pmu_format_attr_group,
 758         &i915_pmu_events_attr_group,
 759         &i915_pmu_cpumask_attr_group,
 760         NULL
 761 };
 762
 763 #ifdef CONFIG_HOTPLUG_CPU
 764 static int i915_pmu_cpu_online(unsigned int cpu, struct hlist_node *node)
 765 {
 766         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 767         unsigned int target;
 768
 769         GEM_BUG_ON(!pmu->base.event_init);
 770
 771         target = cpumask_any_and(&i915_pmu_cpumask, &i915_pmu_cpumask);
 772         /* Select the first online CPU as a designated reader. */
 773         if (target >= nr_cpu_ids)
 774                 cpumask_set_cpu(cpu, &i915_pmu_cpumask);
 775
 776         return 0;
 777 }
 778
 779 static int i915_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node)
 780 {
 781         struct i915_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), node);
 782         unsigned int target;
 783
 784         GEM_BUG_ON(!pmu->base.event_init);
 785
 786         if (cpumask_test_and_clear_cpu(cpu, &i915_pmu_cpumask)) {
 787                 target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu);
 788                 /* Migrate events if there is a valid target */
 789                 if (target < nr_cpu_ids) {
 790                         cpumask_set_cpu(target, &i915_pmu_cpumask);
 791                         perf_pmu_migrate_context(&pmu->base, cpu, target);
 792                 }
 793         }
 794
 795         return 0;
 796 }
 797
 798 static enum cpuhp_state cpuhp_slot = CPUHP_INVALID;
 799 #endif
 800
 801 static int i915_pmu_register_cpuhp_state(struct drm_i915_private *i915)
 802 {
 803 #ifdef CONFIG_HOTPLUG_CPU
 804         enum cpuhp_state slot;
 805         int ret;
 806
 807         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 808                                       "perf/x86/intel/i915:online",
 809                                       i915_pmu_cpu_online,
 810                                       i915_pmu_cpu_offline);
 811         if (ret < 0)
 812                 return ret;
 813
 814         slot = ret;
 815         ret = cpuhp_state_add_instance(slot, &i915->pmu.node);
 816         if (ret) {
 817                 cpuhp_remove_multi_state(slot);
 818                 return ret;
 819         }
 820
 821         cpuhp_slot = slot;
 822 #endif
 823         return 0;
 824 }
 825
 826 static void i915_pmu_unregister_cpuhp_state(struct drm_i915_private *i915)
 827 {
 828 #ifdef CONFIG_HOTPLUG_CPU
 829         WARN_ON(cpuhp_slot == CPUHP_INVALID);
 830         WARN_ON(cpuhp_state_remove_instance(cpuhp_slot, &i915->pmu.node));
 831         cpuhp_remove_multi_state(cpuhp_slot);
 832 #endif
 833 }
 834
 835 void i915_pmu_register(struct drm_i915_private *i915)
 836 {
 837         struct intel_engine_cs *engine;
 838         enum intel_engine_id id;
 839         int ret;
 840
 841         if (INTEL_GEN(i915) <= 2) {
 842                 DRM_INFO("PMU not supported for this GPU.");
 843                 return;
 844         }
 845
 846         i915->pmu.base.attr_groups      = i915_pmu_attr_groups;
 847         i915->pmu.base.task_ctx_nr      = perf_invalid_context;
 848         i915->pmu.base.event_init       = i915_pmu_event_init;
 849         i915->pmu.base.add              = i915_pmu_event_add;
 850         i915->pmu.base.del              = i915_pmu_event_del;
 851         i915->pmu.base.start            = i915_pmu_event_start;
 852         i915->pmu.base.stop             = i915_pmu_event_stop;
 853         i915->pmu.base.read             = i915_pmu_event_read;
 854         i915->pmu.base.event_idx        = i915_pmu_event_event_idx;
 855
 856         spin_lock_init(&i915->pmu.lock);
 857         hrtimer_init(&i915->pmu.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 858         i915->pmu.timer.function = i915_sample;
 859
 860         pmu_init_previous_samples(i915);
 861
 862         for_each_engine(engine, i915, id)
 863                 INIT_DELAYED_WORK(&engine->pmu.disable_busy_stats,
 864                                   __disable_busy_stats);
 865
 866         ret = perf_pmu_register(&i915->pmu.base, "i915", -1);
 867         if (ret)
 868                 goto err;
 869
 870         ret = i915_pmu_register_cpuhp_state(i915);
 871         if (ret)
 872                 goto err_unreg;
 873
 874         return;
 875
 876 err_unreg:
 877         perf_pmu_unregister(&i915->pmu.base);
 878 err:
 879         i915->pmu.base.event_init = NULL;
 880         DRM_NOTE("Failed to register PMU! (err=%d)\n", ret);
 881 }
 882
 883 void i915_pmu_unregister(struct drm_i915_private *i915)
 884 {
 885         struct intel_engine_cs *engine;
 886         enum intel_engine_id id;
 887
 888         if (!i915->pmu.base.event_init)
 889                 return;
 890
 891         WARN_ON(i915->pmu.enable);
 892
 893         hrtimer_cancel(&i915->pmu.timer);
 894
 895         for_each_engine(engine, i915, id) {
 896                 GEM_BUG_ON(engine->pmu.busy_stats);
 897                 flush_delayed_work(&engine->pmu.disable_busy_stats);
 898         }
 899
 900         i915_pmu_unregister_cpuhp_state(i915);
 901
 902         perf_pmu_unregister(&i915->pmu.base);
 903         i915->pmu.base.event_init = NULL;
 904 }