drivers/gpu/drm/i915/gt/intel_rps.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7
   8 #include <drm/i915_drm.h>
   9
  10 #include "display/intel_display.h"
  11 #include "i915_drv.h"
  12 #include "i915_irq.h"
  13 #include "intel_breadcrumbs.h"
  14 #include "intel_gt.h"
  15 #include "intel_gt_clock_utils.h"
  16 #include "intel_gt_irq.h"
  17 #include "intel_gt_pm_irq.h"
  18 #include "intel_gt_regs.h"
  19 #include "intel_mchbar_regs.h"
  20 #include "intel_pcode.h"
  21 #include "intel_rps.h"
  22 #include "vlv_sideband.h"
  23 #include "../../../platform/x86/intel_ips.h"
  24
  25 #define BUSY_MAX_EI     20u /* ms */
  26
  27 /*
  28  * Lock protecting IPS related data structures
  29  */
  30 static DEFINE_SPINLOCK(mchdev_lock);
  31
  32 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
  33 {
  34         return container_of(rps, struct intel_gt, rps);
  35 }
  36
  37 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
  38 {
  39         return rps_to_gt(rps)->i915;
  40 }
  41
  42 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
  43 {
  44         return rps_to_gt(rps)->uncore;
  45 }
  46
  47 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
  48 {
  49         struct intel_gt *gt = rps_to_gt(rps);
  50
  51         return &gt->uc.guc.slpc;
  52 }
  53
  54 static bool rps_uses_slpc(struct intel_rps *rps)
  55 {
  56         struct intel_gt *gt = rps_to_gt(rps);
  57
  58         return intel_uc_uses_guc_slpc(&gt->uc);
  59 }
  60
  61 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
  62 {
  63         return mask & ~rps->pm_intrmsk_mbz;
  64 }
  65
  66 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
  67 {
  68         intel_uncore_write_fw(uncore, reg, val);
  69 }
  70
  71 static void rps_timer(struct timer_list *t)
  72 {
  73         struct intel_rps *rps = from_timer(rps, t, timer);
  74         struct intel_engine_cs *engine;
  75         ktime_t dt, last, timestamp;
  76         enum intel_engine_id id;
  77         s64 max_busy[3] = {};
  78
  79         timestamp = 0;
  80         for_each_engine(engine, rps_to_gt(rps), id) {
  81                 s64 busy;
  82                 int i;
  83
  84                 dt = intel_engine_get_busy_time(engine, &timestamp);
  85                 last = engine->stats.rps;
  86                 engine->stats.rps = dt;
  87
  88                 busy = ktime_to_ns(ktime_sub(dt, last));
  89                 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
  90                         if (busy > max_busy[i])
  91                                 swap(busy, max_busy[i]);
  92                 }
  93         }
  94         last = rps->pm_timestamp;
  95         rps->pm_timestamp = timestamp;
  96
  97         if (intel_rps_is_active(rps)) {
  98                 s64 busy;
  99                 int i;
 100
 101                 dt = ktime_sub(timestamp, last);
 102
 103                 /*
 104                  * Our goal is to evaluate each engine independently, so we run
 105                  * at the lowest clocks required to sustain the heaviest
 106                  * workload. However, a task may be split into sequential
 107                  * dependent operations across a set of engines, such that
 108                  * the independent contributions do not account for high load,
 109                  * but overall the task is GPU bound. For example, consider
 110                  * video decode on vcs followed by colour post-processing
 111                  * on vecs, followed by general post-processing on rcs.
 112                  * Since multi-engines being active does imply a single
 113                  * continuous workload across all engines, we hedge our
 114                  * bets by only contributing a factor of the distributed
 115                  * load into our busyness calculation.
 116                  */
 117                 busy = max_busy[0];
 118                 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
 119                         if (!max_busy[i])
 120                                 break;
 121
 122                         busy += div_u64(max_busy[i], 1 << i);
 123                 }
 124                 GT_TRACE(rps_to_gt(rps),
 125                          "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
 126                          busy, (int)div64_u64(100 * busy, dt),
 127                          max_busy[0], max_busy[1], max_busy[2],
 128                          rps->pm_interval);
 129
 130                 if (100 * busy > rps->power.up_threshold * dt &&
 131                     rps->cur_freq < rps->max_freq_softlimit) {
 132                         rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
 133                         rps->pm_interval = 1;
 134                         schedule_work(&rps->work);
 135                 } else if (100 * busy < rps->power.down_threshold * dt &&
 136                            rps->cur_freq > rps->min_freq_softlimit) {
 137                         rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
 138                         rps->pm_interval = 1;
 139                         schedule_work(&rps->work);
 140                 } else {
 141                         rps->last_adj = 0;
 142                 }
 143
 144                 mod_timer(&rps->timer,
 145                           jiffies + msecs_to_jiffies(rps->pm_interval));
 146                 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
 147         }
 148 }
 149
 150 static void rps_start_timer(struct intel_rps *rps)
 151 {
 152         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 153         rps->pm_interval = 1;
 154         mod_timer(&rps->timer, jiffies + 1);
 155 }
 156
 157 static void rps_stop_timer(struct intel_rps *rps)
 158 {
 159         del_timer_sync(&rps->timer);
 160         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 161         cancel_work_sync(&rps->work);
 162 }
 163
 164 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
 165 {
 166         u32 mask = 0;
 167
 168         /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
 169         if (val > rps->min_freq_softlimit)
 170                 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
 171                          GEN6_PM_RP_DOWN_THRESHOLD |
 172                          GEN6_PM_RP_DOWN_TIMEOUT);
 173
 174         if (val < rps->max_freq_softlimit)
 175                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
 176
 177         mask &= rps->pm_events;
 178
 179         return rps_pm_sanitize_mask(rps, ~mask);
 180 }
 181
 182 static void rps_reset_ei(struct intel_rps *rps)
 183 {
 184         memset(&rps->ei, 0, sizeof(rps->ei));
 185 }
 186
 187 static void rps_enable_interrupts(struct intel_rps *rps)
 188 {
 189         struct intel_gt *gt = rps_to_gt(rps);
 190
 191         GEM_BUG_ON(rps_uses_slpc(rps));
 192
 193         GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
 194                  rps->pm_events, rps_pm_mask(rps, rps->last_freq));
 195
 196         rps_reset_ei(rps);
 197
 198         spin_lock_irq(gt->irq_lock);
 199         gen6_gt_pm_enable_irq(gt, rps->pm_events);
 200         spin_unlock_irq(gt->irq_lock);
 201
 202         intel_uncore_write(gt->uncore,
 203                            GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
 204 }
 205
 206 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
 207 {
 208         gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
 209 }
 210
 211 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
 212 {
 213         while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
 214                 ;
 215 }
 216
 217 static void rps_reset_interrupts(struct intel_rps *rps)
 218 {
 219         struct intel_gt *gt = rps_to_gt(rps);
 220
 221         spin_lock_irq(gt->irq_lock);
 222         if (GRAPHICS_VER(gt->i915) >= 11)
 223                 gen11_rps_reset_interrupts(rps);
 224         else
 225                 gen6_rps_reset_interrupts(rps);
 226
 227         rps->pm_iir = 0;
 228         spin_unlock_irq(gt->irq_lock);
 229 }
 230
 231 static void rps_disable_interrupts(struct intel_rps *rps)
 232 {
 233         struct intel_gt *gt = rps_to_gt(rps);
 234
 235         intel_uncore_write(gt->uncore,
 236                            GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
 237
 238         spin_lock_irq(gt->irq_lock);
 239         gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
 240         spin_unlock_irq(gt->irq_lock);
 241
 242         intel_synchronize_irq(gt->i915);
 243
 244         /*
 245          * Now that we will not be generating any more work, flush any
 246          * outstanding tasks. As we are called on the RPS idle path,
 247          * we will reset the GPU to minimum frequencies, so the current
 248          * state of the worker can be discarded.
 249          */
 250         cancel_work_sync(&rps->work);
 251
 252         rps_reset_interrupts(rps);
 253         GT_TRACE(gt, "interrupts:off\n");
 254 }
 255
 256 static const struct cparams {
 257         u16 i;
 258         u16 t;
 259         u16 m;
 260         u16 c;
 261 } cparams[] = {
 262         { 1, 1333, 301, 28664 },
 263         { 1, 1066, 294, 24460 },
 264         { 1, 800, 294, 25192 },
 265         { 0, 1333, 276, 27605 },
 266         { 0, 1066, 276, 27605 },
 267         { 0, 800, 231, 23784 },
 268 };
 269
 270 static void gen5_rps_init(struct intel_rps *rps)
 271 {
 272         struct drm_i915_private *i915 = rps_to_i915(rps);
 273         struct intel_uncore *uncore = rps_to_uncore(rps);
 274         u8 fmax, fmin, fstart;
 275         u32 rgvmodectl;
 276         int c_m, i;
 277
 278         if (i915->fsb_freq <= 3200)
 279                 c_m = 0;
 280         else if (i915->fsb_freq <= 4800)
 281                 c_m = 1;
 282         else
 283                 c_m = 2;
 284
 285         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
 286                 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
 287                         rps->ips.m = cparams[i].m;
 288                         rps->ips.c = cparams[i].c;
 289                         break;
 290                 }
 291         }
 292
 293         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 294
 295         /* Set up min, max, and cur for interrupt handling */
 296         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
 297         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
 298         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 299                 MEMMODE_FSTART_SHIFT;
 300         drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
 301                 fmax, fmin, fstart);
 302
 303         rps->min_freq = fmax;
 304         rps->efficient_freq = fstart;
 305         rps->max_freq = fmin;
 306 }
 307
 308 static unsigned long
 309 __ips_chipset_val(struct intel_ips *ips)
 310 {
 311         struct intel_uncore *uncore =
 312                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 313         unsigned long now = jiffies_to_msecs(jiffies), dt;
 314         unsigned long result;
 315         u64 total, delta;
 316
 317         lockdep_assert_held(&mchdev_lock);
 318
 319         /*
 320          * Prevent division-by-zero if we are asking too fast.
 321          * Also, we don't get interesting results if we are polling
 322          * faster than once in 10ms, so just return the saved value
 323          * in such cases.
 324          */
 325         dt = now - ips->last_time1;
 326         if (dt <= 10)
 327                 return ips->chipset_power;
 328
 329         /* FIXME: handle per-counter overflow */
 330         total = intel_uncore_read(uncore, DMIEC);
 331         total += intel_uncore_read(uncore, DDREC);
 332         total += intel_uncore_read(uncore, CSIEC);
 333
 334         delta = total - ips->last_count1;
 335
 336         result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
 337
 338         ips->last_count1 = total;
 339         ips->last_time1 = now;
 340
 341         ips->chipset_power = result;
 342
 343         return result;
 344 }
 345
 346 static unsigned long ips_mch_val(struct intel_uncore *uncore)
 347 {
 348         unsigned int m, x, b;
 349         u32 tsfs;
 350
 351         tsfs = intel_uncore_read(uncore, TSFS);
 352         x = intel_uncore_read8(uncore, TR1);
 353
 354         b = tsfs & TSFS_INTR_MASK;
 355         m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
 356
 357         return m * x / 127 - b;
 358 }
 359
 360 static int _pxvid_to_vd(u8 pxvid)
 361 {
 362         if (pxvid == 0)
 363                 return 0;
 364
 365         if (pxvid >= 8 && pxvid < 31)
 366                 pxvid = 31;
 367
 368         return (pxvid + 2) * 125;
 369 }
 370
 371 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
 372 {
 373         const int vd = _pxvid_to_vd(pxvid);
 374
 375         if (INTEL_INFO(i915)->is_mobile)
 376                 return max(vd - 1125, 0);
 377
 378         return vd;
 379 }
 380
 381 static void __gen5_ips_update(struct intel_ips *ips)
 382 {
 383         struct intel_uncore *uncore =
 384                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 385         u64 now, delta, dt;
 386         u32 count;
 387
 388         lockdep_assert_held(&mchdev_lock);
 389
 390         now = ktime_get_raw_ns();
 391         dt = now - ips->last_time2;
 392         do_div(dt, NSEC_PER_MSEC);
 393
 394         /* Don't divide by 0 */
 395         if (dt <= 10)
 396                 return;
 397
 398         count = intel_uncore_read(uncore, GFXEC);
 399         delta = count - ips->last_count2;
 400
 401         ips->last_count2 = count;
 402         ips->last_time2 = now;
 403
 404         /* More magic constants... */
 405         ips->gfx_power = div_u64(delta * 1181, dt * 10);
 406 }
 407
 408 static void gen5_rps_update(struct intel_rps *rps)
 409 {
 410         spin_lock_irq(&mchdev_lock);
 411         __gen5_ips_update(&rps->ips);
 412         spin_unlock_irq(&mchdev_lock);
 413 }
 414
 415 static unsigned int gen5_invert_freq(struct intel_rps *rps,
 416                                      unsigned int val)
 417 {
 418         /* Invert the frequency bin into an ips delay */
 419         val = rps->max_freq - val;
 420         val = rps->min_freq + val;
 421
 422         return val;
 423 }
 424
 425 static int __gen5_rps_set(struct intel_rps *rps, u8 val)
 426 {
 427         struct intel_uncore *uncore = rps_to_uncore(rps);
 428         u16 rgvswctl;
 429
 430         lockdep_assert_held(&mchdev_lock);
 431
 432         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 433         if (rgvswctl & MEMCTL_CMD_STS) {
 434                 drm_dbg(&rps_to_i915(rps)->drm,
 435                         "gpu busy, RCS change rejected\n");
 436                 return -EBUSY; /* still busy with another command */
 437         }
 438
 439         /* Invert the frequency bin into an ips delay */
 440         val = gen5_invert_freq(rps, val);
 441
 442         rgvswctl =
 443                 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
 444                 (val << MEMCTL_FREQ_SHIFT) |
 445                 MEMCTL_SFCAVM;
 446         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 447         intel_uncore_posting_read16(uncore, MEMSWCTL);
 448
 449         rgvswctl |= MEMCTL_CMD_STS;
 450         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 451
 452         return 0;
 453 }
 454
 455 static int gen5_rps_set(struct intel_rps *rps, u8 val)
 456 {
 457         int err;
 458
 459         spin_lock_irq(&mchdev_lock);
 460         err = __gen5_rps_set(rps, val);
 461         spin_unlock_irq(&mchdev_lock);
 462
 463         return err;
 464 }
 465
 466 static unsigned long intel_pxfreq(u32 vidfreq)
 467 {
 468         int div = (vidfreq & 0x3f0000) >> 16;
 469         int post = (vidfreq & 0x3000) >> 12;
 470         int pre = (vidfreq & 0x7);
 471
 472         if (!pre)
 473                 return 0;
 474
 475         return div * 133333 / (pre << post);
 476 }
 477
 478 static unsigned int init_emon(struct intel_uncore *uncore)
 479 {
 480         u8 pxw[16];
 481         int i;
 482
 483         /* Disable to program */
 484         intel_uncore_write(uncore, ECR, 0);
 485         intel_uncore_posting_read(uncore, ECR);
 486
 487         /* Program energy weights for various events */
 488         intel_uncore_write(uncore, SDEW, 0x15040d00);
 489         intel_uncore_write(uncore, CSIEW0, 0x007f0000);
 490         intel_uncore_write(uncore, CSIEW1, 0x1e220004);
 491         intel_uncore_write(uncore, CSIEW2, 0x04000004);
 492
 493         for (i = 0; i < 5; i++)
 494                 intel_uncore_write(uncore, PEW(i), 0);
 495         for (i = 0; i < 3; i++)
 496                 intel_uncore_write(uncore, DEW(i), 0);
 497
 498         /* Program P-state weights to account for frequency power adjustment */
 499         for (i = 0; i < 16; i++) {
 500                 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
 501                 unsigned int freq = intel_pxfreq(pxvidfreq);
 502                 unsigned int vid =
 503                         (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 504                 unsigned int val;
 505
 506                 val = vid * vid * freq / 1000 * 255;
 507                 val /= 127 * 127 * 900;
 508
 509                 pxw[i] = val;
 510         }
 511         /* Render standby states get 0 weight */
 512         pxw[14] = 0;
 513         pxw[15] = 0;
 514
 515         for (i = 0; i < 4; i++) {
 516                 intel_uncore_write(uncore, PXW(i),
 517                                    pxw[i * 4 + 0] << 24 |
 518                                    pxw[i * 4 + 1] << 16 |
 519                                    pxw[i * 4 + 2] <<  8 |
 520                                    pxw[i * 4 + 3] <<  0);
 521         }
 522
 523         /* Adjust magic regs to magic values (more experimental results) */
 524         intel_uncore_write(uncore, OGW0, 0);
 525         intel_uncore_write(uncore, OGW1, 0);
 526         intel_uncore_write(uncore, EG0, 0x00007f00);
 527         intel_uncore_write(uncore, EG1, 0x0000000e);
 528         intel_uncore_write(uncore, EG2, 0x000e0000);
 529         intel_uncore_write(uncore, EG3, 0x68000300);
 530         intel_uncore_write(uncore, EG4, 0x42000000);
 531         intel_uncore_write(uncore, EG5, 0x00140031);
 532         intel_uncore_write(uncore, EG6, 0);
 533         intel_uncore_write(uncore, EG7, 0);
 534
 535         for (i = 0; i < 8; i++)
 536                 intel_uncore_write(uncore, PXWL(i), 0);
 537
 538         /* Enable PMON + select events */
 539         intel_uncore_write(uncore, ECR, 0x80000019);
 540
 541         return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
 542 }
 543
 544 static bool gen5_rps_enable(struct intel_rps *rps)
 545 {
 546         struct drm_i915_private *i915 = rps_to_i915(rps);
 547         struct intel_uncore *uncore = rps_to_uncore(rps);
 548         u8 fstart, vstart;
 549         u32 rgvmodectl;
 550
 551         spin_lock_irq(&mchdev_lock);
 552
 553         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 554
 555         /* Enable temp reporting */
 556         intel_uncore_write16(uncore, PMMISC,
 557                              intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
 558         intel_uncore_write16(uncore, TSC1,
 559                              intel_uncore_read16(uncore, TSC1) | TSE);
 560
 561         /* 100ms RC evaluation intervals */
 562         intel_uncore_write(uncore, RCUPEI, 100000);
 563         intel_uncore_write(uncore, RCDNEI, 100000);
 564
 565         /* Set max/min thresholds to 90ms and 80ms respectively */
 566         intel_uncore_write(uncore, RCBMAXAVG, 90000);
 567         intel_uncore_write(uncore, RCBMINAVG, 80000);
 568
 569         intel_uncore_write(uncore, MEMIHYST, 1);
 570
 571         /* Set up min, max, and cur for interrupt handling */
 572         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 573                 MEMMODE_FSTART_SHIFT;
 574
 575         vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
 576                   PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 577
 578         intel_uncore_write(uncore,
 579                            MEMINTREN,
 580                            MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
 581
 582         intel_uncore_write(uncore, VIDSTART, vstart);
 583         intel_uncore_posting_read(uncore, VIDSTART);
 584
 585         rgvmodectl |= MEMMODE_SWMODE_EN;
 586         intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
 587
 588         if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
 589                              MEMCTL_CMD_STS) == 0, 10))
 590                 drm_err(&uncore->i915->drm,
 591                         "stuck trying to change perf mode\n");
 592         mdelay(1);
 593
 594         __gen5_rps_set(rps, rps->cur_freq);
 595
 596         rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
 597         rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
 598         rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
 599         rps->ips.last_time1 = jiffies_to_msecs(jiffies);
 600
 601         rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
 602         rps->ips.last_time2 = ktime_get_raw_ns();
 603
 604         spin_lock(&i915->irq_lock);
 605         ilk_enable_display_irq(i915, DE_PCU_EVENT);
 606         spin_unlock(&i915->irq_lock);
 607
 608         spin_unlock_irq(&mchdev_lock);
 609
 610         rps->ips.corr = init_emon(uncore);
 611
 612         return true;
 613 }
 614
 615 static void gen5_rps_disable(struct intel_rps *rps)
 616 {
 617         struct drm_i915_private *i915 = rps_to_i915(rps);
 618         struct intel_uncore *uncore = rps_to_uncore(rps);
 619         u16 rgvswctl;
 620
 621         spin_lock_irq(&mchdev_lock);
 622
 623         spin_lock(&i915->irq_lock);
 624         ilk_disable_display_irq(i915, DE_PCU_EVENT);
 625         spin_unlock(&i915->irq_lock);
 626
 627         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 628
 629         /* Ack interrupts, disable EFC interrupt */
 630         intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0);
 631         intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
 632
 633         /* Go back to the starting frequency */
 634         __gen5_rps_set(rps, rps->idle_freq);
 635         mdelay(1);
 636         rgvswctl |= MEMCTL_CMD_STS;
 637         intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
 638         mdelay(1);
 639
 640         spin_unlock_irq(&mchdev_lock);
 641 }
 642
 643 static u32 rps_limits(struct intel_rps *rps, u8 val)
 644 {
 645         u32 limits;
 646
 647         /*
 648          * Only set the down limit when we've reached the lowest level to avoid
 649          * getting more interrupts, otherwise leave this clear. This prevents a
 650          * race in the hw when coming out of rc6: There's a tiny window where
 651          * the hw runs at the minimal clock before selecting the desired
 652          * frequency, if the down threshold expires in that window we will not
 653          * receive a down interrupt.
 654          */
 655         if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
 656                 limits = rps->max_freq_softlimit << 23;
 657                 if (val <= rps->min_freq_softlimit)
 658                         limits |= rps->min_freq_softlimit << 14;
 659         } else {
 660                 limits = rps->max_freq_softlimit << 24;
 661                 if (val <= rps->min_freq_softlimit)
 662                         limits |= rps->min_freq_softlimit << 16;
 663         }
 664
 665         return limits;
 666 }
 667
 668 static void rps_set_power(struct intel_rps *rps, int new_power)
 669 {
 670         struct intel_gt *gt = rps_to_gt(rps);
 671         struct intel_uncore *uncore = gt->uncore;
 672         u32 threshold_up = 0, threshold_down = 0; /* in % */
 673         u32 ei_up = 0, ei_down = 0;
 674
 675         lockdep_assert_held(&rps->power.mutex);
 676
 677         if (new_power == rps->power.mode)
 678                 return;
 679
 680         threshold_up = 95;
 681         threshold_down = 85;
 682
 683         /* Note the units here are not exactly 1us, but 1280ns. */
 684         switch (new_power) {
 685         case LOW_POWER:
 686                 ei_up = 16000;
 687                 ei_down = 32000;
 688                 break;
 689
 690         case BETWEEN:
 691                 ei_up = 13000;
 692                 ei_down = 32000;
 693                 break;
 694
 695         case HIGH_POWER:
 696                 ei_up = 10000;
 697                 ei_down = 32000;
 698                 break;
 699         }
 700
 701         /* When byt can survive without system hang with dynamic
 702          * sw freq adjustments, this restriction can be lifted.
 703          */
 704         if (IS_VALLEYVIEW(gt->i915))
 705                 goto skip_hw_write;
 706
 707         GT_TRACE(gt,
 708                  "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
 709                  new_power, threshold_up, ei_up, threshold_down, ei_down);
 710
 711         set(uncore, GEN6_RP_UP_EI,
 712             intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
 713         set(uncore, GEN6_RP_UP_THRESHOLD,
 714             intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10));
 715
 716         set(uncore, GEN6_RP_DOWN_EI,
 717             intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
 718         set(uncore, GEN6_RP_DOWN_THRESHOLD,
 719             intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10));
 720
 721         set(uncore, GEN6_RP_CONTROL,
 722             (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
 723             GEN6_RP_MEDIA_HW_NORMAL_MODE |
 724             GEN6_RP_MEDIA_IS_GFX |
 725             GEN6_RP_ENABLE |
 726             GEN6_RP_UP_BUSY_AVG |
 727             GEN6_RP_DOWN_IDLE_AVG);
 728
 729 skip_hw_write:
 730         rps->power.mode = new_power;
 731         rps->power.up_threshold = threshold_up;
 732         rps->power.down_threshold = threshold_down;
 733 }
 734
 735 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
 736 {
 737         int new_power;
 738
 739         new_power = rps->power.mode;
 740         switch (rps->power.mode) {
 741         case LOW_POWER:
 742                 if (val > rps->efficient_freq + 1 &&
 743                     val > rps->cur_freq)
 744                         new_power = BETWEEN;
 745                 break;
 746
 747         case BETWEEN:
 748                 if (val <= rps->efficient_freq &&
 749                     val < rps->cur_freq)
 750                         new_power = LOW_POWER;
 751                 else if (val >= rps->rp0_freq &&
 752                          val > rps->cur_freq)
 753                         new_power = HIGH_POWER;
 754                 break;
 755
 756         case HIGH_POWER:
 757                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
 758                     val < rps->cur_freq)
 759                         new_power = BETWEEN;
 760                 break;
 761         }
 762         /* Max/min bins are special */
 763         if (val <= rps->min_freq_softlimit)
 764                 new_power = LOW_POWER;
 765         if (val >= rps->max_freq_softlimit)
 766                 new_power = HIGH_POWER;
 767
 768         mutex_lock(&rps->power.mutex);
 769         if (rps->power.interactive)
 770                 new_power = HIGH_POWER;
 771         rps_set_power(rps, new_power);
 772         mutex_unlock(&rps->power.mutex);
 773 }
 774
 775 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
 776 {
 777         GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
 778                  str_yes_no(interactive));
 779
 780         mutex_lock(&rps->power.mutex);
 781         if (interactive) {
 782                 if (!rps->power.interactive++ && intel_rps_is_active(rps))
 783                         rps_set_power(rps, HIGH_POWER);
 784         } else {
 785                 GEM_BUG_ON(!rps->power.interactive);
 786                 rps->power.interactive--;
 787         }
 788         mutex_unlock(&rps->power.mutex);
 789 }
 790
 791 static int gen6_rps_set(struct intel_rps *rps, u8 val)
 792 {
 793         struct intel_uncore *uncore = rps_to_uncore(rps);
 794         struct drm_i915_private *i915 = rps_to_i915(rps);
 795         u32 swreq;
 796
 797         GEM_BUG_ON(rps_uses_slpc(rps));
 798
 799         if (GRAPHICS_VER(i915) >= 9)
 800                 swreq = GEN9_FREQUENCY(val);
 801         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
 802                 swreq = HSW_FREQUENCY(val);
 803         else
 804                 swreq = (GEN6_FREQUENCY(val) |
 805                          GEN6_OFFSET(0) |
 806                          GEN6_AGGRESSIVE_TURBO);
 807         set(uncore, GEN6_RPNSWREQ, swreq);
 808
 809         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
 810                  val, intel_gpu_freq(rps, val), swreq);
 811
 812         return 0;
 813 }
 814
 815 static int vlv_rps_set(struct intel_rps *rps, u8 val)
 816 {
 817         struct drm_i915_private *i915 = rps_to_i915(rps);
 818         int err;
 819
 820         vlv_punit_get(i915);
 821         err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
 822         vlv_punit_put(i915);
 823
 824         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
 825                  val, intel_gpu_freq(rps, val));
 826
 827         return err;
 828 }
 829
 830 static int rps_set(struct intel_rps *rps, u8 val, bool update)
 831 {
 832         struct drm_i915_private *i915 = rps_to_i915(rps);
 833         int err;
 834
 835         if (val == rps->last_freq)
 836                 return 0;
 837
 838         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 839                 err = vlv_rps_set(rps, val);
 840         else if (GRAPHICS_VER(i915) >= 6)
 841                 err = gen6_rps_set(rps, val);
 842         else
 843                 err = gen5_rps_set(rps, val);
 844         if (err)
 845                 return err;
 846
 847         if (update && GRAPHICS_VER(i915) >= 6)
 848                 gen6_rps_set_thresholds(rps, val);
 849         rps->last_freq = val;
 850
 851         return 0;
 852 }
 853
 854 void intel_rps_unpark(struct intel_rps *rps)
 855 {
 856         if (!intel_rps_is_enabled(rps))
 857                 return;
 858
 859         GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
 860
 861         /*
 862          * Use the user's desired frequency as a guide, but for better
 863          * performance, jump directly to RPe as our starting frequency.
 864          */
 865         mutex_lock(&rps->lock);
 866
 867         intel_rps_set_active(rps);
 868         intel_rps_set(rps,
 869                       clamp(rps->cur_freq,
 870                             rps->min_freq_softlimit,
 871                             rps->max_freq_softlimit));
 872
 873         mutex_unlock(&rps->lock);
 874
 875         rps->pm_iir = 0;
 876         if (intel_rps_has_interrupts(rps))
 877                 rps_enable_interrupts(rps);
 878         if (intel_rps_uses_timer(rps))
 879                 rps_start_timer(rps);
 880
 881         if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
 882                 gen5_rps_update(rps);
 883 }
 884
 885 void intel_rps_park(struct intel_rps *rps)
 886 {
 887         int adj;
 888
 889         if (!intel_rps_is_enabled(rps))
 890                 return;
 891
 892         if (!intel_rps_clear_active(rps))
 893                 return;
 894
 895         if (intel_rps_uses_timer(rps))
 896                 rps_stop_timer(rps);
 897         if (intel_rps_has_interrupts(rps))
 898                 rps_disable_interrupts(rps);
 899
 900         if (rps->last_freq <= rps->idle_freq)
 901                 return;
 902
 903         /*
 904          * The punit delays the write of the frequency and voltage until it
 905          * determines the GPU is awake. During normal usage we don't want to
 906          * waste power changing the frequency if the GPU is sleeping (rc6).
 907          * However, the GPU and driver is now idle and we do not want to delay
 908          * switching to minimum voltage (reducing power whilst idle) as we do
 909          * not expect to be woken in the near future and so must flush the
 910          * change by waking the device.
 911          *
 912          * We choose to take the media powerwell (either would do to trick the
 913          * punit into committing the voltage change) as that takes a lot less
 914          * power than the render powerwell.
 915          */
 916         intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 917         rps_set(rps, rps->idle_freq, false);
 918         intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 919
 920         /*
 921          * Since we will try and restart from the previously requested
 922          * frequency on unparking, treat this idle point as a downclock
 923          * interrupt and reduce the frequency for resume. If we park/unpark
 924          * more frequently than the rps worker can run, we will not respond
 925          * to any EI and never see a change in frequency.
 926          *
 927          * (Note we accommodate Cherryview's limitation of only using an
 928          * even bin by applying it to all.)
 929          */
 930         adj = rps->last_adj;
 931         if (adj < 0)
 932                 adj *= 2;
 933         else /* CHV needs even encode values */
 934                 adj = -2;
 935         rps->last_adj = adj;
 936         rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
 937         if (rps->cur_freq < rps->efficient_freq) {
 938                 rps->cur_freq = rps->efficient_freq;
 939                 rps->last_adj = 0;
 940         }
 941
 942         GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
 943 }
 944
 945 u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
 946 {
 947         struct intel_guc_slpc *slpc;
 948
 949         if (rps_uses_slpc(rps)) {
 950                 slpc = rps_to_slpc(rps);
 951
 952                 return slpc->boost_freq;
 953         } else {
 954                 return intel_gpu_freq(rps, rps->boost_freq);
 955         }
 956 }
 957
 958 static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
 959 {
 960         bool boost = false;
 961
 962         /* Validate against (static) hardware limits */
 963         val = intel_freq_opcode(rps, val);
 964         if (val < rps->min_freq || val > rps->max_freq)
 965                 return -EINVAL;
 966
 967         mutex_lock(&rps->lock);
 968         if (val != rps->boost_freq) {
 969                 rps->boost_freq = val;
 970                 boost = atomic_read(&rps->num_waiters);
 971         }
 972         mutex_unlock(&rps->lock);
 973         if (boost)
 974                 schedule_work(&rps->work);
 975
 976         return 0;
 977 }
 978
 979 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
 980 {
 981         struct intel_guc_slpc *slpc;
 982
 983         if (rps_uses_slpc(rps)) {
 984                 slpc = rps_to_slpc(rps);
 985
 986                 return intel_guc_slpc_set_boost_freq(slpc, freq);
 987         } else {
 988                 return rps_set_boost_freq(rps, freq);
 989         }
 990 }
 991
 992 void intel_rps_dec_waiters(struct intel_rps *rps)
 993 {
 994         struct intel_guc_slpc *slpc;
 995
 996         if (rps_uses_slpc(rps)) {
 997                 slpc = rps_to_slpc(rps);
 998
 999                 intel_guc_slpc_dec_waiters(slpc);
1000         } else {
1001                 atomic_dec(&rps->num_waiters);
1002         }
1003 }
1004
1005 void intel_rps_boost(struct i915_request *rq)
1006 {
1007         struct intel_guc_slpc *slpc;
1008
1009         if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1010                 return;
1011
1012         /* Serializes with i915_request_retire() */
1013         if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1014                 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1015
1016                 if (rps_uses_slpc(rps)) {
1017                         slpc = rps_to_slpc(rps);
1018
1019                         if (slpc->min_freq_softlimit >= slpc->boost_freq)
1020                                 return;
1021
1022                         /* Return if old value is non zero */
1023                         if (!atomic_fetch_inc(&slpc->num_waiters)) {
1024                                 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1025                                          rq->fence.context, rq->fence.seqno);
1026                                 schedule_work(&slpc->boost_work);
1027                         }
1028
1029                         return;
1030                 }
1031
1032                 if (atomic_fetch_inc(&rps->num_waiters))
1033                         return;
1034
1035                 if (!intel_rps_is_active(rps))
1036                         return;
1037
1038                 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1039                          rq->fence.context, rq->fence.seqno);
1040
1041                 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1042                         schedule_work(&rps->work);
1043
1044                 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1045         }
1046 }
1047
1048 int intel_rps_set(struct intel_rps *rps, u8 val)
1049 {
1050         int err;
1051
1052         lockdep_assert_held(&rps->lock);
1053         GEM_BUG_ON(val > rps->max_freq);
1054         GEM_BUG_ON(val < rps->min_freq);
1055
1056         if (intel_rps_is_active(rps)) {
1057                 err = rps_set(rps, val, true);
1058                 if (err)
1059                         return err;
1060
1061                 /*
1062                  * Make sure we continue to get interrupts
1063                  * until we hit the minimum or maximum frequencies.
1064                  */
1065                 if (intel_rps_has_interrupts(rps)) {
1066                         struct intel_uncore *uncore = rps_to_uncore(rps);
1067
1068                         set(uncore,
1069                             GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
1070
1071                         set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
1072                 }
1073         }
1074
1075         rps->cur_freq = val;
1076         return 0;
1077 }
1078
1079 static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1080 {
1081         struct drm_i915_private *i915 = rps_to_i915(rps);
1082         struct intel_uncore *uncore = rps_to_uncore(rps);
1083
1084         if (IS_PONTEVECCHIO(i915))
1085                 return intel_uncore_read(uncore, PVC_RP_STATE_CAP);
1086         else if (IS_XEHPSDV(i915))
1087                 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP);
1088         else if (IS_GEN9_LP(i915))
1089                 return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1090         else
1091                 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1092 }
1093
1094 static void
1095 mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1096 {
1097         struct intel_uncore *uncore = rps_to_uncore(rps);
1098         u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ?
1099                                 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) :
1100                                 intel_uncore_read(uncore, MTL_RP_STATE_CAP);
1101         u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ?
1102                         intel_uncore_read(uncore, MTL_MPE_FREQUENCY) :
1103                         intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY);
1104
1105         /* MTL values are in units of 16.67 MHz */
1106         caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap);
1107         caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap);
1108         caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe);
1109 }
1110
1111 static void
1112 __gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1113 {
1114         struct drm_i915_private *i915 = rps_to_i915(rps);
1115         u32 rp_state_cap;
1116
1117         rp_state_cap = intel_rps_read_state_cap(rps);
1118
1119         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
1120         if (IS_GEN9_LP(i915)) {
1121                 caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1122                 caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1123                 caps->min_freq = (rp_state_cap >>  0) & 0xff;
1124         } else {
1125                 caps->rp0_freq = (rp_state_cap >>  0) & 0xff;
1126                 if (GRAPHICS_VER(i915) >= 10)
1127                         caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1128                                                        intel_uncore_read(to_gt(i915)->uncore,
1129                                                        GEN10_FREQ_INFO_REC));
1130                 else
1131                         caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1132                 caps->min_freq = (rp_state_cap >> 16) & 0xff;
1133         }
1134
1135         if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1136                 /*
1137                  * In this case rp_state_cap register reports frequencies in
1138                  * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1139                  * units of 16.67 MHz
1140                  */
1141                 caps->rp0_freq *= GEN9_FREQ_SCALER;
1142                 caps->rp1_freq *= GEN9_FREQ_SCALER;
1143                 caps->min_freq *= GEN9_FREQ_SCALER;
1144         }
1145 }
1146
1147 /**
1148  * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1149  * @rps: the intel_rps structure
1150  * @caps: returned freq caps
1151  *
1152  * Returned "caps" frequencies should be converted to MHz using
1153  * intel_gpu_freq()
1154  */
1155 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1156 {
1157         struct drm_i915_private *i915 = rps_to_i915(rps);
1158
1159         if (IS_METEORLAKE(i915))
1160                 return mtl_get_freq_caps(rps, caps);
1161         else
1162                 return __gen6_rps_get_freq_caps(rps, caps);
1163 }
1164
1165 static void gen6_rps_init(struct intel_rps *rps)
1166 {
1167         struct drm_i915_private *i915 = rps_to_i915(rps);
1168         struct intel_rps_freq_caps caps;
1169
1170         gen6_rps_get_freq_caps(rps, &caps);
1171         rps->rp0_freq = caps.rp0_freq;
1172         rps->rp1_freq = caps.rp1_freq;
1173         rps->min_freq = caps.min_freq;
1174
1175         /* hw_max = RP0 until we check for overclocking */
1176         rps->max_freq = rps->rp0_freq;
1177
1178         rps->efficient_freq = rps->rp1_freq;
1179         if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1180             IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1181                 u32 ddcc_status = 0;
1182                 u32 mult = 1;
1183
1184                 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1185                         mult = GEN9_FREQ_SCALER;
1186                 if (snb_pcode_read(rps_to_gt(rps)->uncore,
1187                                    HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1188                                    &ddcc_status, NULL) == 0)
1189                         rps->efficient_freq =
1190                                 clamp_t(u32,
1191                                         ((ddcc_status >> 8) & 0xff) * mult,
1192                                         rps->min_freq,
1193                                         rps->max_freq);
1194         }
1195 }
1196
1197 static bool rps_reset(struct intel_rps *rps)
1198 {
1199         struct drm_i915_private *i915 = rps_to_i915(rps);
1200
1201         /* force a reset */
1202         rps->power.mode = -1;
1203         rps->last_freq = -1;
1204
1205         if (rps_set(rps, rps->min_freq, true)) {
1206                 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1207                 return false;
1208         }
1209
1210         rps->cur_freq = rps->min_freq;
1211         return true;
1212 }
1213
1214 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
1215 static bool gen9_rps_enable(struct intel_rps *rps)
1216 {
1217         struct intel_gt *gt = rps_to_gt(rps);
1218         struct intel_uncore *uncore = gt->uncore;
1219
1220         /* Program defaults and thresholds for RPS */
1221         if (GRAPHICS_VER(gt->i915) == 9)
1222                 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1223                                       GEN9_FREQUENCY(rps->rp1_freq));
1224
1225         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1226
1227         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1228
1229         return rps_reset(rps);
1230 }
1231
1232 static bool gen8_rps_enable(struct intel_rps *rps)
1233 {
1234         struct intel_uncore *uncore = rps_to_uncore(rps);
1235
1236         intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1237                               HSW_FREQUENCY(rps->rp1_freq));
1238
1239         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1240
1241         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1242
1243         return rps_reset(rps);
1244 }
1245
1246 static bool gen6_rps_enable(struct intel_rps *rps)
1247 {
1248         struct intel_uncore *uncore = rps_to_uncore(rps);
1249
1250         /* Power down if completely idle for over 50ms */
1251         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1252         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1253
1254         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1255                           GEN6_PM_RP_DOWN_THRESHOLD |
1256                           GEN6_PM_RP_DOWN_TIMEOUT);
1257
1258         return rps_reset(rps);
1259 }
1260
1261 static int chv_rps_max_freq(struct intel_rps *rps)
1262 {
1263         struct drm_i915_private *i915 = rps_to_i915(rps);
1264         struct intel_gt *gt = rps_to_gt(rps);
1265         u32 val;
1266
1267         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1268
1269         switch (gt->info.sseu.eu_total) {
1270         case 8:
1271                 /* (2 * 4) config */
1272                 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1273                 break;
1274         case 12:
1275                 /* (2 * 6) config */
1276                 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1277                 break;
1278         case 16:
1279                 /* (2 * 8) config */
1280         default:
1281                 /* Setting (2 * 8) Min RP0 for any other combination */
1282                 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1283                 break;
1284         }
1285
1286         return val & FB_GFX_FREQ_FUSE_MASK;
1287 }
1288
1289 static int chv_rps_rpe_freq(struct intel_rps *rps)
1290 {
1291         struct drm_i915_private *i915 = rps_to_i915(rps);
1292         u32 val;
1293
1294         val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1295         val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1296
1297         return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1298 }
1299
1300 static int chv_rps_guar_freq(struct intel_rps *rps)
1301 {
1302         struct drm_i915_private *i915 = rps_to_i915(rps);
1303         u32 val;
1304
1305         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1306
1307         return val & FB_GFX_FREQ_FUSE_MASK;
1308 }
1309
1310 static u32 chv_rps_min_freq(struct intel_rps *rps)
1311 {
1312         struct drm_i915_private *i915 = rps_to_i915(rps);
1313         u32 val;
1314
1315         val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1316         val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1317
1318         return val & FB_GFX_FREQ_FUSE_MASK;
1319 }
1320
1321 static bool chv_rps_enable(struct intel_rps *rps)
1322 {
1323         struct intel_uncore *uncore = rps_to_uncore(rps);
1324         struct drm_i915_private *i915 = rps_to_i915(rps);
1325         u32 val;
1326
1327         /* 1: Program defaults and thresholds for RPS*/
1328         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1329         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1330         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1331         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1332         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1333
1334         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1335
1336         /* 2: Enable RPS */
1337         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1338                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1339                               GEN6_RP_MEDIA_IS_GFX |
1340                               GEN6_RP_ENABLE |
1341                               GEN6_RP_UP_BUSY_AVG |
1342                               GEN6_RP_DOWN_IDLE_AVG);
1343
1344         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1345                           GEN6_PM_RP_DOWN_THRESHOLD |
1346                           GEN6_PM_RP_DOWN_TIMEOUT);
1347
1348         /* Setting Fixed Bias */
1349         vlv_punit_get(i915);
1350
1351         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1352         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1353
1354         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1355
1356         vlv_punit_put(i915);
1357
1358         /* RPS code assumes GPLL is used */
1359         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1360                       "GPLL not enabled\n");
1361
1362         drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1363                 str_yes_no(val & GPLLENABLE));
1364         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1365
1366         return rps_reset(rps);
1367 }
1368
1369 static int vlv_rps_guar_freq(struct intel_rps *rps)
1370 {
1371         struct drm_i915_private *i915 = rps_to_i915(rps);
1372         u32 val, rp1;
1373
1374         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1375
1376         rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1377         rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1378
1379         return rp1;
1380 }
1381
1382 static int vlv_rps_max_freq(struct intel_rps *rps)
1383 {
1384         struct drm_i915_private *i915 = rps_to_i915(rps);
1385         u32 val, rp0;
1386
1387         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1388
1389         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1390         /* Clamp to max */
1391         rp0 = min_t(u32, rp0, 0xea);
1392
1393         return rp0;
1394 }
1395
1396 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1397 {
1398         struct drm_i915_private *i915 = rps_to_i915(rps);
1399         u32 val, rpe;
1400
1401         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1402         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1403         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1404         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1405
1406         return rpe;
1407 }
1408
1409 static int vlv_rps_min_freq(struct intel_rps *rps)
1410 {
1411         struct drm_i915_private *i915 = rps_to_i915(rps);
1412         u32 val;
1413
1414         val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1415         /*
1416          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1417          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1418          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1419          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1420          * to make sure it matches what Punit accepts.
1421          */
1422         return max_t(u32, val, 0xc0);
1423 }
1424
1425 static bool vlv_rps_enable(struct intel_rps *rps)
1426 {
1427         struct intel_uncore *uncore = rps_to_uncore(rps);
1428         struct drm_i915_private *i915 = rps_to_i915(rps);
1429         u32 val;
1430
1431         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1432         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1433         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1434         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1435         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1436
1437         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1438
1439         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1440                               GEN6_RP_MEDIA_TURBO |
1441                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1442                               GEN6_RP_MEDIA_IS_GFX |
1443                               GEN6_RP_ENABLE |
1444                               GEN6_RP_UP_BUSY_AVG |
1445                               GEN6_RP_DOWN_IDLE_CONT);
1446
1447         /* WaGsvRC0ResidencyMethod:vlv */
1448         rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1449
1450         vlv_punit_get(i915);
1451
1452         /* Setting Fixed Bias */
1453         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1454         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1455
1456         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1457
1458         vlv_punit_put(i915);
1459
1460         /* RPS code assumes GPLL is used */
1461         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1462                       "GPLL not enabled\n");
1463
1464         drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1465                 str_yes_no(val & GPLLENABLE));
1466         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1467
1468         return rps_reset(rps);
1469 }
1470
1471 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1472 {
1473         struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1474         struct intel_uncore *uncore = rps_to_uncore(rps);
1475         unsigned int t, state1, state2;
1476         u32 pxvid, ext_v;
1477         u64 corr, corr2;
1478
1479         lockdep_assert_held(&mchdev_lock);
1480
1481         pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1482         pxvid = (pxvid >> 24) & 0x7f;
1483         ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1484
1485         state1 = ext_v;
1486
1487         /* Revel in the empirically derived constants */
1488
1489         /* Correction factor in 1/100000 units */
1490         t = ips_mch_val(uncore);
1491         if (t > 80)
1492                 corr = t * 2349 + 135940;
1493         else if (t >= 50)
1494                 corr = t * 964 + 29317;
1495         else /* < 50 */
1496                 corr = t * 301 + 1004;
1497
1498         corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1499         corr2 = div_u64(corr, 100000) * ips->corr;
1500
1501         state2 = div_u64(corr2 * state1, 10000);
1502         state2 /= 100; /* convert to mW */
1503
1504         __gen5_ips_update(ips);
1505
1506         return ips->gfx_power + state2;
1507 }
1508
1509 static bool has_busy_stats(struct intel_rps *rps)
1510 {
1511         struct intel_engine_cs *engine;
1512         enum intel_engine_id id;
1513
1514         for_each_engine(engine, rps_to_gt(rps), id) {
1515                 if (!intel_engine_supports_stats(engine))
1516                         return false;
1517         }
1518
1519         return true;
1520 }
1521
1522 void intel_rps_enable(struct intel_rps *rps)
1523 {
1524         struct drm_i915_private *i915 = rps_to_i915(rps);
1525         struct intel_uncore *uncore = rps_to_uncore(rps);
1526         bool enabled = false;
1527
1528         if (!HAS_RPS(i915))
1529                 return;
1530
1531         if (rps_uses_slpc(rps))
1532                 return;
1533
1534         intel_gt_check_clock_frequency(rps_to_gt(rps));
1535
1536         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1537         if (rps->max_freq <= rps->min_freq)
1538                 /* leave disabled, no room for dynamic reclocking */;
1539         else if (IS_CHERRYVIEW(i915))
1540                 enabled = chv_rps_enable(rps);
1541         else if (IS_VALLEYVIEW(i915))
1542                 enabled = vlv_rps_enable(rps);
1543         else if (GRAPHICS_VER(i915) >= 9)
1544                 enabled = gen9_rps_enable(rps);
1545         else if (GRAPHICS_VER(i915) >= 8)
1546                 enabled = gen8_rps_enable(rps);
1547         else if (GRAPHICS_VER(i915) >= 6)
1548                 enabled = gen6_rps_enable(rps);
1549         else if (IS_IRONLAKE_M(i915))
1550                 enabled = gen5_rps_enable(rps);
1551         else
1552                 MISSING_CASE(GRAPHICS_VER(i915));
1553         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1554         if (!enabled)
1555                 return;
1556
1557         GT_TRACE(rps_to_gt(rps),
1558                  "min:%x, max:%x, freq:[%d, %d]\n",
1559                  rps->min_freq, rps->max_freq,
1560                  intel_gpu_freq(rps, rps->min_freq),
1561                  intel_gpu_freq(rps, rps->max_freq));
1562
1563         GEM_BUG_ON(rps->max_freq < rps->min_freq);
1564         GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1565
1566         GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1567         GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1568
1569         if (has_busy_stats(rps))
1570                 intel_rps_set_timer(rps);
1571         else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1572                 intel_rps_set_interrupts(rps);
1573         else
1574                 /* Ironlake currently uses intel_ips.ko */ {}
1575
1576         intel_rps_set_enabled(rps);
1577 }
1578
1579 static void gen6_rps_disable(struct intel_rps *rps)
1580 {
1581         set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1582 }
1583
1584 void intel_rps_disable(struct intel_rps *rps)
1585 {
1586         struct drm_i915_private *i915 = rps_to_i915(rps);
1587
1588         if (!intel_rps_is_enabled(rps))
1589                 return;
1590
1591         intel_rps_clear_enabled(rps);
1592         intel_rps_clear_interrupts(rps);
1593         intel_rps_clear_timer(rps);
1594
1595         if (GRAPHICS_VER(i915) >= 6)
1596                 gen6_rps_disable(rps);
1597         else if (IS_IRONLAKE_M(i915))
1598                 gen5_rps_disable(rps);
1599 }
1600
1601 static int byt_gpu_freq(struct intel_rps *rps, int val)
1602 {
1603         /*
1604          * N = val - 0xb7
1605          * Slow = Fast = GPLL ref * N
1606          */
1607         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1608 }
1609
1610 static int byt_freq_opcode(struct intel_rps *rps, int val)
1611 {
1612         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1613 }
1614
1615 static int chv_gpu_freq(struct intel_rps *rps, int val)
1616 {
1617         /*
1618          * N = val / 2
1619          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1620          */
1621         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1622 }
1623
1624 static int chv_freq_opcode(struct intel_rps *rps, int val)
1625 {
1626         /* CHV needs even values */
1627         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1628 }
1629
1630 int intel_gpu_freq(struct intel_rps *rps, int val)
1631 {
1632         struct drm_i915_private *i915 = rps_to_i915(rps);
1633
1634         if (GRAPHICS_VER(i915) >= 9)
1635                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1636                                          GEN9_FREQ_SCALER);
1637         else if (IS_CHERRYVIEW(i915))
1638                 return chv_gpu_freq(rps, val);
1639         else if (IS_VALLEYVIEW(i915))
1640                 return byt_gpu_freq(rps, val);
1641         else if (GRAPHICS_VER(i915) >= 6)
1642                 return val * GT_FREQUENCY_MULTIPLIER;
1643         else
1644                 return val;
1645 }
1646
1647 int intel_freq_opcode(struct intel_rps *rps, int val)
1648 {
1649         struct drm_i915_private *i915 = rps_to_i915(rps);
1650
1651         if (GRAPHICS_VER(i915) >= 9)
1652                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1653                                          GT_FREQUENCY_MULTIPLIER);
1654         else if (IS_CHERRYVIEW(i915))
1655                 return chv_freq_opcode(rps, val);
1656         else if (IS_VALLEYVIEW(i915))
1657                 return byt_freq_opcode(rps, val);
1658         else if (GRAPHICS_VER(i915) >= 6)
1659                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1660         else
1661                 return val;
1662 }
1663
1664 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1665 {
1666         struct drm_i915_private *i915 = rps_to_i915(rps);
1667
1668         rps->gpll_ref_freq =
1669                 vlv_get_cck_clock(i915, "GPLL ref",
1670                                   CCK_GPLL_CLOCK_CONTROL,
1671                                   i915->czclk_freq);
1672
1673         drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1674                 rps->gpll_ref_freq);
1675 }
1676
1677 static void vlv_rps_init(struct intel_rps *rps)
1678 {
1679         struct drm_i915_private *i915 = rps_to_i915(rps);
1680
1681         vlv_iosf_sb_get(i915,
1682                         BIT(VLV_IOSF_SB_PUNIT) |
1683                         BIT(VLV_IOSF_SB_NC) |
1684                         BIT(VLV_IOSF_SB_CCK));
1685
1686         vlv_init_gpll_ref_freq(rps);
1687
1688         rps->max_freq = vlv_rps_max_freq(rps);
1689         rps->rp0_freq = rps->max_freq;
1690         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1691                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1692
1693         rps->efficient_freq = vlv_rps_rpe_freq(rps);
1694         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1695                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1696
1697         rps->rp1_freq = vlv_rps_guar_freq(rps);
1698         drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1699                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1700
1701         rps->min_freq = vlv_rps_min_freq(rps);
1702         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1703                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1704
1705         vlv_iosf_sb_put(i915,
1706                         BIT(VLV_IOSF_SB_PUNIT) |
1707                         BIT(VLV_IOSF_SB_NC) |
1708                         BIT(VLV_IOSF_SB_CCK));
1709 }
1710
1711 static void chv_rps_init(struct intel_rps *rps)
1712 {
1713         struct drm_i915_private *i915 = rps_to_i915(rps);
1714
1715         vlv_iosf_sb_get(i915,
1716                         BIT(VLV_IOSF_SB_PUNIT) |
1717                         BIT(VLV_IOSF_SB_NC) |
1718                         BIT(VLV_IOSF_SB_CCK));
1719
1720         vlv_init_gpll_ref_freq(rps);
1721
1722         rps->max_freq = chv_rps_max_freq(rps);
1723         rps->rp0_freq = rps->max_freq;
1724         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1725                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1726
1727         rps->efficient_freq = chv_rps_rpe_freq(rps);
1728         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1729                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1730
1731         rps->rp1_freq = chv_rps_guar_freq(rps);
1732         drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1733                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1734
1735         rps->min_freq = chv_rps_min_freq(rps);
1736         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1737                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1738
1739         vlv_iosf_sb_put(i915,
1740                         BIT(VLV_IOSF_SB_PUNIT) |
1741                         BIT(VLV_IOSF_SB_NC) |
1742                         BIT(VLV_IOSF_SB_CCK));
1743
1744         drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1745                                    rps->rp1_freq | rps->min_freq) & 1,
1746                       "Odd GPU freq values\n");
1747 }
1748
1749 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1750 {
1751         ei->ktime = ktime_get_raw();
1752         ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1753         ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1754 }
1755
1756 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1757 {
1758         struct intel_uncore *uncore = rps_to_uncore(rps);
1759         const struct intel_rps_ei *prev = &rps->ei;
1760         struct intel_rps_ei now;
1761         u32 events = 0;
1762
1763         if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1764                 return 0;
1765
1766         vlv_c0_read(uncore, &now);
1767
1768         if (prev->ktime) {
1769                 u64 time, c0;
1770                 u32 render, media;
1771
1772                 time = ktime_us_delta(now.ktime, prev->ktime);
1773
1774                 time *= rps_to_i915(rps)->czclk_freq;
1775
1776                 /* Workload can be split between render + media,
1777                  * e.g. SwapBuffers being blitted in X after being rendered in
1778                  * mesa. To account for this we need to combine both engines
1779                  * into our activity counter.
1780                  */
1781                 render = now.render_c0 - prev->render_c0;
1782                 media = now.media_c0 - prev->media_c0;
1783                 c0 = max(render, media);
1784                 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1785
1786                 if (c0 > time * rps->power.up_threshold)
1787                         events = GEN6_PM_RP_UP_THRESHOLD;
1788                 else if (c0 < time * rps->power.down_threshold)
1789                         events = GEN6_PM_RP_DOWN_THRESHOLD;
1790         }
1791
1792         rps->ei = now;
1793         return events;
1794 }
1795
1796 static void rps_work(struct work_struct *work)
1797 {
1798         struct intel_rps *rps = container_of(work, typeof(*rps), work);
1799         struct intel_gt *gt = rps_to_gt(rps);
1800         struct drm_i915_private *i915 = rps_to_i915(rps);
1801         bool client_boost = false;
1802         int new_freq, adj, min, max;
1803         u32 pm_iir = 0;
1804
1805         spin_lock_irq(gt->irq_lock);
1806         pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1807         client_boost = atomic_read(&rps->num_waiters);
1808         spin_unlock_irq(gt->irq_lock);
1809
1810         /* Make sure we didn't queue anything we're not going to process. */
1811         if (!pm_iir && !client_boost)
1812                 goto out;
1813
1814         mutex_lock(&rps->lock);
1815         if (!intel_rps_is_active(rps)) {
1816                 mutex_unlock(&rps->lock);
1817                 return;
1818         }
1819
1820         pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1821
1822         adj = rps->last_adj;
1823         new_freq = rps->cur_freq;
1824         min = rps->min_freq_softlimit;
1825         max = rps->max_freq_softlimit;
1826         if (client_boost)
1827                 max = rps->max_freq;
1828
1829         GT_TRACE(gt,
1830                  "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1831                  pm_iir, str_yes_no(client_boost),
1832                  adj, new_freq, min, max);
1833
1834         if (client_boost && new_freq < rps->boost_freq) {
1835                 new_freq = rps->boost_freq;
1836                 adj = 0;
1837         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1838                 if (adj > 0)
1839                         adj *= 2;
1840                 else /* CHV needs even encode values */
1841                         adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1842
1843                 if (new_freq >= rps->max_freq_softlimit)
1844                         adj = 0;
1845         } else if (client_boost) {
1846                 adj = 0;
1847         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1848                 if (rps->cur_freq > rps->efficient_freq)
1849                         new_freq = rps->efficient_freq;
1850                 else if (rps->cur_freq > rps->min_freq_softlimit)
1851                         new_freq = rps->min_freq_softlimit;
1852                 adj = 0;
1853         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1854                 if (adj < 0)
1855                         adj *= 2;
1856                 else /* CHV needs even encode values */
1857                         adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1858
1859                 if (new_freq <= rps->min_freq_softlimit)
1860                         adj = 0;
1861         } else { /* unknown event */
1862                 adj = 0;
1863         }
1864
1865         /*
1866          * sysfs frequency limits may have snuck in while
1867          * servicing the interrupt
1868          */
1869         new_freq += adj;
1870         new_freq = clamp_t(int, new_freq, min, max);
1871
1872         if (intel_rps_set(rps, new_freq)) {
1873                 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1874                 adj = 0;
1875         }
1876         rps->last_adj = adj;
1877
1878         mutex_unlock(&rps->lock);
1879
1880 out:
1881         spin_lock_irq(gt->irq_lock);
1882         gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1883         spin_unlock_irq(gt->irq_lock);
1884 }
1885
1886 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1887 {
1888         struct intel_gt *gt = rps_to_gt(rps);
1889         const u32 events = rps->pm_events & pm_iir;
1890
1891         lockdep_assert_held(gt->irq_lock);
1892
1893         if (unlikely(!events))
1894                 return;
1895
1896         GT_TRACE(gt, "irq events:%x\n", events);
1897
1898         gen6_gt_pm_mask_irq(gt, events);
1899
1900         rps->pm_iir |= events;
1901         schedule_work(&rps->work);
1902 }
1903
1904 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1905 {
1906         struct intel_gt *gt = rps_to_gt(rps);
1907         u32 events;
1908
1909         events = pm_iir & rps->pm_events;
1910         if (events) {
1911                 spin_lock(gt->irq_lock);
1912
1913                 GT_TRACE(gt, "irq events:%x\n", events);
1914
1915                 gen6_gt_pm_mask_irq(gt, events);
1916                 rps->pm_iir |= events;
1917
1918                 schedule_work(&rps->work);
1919                 spin_unlock(gt->irq_lock);
1920         }
1921
1922         if (GRAPHICS_VER(gt->i915) >= 8)
1923                 return;
1924
1925         if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1926                 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1927
1928         if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1929                 drm_dbg(&rps_to_i915(rps)->drm,
1930                         "Command parser error, pm_iir 0x%08x\n", pm_iir);
1931 }
1932
1933 void gen5_rps_irq_handler(struct intel_rps *rps)
1934 {
1935         struct intel_uncore *uncore = rps_to_uncore(rps);
1936         u32 busy_up, busy_down, max_avg, min_avg;
1937         u8 new_freq;
1938
1939         spin_lock(&mchdev_lock);
1940
1941         intel_uncore_write16(uncore,
1942                              MEMINTRSTS,
1943                              intel_uncore_read(uncore, MEMINTRSTS));
1944
1945         intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1946         busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1947         busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1948         max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1949         min_avg = intel_uncore_read(uncore, RCBMINAVG);
1950
1951         /* Handle RCS change request from hw */
1952         new_freq = rps->cur_freq;
1953         if (busy_up > max_avg)
1954                 new_freq++;
1955         else if (busy_down < min_avg)
1956                 new_freq--;
1957         new_freq = clamp(new_freq,
1958                          rps->min_freq_softlimit,
1959                          rps->max_freq_softlimit);
1960
1961         if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1962                 rps->cur_freq = new_freq;
1963
1964         spin_unlock(&mchdev_lock);
1965 }
1966
1967 void intel_rps_init_early(struct intel_rps *rps)
1968 {
1969         mutex_init(&rps->lock);
1970         mutex_init(&rps->power.mutex);
1971
1972         INIT_WORK(&rps->work, rps_work);
1973         timer_setup(&rps->timer, rps_timer, 0);
1974
1975         atomic_set(&rps->num_waiters, 0);
1976 }
1977
1978 void intel_rps_init(struct intel_rps *rps)
1979 {
1980         struct drm_i915_private *i915 = rps_to_i915(rps);
1981
1982         if (rps_uses_slpc(rps))
1983                 return;
1984
1985         if (IS_CHERRYVIEW(i915))
1986                 chv_rps_init(rps);
1987         else if (IS_VALLEYVIEW(i915))
1988                 vlv_rps_init(rps);
1989         else if (GRAPHICS_VER(i915) >= 6)
1990                 gen6_rps_init(rps);
1991         else if (IS_IRONLAKE_M(i915))
1992                 gen5_rps_init(rps);
1993
1994         /* Derive initial user preferences/limits from the hardware limits */
1995         rps->max_freq_softlimit = rps->max_freq;
1996         rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
1997         rps->min_freq_softlimit = rps->min_freq;
1998         rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
1999
2000         /* After setting max-softlimit, find the overclock max freq */
2001         if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
2002                 u32 params = 0;
2003
2004                 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, &params, NULL);
2005                 if (params & BIT(31)) { /* OC supported */
2006                         drm_dbg(&i915->drm,
2007                                 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2008                                 (rps->max_freq & 0xff) * 50,
2009                                 (params & 0xff) * 50);
2010                         rps->max_freq = params & 0xff;
2011                 }
2012         }
2013
2014         /* Finally allow us to boost to max by default */
2015         rps->boost_freq = rps->max_freq;
2016         rps->idle_freq = rps->min_freq;
2017
2018         /* Start in the middle, from here we will autotune based on workload */
2019         rps->cur_freq = rps->efficient_freq;
2020
2021         rps->pm_intrmsk_mbz = 0;
2022
2023         /*
2024          * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2025          * if GEN6_PM_UP_EI_EXPIRED is masked.
2026          *
2027          * TODO: verify if this can be reproduced on VLV,CHV.
2028          */
2029         if (GRAPHICS_VER(i915) <= 7)
2030                 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2031
2032         if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2033                 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2034
2035         /* GuC needs ARAT expired interrupt unmasked */
2036         if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2037                 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2038 }
2039
2040 void intel_rps_sanitize(struct intel_rps *rps)
2041 {
2042         if (rps_uses_slpc(rps))
2043                 return;
2044
2045         if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2046                 rps_disable_interrupts(rps);
2047 }
2048
2049 u32 intel_rps_read_rpstat(struct intel_rps *rps)
2050 {
2051         struct drm_i915_private *i915 = rps_to_i915(rps);
2052         i915_reg_t rpstat;
2053
2054         rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1;
2055
2056         return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat);
2057 }
2058
2059 static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2060 {
2061         struct drm_i915_private *i915 = rps_to_i915(rps);
2062         u32 cagf;
2063
2064         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2065                 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat);
2066         else if (GRAPHICS_VER(i915) >= 12)
2067                 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat);
2068         else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2069                 cagf = REG_FIELD_GET(RPE_MASK, rpstat);
2070         else if (GRAPHICS_VER(i915) >= 9)
2071                 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat);
2072         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2073                 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat);
2074         else if (GRAPHICS_VER(i915) >= 6)
2075                 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat);
2076         else
2077                 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat));
2078
2079         return cagf;
2080 }
2081
2082 static u32 __read_cagf(struct intel_rps *rps, bool take_fw)
2083 {
2084         struct drm_i915_private *i915 = rps_to_i915(rps);
2085         struct intel_uncore *uncore = rps_to_uncore(rps);
2086         i915_reg_t r = INVALID_MMIO_REG;
2087         u32 freq;
2088
2089         /*
2090          * For Gen12+ reading freq from HW does not need a forcewake and
2091          * registers will return 0 freq when GT is in RC6
2092          */
2093         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2094                 r = MTL_MIRROR_TARGET_WP1;
2095         } else if (GRAPHICS_VER(i915) >= 12) {
2096                 r = GEN12_RPSTAT1;
2097         } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2098                 vlv_punit_get(i915);
2099                 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2100                 vlv_punit_put(i915);
2101         } else if (GRAPHICS_VER(i915) >= 6) {
2102                 r = GEN6_RPSTAT1;
2103         } else {
2104                 r = MEMSTAT_ILK;
2105         }
2106
2107         if (i915_mmio_reg_valid(r))
2108                 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r);
2109
2110         return intel_rps_get_cagf(rps, freq);
2111 }
2112
2113 static u32 read_cagf(struct intel_rps *rps)
2114 {
2115         return __read_cagf(rps, true);
2116 }
2117
2118 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2119 {
2120         struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2121         intel_wakeref_t wakeref;
2122         u32 freq = 0;
2123
2124         with_intel_runtime_pm_if_in_use(rpm, wakeref)
2125                 freq = intel_gpu_freq(rps, read_cagf(rps));
2126
2127         return freq;
2128 }
2129
2130 u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)
2131 {
2132         return intel_gpu_freq(rps, __read_cagf(rps, false));
2133 }
2134
2135 static u32 intel_rps_read_punit_req(struct intel_rps *rps)
2136 {
2137         struct intel_uncore *uncore = rps_to_uncore(rps);
2138         struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2139         intel_wakeref_t wakeref;
2140         u32 freq = 0;
2141
2142         with_intel_runtime_pm_if_in_use(rpm, wakeref)
2143                 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2144
2145         return freq;
2146 }
2147
2148 static u32 intel_rps_get_req(u32 pureq)
2149 {
2150         u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2151
2152         return req;
2153 }
2154
2155 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2156 {
2157         u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2158
2159         return intel_gpu_freq(rps, freq);
2160 }
2161
2162 u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2163 {
2164         if (rps_uses_slpc(rps))
2165                 return intel_rps_read_punit_req_frequency(rps);
2166         else
2167                 return intel_gpu_freq(rps, rps->cur_freq);
2168 }
2169
2170 u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2171 {
2172         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2173
2174         if (rps_uses_slpc(rps))
2175                 return slpc->max_freq_softlimit;
2176         else
2177                 return intel_gpu_freq(rps, rps->max_freq_softlimit);
2178 }
2179
2180 /**
2181  * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2182  * @rps: the intel_rps structure
2183  *
2184  * Returns the max frequency in a raw format. In newer platforms raw is in
2185  * units of 50 MHz.
2186  */
2187 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2188 {
2189         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2190         u32 freq;
2191
2192         if (rps_uses_slpc(rps)) {
2193                 return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2194                                          GT_FREQUENCY_MULTIPLIER);
2195         } else {
2196                 freq = rps->max_freq;
2197                 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2198                         /* Convert GT frequency to 50 MHz units */
2199                         freq /= GEN9_FREQ_SCALER;
2200                 }
2201                 return freq;
2202         }
2203 }
2204
2205 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2206 {
2207         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2208
2209         if (rps_uses_slpc(rps))
2210                 return slpc->rp0_freq;
2211         else
2212                 return intel_gpu_freq(rps, rps->rp0_freq);
2213 }
2214
2215 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2216 {
2217         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2218
2219         if (rps_uses_slpc(rps))
2220                 return slpc->rp1_freq;
2221         else
2222                 return intel_gpu_freq(rps, rps->rp1_freq);
2223 }
2224
2225 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2226 {
2227         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2228
2229         if (rps_uses_slpc(rps))
2230                 return slpc->min_freq;
2231         else
2232                 return intel_gpu_freq(rps, rps->min_freq);
2233 }
2234
2235 static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2236 {
2237         struct intel_gt *gt = rps_to_gt(rps);
2238         struct drm_i915_private *i915 = gt->i915;
2239         struct intel_uncore *uncore = gt->uncore;
2240         struct intel_rps_freq_caps caps;
2241         u32 rp_state_limits;
2242         u32 gt_perf_status;
2243         u32 rpmodectl, rpinclimit, rpdeclimit;
2244         u32 rpstat, cagf, reqf;
2245         u32 rpcurupei, rpcurup, rpprevup;
2246         u32 rpcurdownei, rpcurdown, rpprevdown;
2247         u32 rpupei, rpupt, rpdownei, rpdownt;
2248         u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
2249
2250         rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS);
2251         gen6_rps_get_freq_caps(rps, &caps);
2252         if (IS_GEN9_LP(i915))
2253                 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS);
2254         else
2255                 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS);
2256
2257         /* RPSTAT1 is in the GT power well */
2258         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
2259
2260         reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2261         if (GRAPHICS_VER(i915) >= 9) {
2262                 reqf >>= 23;
2263         } else {
2264                 reqf &= ~GEN6_TURBO_DISABLE;
2265                 if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2266                         reqf >>= 24;
2267                 else
2268                         reqf >>= 25;
2269         }
2270         reqf = intel_gpu_freq(rps, reqf);
2271
2272         rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL);
2273         rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2274         rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2275
2276         rpstat = intel_rps_read_rpstat(rps);
2277         rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK;
2278         rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK;
2279         rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK;
2280         rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
2281         rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
2282         rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
2283
2284         rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI);
2285         rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2286
2287         rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI);
2288         rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2289
2290         cagf = intel_rps_read_actual_frequency(rps);
2291
2292         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
2293
2294         if (GRAPHICS_VER(i915) >= 11) {
2295                 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE);
2296                 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK);
2297                 /*
2298                  * The equivalent to the PM ISR & IIR cannot be read
2299                  * without affecting the current state of the system
2300                  */
2301                 pm_isr = 0;
2302                 pm_iir = 0;
2303         } else if (GRAPHICS_VER(i915) >= 8) {
2304                 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2));
2305                 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2));
2306                 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2));
2307                 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2));
2308         } else {
2309                 pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
2310                 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
2311                 pm_isr = intel_uncore_read(uncore, GEN6_PMISR);
2312                 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR);
2313         }
2314         pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2315
2316         drm_printf(p, "Video Turbo Mode: %s\n",
2317                    str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO));
2318         drm_printf(p, "HW control enabled: %s\n",
2319                    str_yes_no(rpmodectl & GEN6_RP_ENABLE));
2320         drm_printf(p, "SW control enabled: %s\n",
2321                    str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE));
2322
2323         drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n",
2324                    pm_ier, pm_imr, pm_mask);
2325         if (GRAPHICS_VER(i915) <= 10)
2326                 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n",
2327                            pm_isr, pm_iir);
2328         drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2329                    rps->pm_intrmsk_mbz);
2330         drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
2331         drm_printf(p, "Render p-state ratio: %d\n",
2332                    (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8);
2333         drm_printf(p, "Render p-state VID: %d\n",
2334                    gt_perf_status & 0xff);
2335         drm_printf(p, "Render p-state limit: %d\n",
2336                    rp_state_limits & 0xff);
2337         drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat);
2338         drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl);
2339         drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit);
2340         drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit);
2341         drm_printf(p, "RPNSWREQ: %dMHz\n", reqf);
2342         drm_printf(p, "CAGF: %dMHz\n", cagf);
2343         drm_printf(p, "RP CUR UP EI: %d (%lldns)\n",
2344                    rpcurupei,
2345                    intel_gt_pm_interval_to_ns(gt, rpcurupei));
2346         drm_printf(p, "RP CUR UP: %d (%lldns)\n",
2347                    rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup));
2348         drm_printf(p, "RP PREV UP: %d (%lldns)\n",
2349                    rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup));
2350         drm_printf(p, "Up threshold: %d%%\n",
2351                    rps->power.up_threshold);
2352         drm_printf(p, "RP UP EI: %d (%lldns)\n",
2353                    rpupei, intel_gt_pm_interval_to_ns(gt, rpupei));
2354         drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n",
2355                    rpupt, intel_gt_pm_interval_to_ns(gt, rpupt));
2356
2357         drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n",
2358                    rpcurdownei,
2359                    intel_gt_pm_interval_to_ns(gt, rpcurdownei));
2360         drm_printf(p, "RP CUR DOWN: %d (%lldns)\n",
2361                    rpcurdown,
2362                    intel_gt_pm_interval_to_ns(gt, rpcurdown));
2363         drm_printf(p, "RP PREV DOWN: %d (%lldns)\n",
2364                    rpprevdown,
2365                    intel_gt_pm_interval_to_ns(gt, rpprevdown));
2366         drm_printf(p, "Down threshold: %d%%\n",
2367                    rps->power.down_threshold);
2368         drm_printf(p, "RP DOWN EI: %d (%lldns)\n",
2369                    rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei));
2370         drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n",
2371                    rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt));
2372
2373         drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2374                    intel_gpu_freq(rps, caps.min_freq));
2375         drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2376                    intel_gpu_freq(rps, caps.rp1_freq));
2377         drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2378                    intel_gpu_freq(rps, caps.rp0_freq));
2379         drm_printf(p, "Max overclocked frequency: %dMHz\n",
2380                    intel_gpu_freq(rps, rps->max_freq));
2381
2382         drm_printf(p, "Current freq: %d MHz\n",
2383                    intel_gpu_freq(rps, rps->cur_freq));
2384         drm_printf(p, "Actual freq: %d MHz\n", cagf);
2385         drm_printf(p, "Idle freq: %d MHz\n",
2386                    intel_gpu_freq(rps, rps->idle_freq));
2387         drm_printf(p, "Min freq: %d MHz\n",
2388                    intel_gpu_freq(rps, rps->min_freq));
2389         drm_printf(p, "Boost freq: %d MHz\n",
2390                    intel_gpu_freq(rps, rps->boost_freq));
2391         drm_printf(p, "Max freq: %d MHz\n",
2392                    intel_gpu_freq(rps, rps->max_freq));
2393         drm_printf(p,
2394                    "efficient (RPe) frequency: %d MHz\n",
2395                    intel_gpu_freq(rps, rps->efficient_freq));
2396 }
2397
2398 static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2399 {
2400         struct intel_gt *gt = rps_to_gt(rps);
2401         struct intel_uncore *uncore = gt->uncore;
2402         struct intel_rps_freq_caps caps;
2403         u32 pm_mask;
2404
2405         gen6_rps_get_freq_caps(rps, &caps);
2406         pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2407
2408         drm_printf(p, "PM MASK=0x%08x\n", pm_mask);
2409         drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2410                    rps->pm_intrmsk_mbz);
2411         drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps));
2412         drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps));
2413         drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2414                    intel_gpu_freq(rps, caps.min_freq));
2415         drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2416                    intel_gpu_freq(rps, caps.rp1_freq));
2417         drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2418                    intel_gpu_freq(rps, caps.rp0_freq));
2419         drm_printf(p, "Current freq: %d MHz\n",
2420                    intel_rps_get_requested_frequency(rps));
2421         drm_printf(p, "Actual freq: %d MHz\n",
2422                    intel_rps_read_actual_frequency(rps));
2423         drm_printf(p, "Min freq: %d MHz\n",
2424                    intel_rps_get_min_frequency(rps));
2425         drm_printf(p, "Boost freq: %d MHz\n",
2426                    intel_rps_get_boost_frequency(rps));
2427         drm_printf(p, "Max freq: %d MHz\n",
2428                    intel_rps_get_max_frequency(rps));
2429         drm_printf(p,
2430                    "efficient (RPe) frequency: %d MHz\n",
2431                    intel_gpu_freq(rps, caps.rp1_freq));
2432 }
2433
2434 void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2435 {
2436         if (rps_uses_slpc(rps))
2437                 return slpc_frequency_dump(rps, p);
2438         else
2439                 return rps_frequency_dump(rps, p);
2440 }
2441
2442 static int set_max_freq(struct intel_rps *rps, u32 val)
2443 {
2444         struct drm_i915_private *i915 = rps_to_i915(rps);
2445         int ret = 0;
2446
2447         mutex_lock(&rps->lock);
2448
2449         val = intel_freq_opcode(rps, val);
2450         if (val < rps->min_freq ||
2451             val > rps->max_freq ||
2452             val < rps->min_freq_softlimit) {
2453                 ret = -EINVAL;
2454                 goto unlock;
2455         }
2456
2457         if (val > rps->rp0_freq)
2458                 drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2459                         intel_gpu_freq(rps, val));
2460
2461         rps->max_freq_softlimit = val;
2462
2463         val = clamp_t(int, rps->cur_freq,
2464                       rps->min_freq_softlimit,
2465                       rps->max_freq_softlimit);
2466
2467         /*
2468          * We still need *_set_rps to process the new max_delay and
2469          * update the interrupt limits and PMINTRMSK even though
2470          * frequency request may be unchanged.
2471          */
2472         intel_rps_set(rps, val);
2473
2474 unlock:
2475         mutex_unlock(&rps->lock);
2476
2477         return ret;
2478 }
2479
2480 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2481 {
2482         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2483
2484         if (rps_uses_slpc(rps))
2485                 return intel_guc_slpc_set_max_freq(slpc, val);
2486         else
2487                 return set_max_freq(rps, val);
2488 }
2489
2490 u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2491 {
2492         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2493
2494         if (rps_uses_slpc(rps))
2495                 return slpc->min_freq_softlimit;
2496         else
2497                 return intel_gpu_freq(rps, rps->min_freq_softlimit);
2498 }
2499
2500 /**
2501  * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2502  * @rps: the intel_rps structure
2503  *
2504  * Returns the min frequency in a raw format. In newer platforms raw is in
2505  * units of 50 MHz.
2506  */
2507 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2508 {
2509         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2510         u32 freq;
2511
2512         if (rps_uses_slpc(rps)) {
2513                 return DIV_ROUND_CLOSEST(slpc->min_freq,
2514                                          GT_FREQUENCY_MULTIPLIER);
2515         } else {
2516                 freq = rps->min_freq;
2517                 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2518                         /* Convert GT frequency to 50 MHz units */
2519                         freq /= GEN9_FREQ_SCALER;
2520                 }
2521                 return freq;
2522         }
2523 }
2524
2525 static int set_min_freq(struct intel_rps *rps, u32 val)
2526 {
2527         int ret = 0;
2528
2529         mutex_lock(&rps->lock);
2530
2531         val = intel_freq_opcode(rps, val);
2532         if (val < rps->min_freq ||
2533             val > rps->max_freq ||
2534             val > rps->max_freq_softlimit) {
2535                 ret = -EINVAL;
2536                 goto unlock;
2537         }
2538
2539         rps->min_freq_softlimit = val;
2540
2541         val = clamp_t(int, rps->cur_freq,
2542                       rps->min_freq_softlimit,
2543                       rps->max_freq_softlimit);
2544
2545         /*
2546          * We still need *_set_rps to process the new min_delay and
2547          * update the interrupt limits and PMINTRMSK even though
2548          * frequency request may be unchanged.
2549          */
2550         intel_rps_set(rps, val);
2551
2552 unlock:
2553         mutex_unlock(&rps->lock);
2554
2555         return ret;
2556 }
2557
2558 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2559 {
2560         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2561
2562         if (rps_uses_slpc(rps))
2563                 return intel_guc_slpc_set_min_freq(slpc, val);
2564         else
2565                 return set_min_freq(rps, val);
2566 }
2567
2568 static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2569 {
2570         struct intel_uncore *uncore = rps_to_uncore(rps);
2571         u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2572
2573         /* Allow punit to process software requests */
2574         intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2575 }
2576
2577 void intel_rps_raise_unslice(struct intel_rps *rps)
2578 {
2579         struct intel_uncore *uncore = rps_to_uncore(rps);
2580
2581         mutex_lock(&rps->lock);
2582
2583         if (rps_uses_slpc(rps)) {
2584                 /* RP limits have not been initialized yet for SLPC path */
2585                 struct intel_rps_freq_caps caps;
2586
2587                 gen6_rps_get_freq_caps(rps, &caps);
2588
2589                 intel_rps_set_manual(rps, true);
2590                 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2591                                    ((caps.rp0_freq <<
2592                                    GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2593                                    GEN9_IGNORE_SLICE_RATIO));
2594                 intel_rps_set_manual(rps, false);
2595         } else {
2596                 intel_rps_set(rps, rps->rp0_freq);
2597         }
2598
2599         mutex_unlock(&rps->lock);
2600 }
2601
2602 void intel_rps_lower_unslice(struct intel_rps *rps)
2603 {
2604         struct intel_uncore *uncore = rps_to_uncore(rps);
2605
2606         mutex_lock(&rps->lock);
2607
2608         if (rps_uses_slpc(rps)) {
2609                 /* RP limits have not been initialized yet for SLPC path */
2610                 struct intel_rps_freq_caps caps;
2611
2612                 gen6_rps_get_freq_caps(rps, &caps);
2613
2614                 intel_rps_set_manual(rps, true);
2615                 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2616                                    ((caps.min_freq <<
2617                                    GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2618                                    GEN9_IGNORE_SLICE_RATIO));
2619                 intel_rps_set_manual(rps, false);
2620         } else {
2621                 intel_rps_set(rps, rps->min_freq);
2622         }
2623
2624         mutex_unlock(&rps->lock);
2625 }
2626
2627 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2628 {
2629         struct intel_gt *gt = rps_to_gt(rps);
2630         intel_wakeref_t wakeref;
2631         u32 val;
2632
2633         with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2634                 val = intel_uncore_read(gt->uncore, reg32);
2635
2636         return val;
2637 }
2638
2639 bool rps_read_mask_mmio(struct intel_rps *rps,
2640                         i915_reg_t reg32, u32 mask)
2641 {
2642         return rps_read_mmio(rps, reg32) & mask;
2643 }
2644
2645 /* External interface for intel_ips.ko */
2646
2647 static struct drm_i915_private __rcu *ips_mchdev;
2648
2649 /*
2650  * Tells the intel_ips driver that the i915 driver is now loaded, if
2651  * IPS got loaded first.
2652  *
2653  * This awkward dance is so that neither module has to depend on the
2654  * other in order for IPS to do the appropriate communication of
2655  * GPU turbo limits to i915.
2656  */
2657 static void
2658 ips_ping_for_i915_load(void)
2659 {
2660         void (*link)(void);
2661
2662         link = symbol_get(ips_link_to_i915_driver);
2663         if (link) {
2664                 link();
2665                 symbol_put(ips_link_to_i915_driver);
2666         }
2667 }
2668
2669 void intel_rps_driver_register(struct intel_rps *rps)
2670 {
2671         struct intel_gt *gt = rps_to_gt(rps);
2672
2673         /*
2674          * We only register the i915 ips part with intel-ips once everything is
2675          * set up, to avoid intel-ips sneaking in and reading bogus values.
2676          */
2677         if (GRAPHICS_VER(gt->i915) == 5) {
2678                 GEM_BUG_ON(ips_mchdev);
2679                 rcu_assign_pointer(ips_mchdev, gt->i915);
2680                 ips_ping_for_i915_load();
2681         }
2682 }
2683
2684 void intel_rps_driver_unregister(struct intel_rps *rps)
2685 {
2686         if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2687                 rcu_assign_pointer(ips_mchdev, NULL);
2688 }
2689
2690 static struct drm_i915_private *mchdev_get(void)
2691 {
2692         struct drm_i915_private *i915;
2693
2694         rcu_read_lock();
2695         i915 = rcu_dereference(ips_mchdev);
2696         if (i915 && !kref_get_unless_zero(&i915->drm.ref))
2697                 i915 = NULL;
2698         rcu_read_unlock();
2699
2700         return i915;
2701 }
2702
2703 /**
2704  * i915_read_mch_val - return value for IPS use
2705  *
2706  * Calculate and return a value for the IPS driver to use when deciding whether
2707  * we have thermal and power headroom to increase CPU or GPU power budget.
2708  */
2709 unsigned long i915_read_mch_val(void)
2710 {
2711         struct drm_i915_private *i915;
2712         unsigned long chipset_val = 0;
2713         unsigned long graphics_val = 0;
2714         intel_wakeref_t wakeref;
2715
2716         i915 = mchdev_get();
2717         if (!i915)
2718                 return 0;
2719
2720         with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2721                 struct intel_ips *ips = &to_gt(i915)->rps.ips;
2722
2723                 spin_lock_irq(&mchdev_lock);
2724                 chipset_val = __ips_chipset_val(ips);
2725                 graphics_val = __ips_gfx_val(ips);
2726                 spin_unlock_irq(&mchdev_lock);
2727         }
2728
2729         drm_dev_put(&i915->drm);
2730         return chipset_val + graphics_val;
2731 }
2732 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2733
2734 /**
2735  * i915_gpu_raise - raise GPU frequency limit
2736  *
2737  * Raise the limit; IPS indicates we have thermal headroom.
2738  */
2739 bool i915_gpu_raise(void)
2740 {
2741         struct drm_i915_private *i915;
2742         struct intel_rps *rps;
2743
2744         i915 = mchdev_get();
2745         if (!i915)
2746                 return false;
2747
2748         rps = &to_gt(i915)->rps;
2749
2750         spin_lock_irq(&mchdev_lock);
2751         if (rps->max_freq_softlimit < rps->max_freq)
2752                 rps->max_freq_softlimit++;
2753         spin_unlock_irq(&mchdev_lock);
2754
2755         drm_dev_put(&i915->drm);
2756         return true;
2757 }
2758 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2759
2760 /**
2761  * i915_gpu_lower - lower GPU frequency limit
2762  *
2763  * IPS indicates we're close to a thermal limit, so throttle back the GPU
2764  * frequency maximum.
2765  */
2766 bool i915_gpu_lower(void)
2767 {
2768         struct drm_i915_private *i915;
2769         struct intel_rps *rps;
2770
2771         i915 = mchdev_get();
2772         if (!i915)
2773                 return false;
2774
2775         rps = &to_gt(i915)->rps;
2776
2777         spin_lock_irq(&mchdev_lock);
2778         if (rps->max_freq_softlimit > rps->min_freq)
2779                 rps->max_freq_softlimit--;
2780         spin_unlock_irq(&mchdev_lock);
2781
2782         drm_dev_put(&i915->drm);
2783         return true;
2784 }
2785 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2786
2787 /**
2788  * i915_gpu_busy - indicate GPU business to IPS
2789  *
2790  * Tell the IPS driver whether or not the GPU is busy.
2791  */
2792 bool i915_gpu_busy(void)
2793 {
2794         struct drm_i915_private *i915;
2795         bool ret;
2796
2797         i915 = mchdev_get();
2798         if (!i915)
2799                 return false;
2800
2801         ret = to_gt(i915)->awake;
2802
2803         drm_dev_put(&i915->drm);
2804         return ret;
2805 }
2806 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2807
2808 /**
2809  * i915_gpu_turbo_disable - disable graphics turbo
2810  *
2811  * Disable graphics turbo by resetting the max frequency and setting the
2812  * current frequency to the default.
2813  */
2814 bool i915_gpu_turbo_disable(void)
2815 {
2816         struct drm_i915_private *i915;
2817         struct intel_rps *rps;
2818         bool ret;
2819
2820         i915 = mchdev_get();
2821         if (!i915)
2822                 return false;
2823
2824         rps = &to_gt(i915)->rps;
2825
2826         spin_lock_irq(&mchdev_lock);
2827         rps->max_freq_softlimit = rps->min_freq;
2828         ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
2829         spin_unlock_irq(&mchdev_lock);
2830
2831         drm_dev_put(&i915->drm);
2832         return ret;
2833 }
2834 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2835
2836 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2837 #include "selftest_rps.c"
2838 #include "selftest_slpc.c"
2839 #endif