drivers/gpu/drm/i915/gt/intel_rps.c

   1 // SPDX-License-Identifier: MIT
   2 /*
   3  * Copyright © 2019 Intel Corporation
   4  */
   5
   6 #include <linux/string_helpers.h>
   7
   8 #include <drm/i915_drm.h>
   9
  10 #include "display/intel_display.h"
  11 #include "display/intel_display_irq.h"
  12 #include "i915_drv.h"
  13 #include "i915_irq.h"
  14 #include "i915_reg.h"
  15 #include "intel_breadcrumbs.h"
  16 #include "intel_gt.h"
  17 #include "intel_gt_clock_utils.h"
  18 #include "intel_gt_irq.h"
  19 #include "intel_gt_pm.h"
  20 #include "intel_gt_pm_irq.h"
  21 #include "intel_gt_print.h"
  22 #include "intel_gt_regs.h"
  23 #include "intel_mchbar_regs.h"
  24 #include "intel_pcode.h"
  25 #include "intel_rps.h"
  26 #include "vlv_sideband.h"
  27 #include "../../../platform/x86/intel_ips.h"
  28
  29 #define BUSY_MAX_EI     20u /* ms */
  30
  31 /*
  32  * Lock protecting IPS related data structures
  33  */
  34 static DEFINE_SPINLOCK(mchdev_lock);
  35
  36 static struct intel_gt *rps_to_gt(struct intel_rps *rps)
  37 {
  38         return container_of(rps, struct intel_gt, rps);
  39 }
  40
  41 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
  42 {
  43         return rps_to_gt(rps)->i915;
  44 }
  45
  46 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
  47 {
  48         return rps_to_gt(rps)->uncore;
  49 }
  50
  51 static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
  52 {
  53         struct intel_gt *gt = rps_to_gt(rps);
  54
  55         return &gt_to_guc(gt)->slpc;
  56 }
  57
  58 static bool rps_uses_slpc(struct intel_rps *rps)
  59 {
  60         struct intel_gt *gt = rps_to_gt(rps);
  61
  62         return intel_uc_uses_guc_slpc(&gt->uc);
  63 }
  64
  65 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
  66 {
  67         return mask & ~rps->pm_intrmsk_mbz;
  68 }
  69
  70 static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
  71 {
  72         intel_uncore_write_fw(uncore, reg, val);
  73 }
  74
  75 static void rps_timer(struct timer_list *t)
  76 {
  77         struct intel_rps *rps = from_timer(rps, t, timer);
  78         struct intel_gt *gt = rps_to_gt(rps);
  79         struct intel_engine_cs *engine;
  80         ktime_t dt, last, timestamp;
  81         enum intel_engine_id id;
  82         s64 max_busy[3] = {};
  83
  84         timestamp = 0;
  85         for_each_engine(engine, gt, id) {
  86                 s64 busy;
  87                 int i;
  88
  89                 dt = intel_engine_get_busy_time(engine, &timestamp);
  90                 last = engine->stats.rps;
  91                 engine->stats.rps = dt;
  92
  93                 busy = ktime_to_ns(ktime_sub(dt, last));
  94                 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
  95                         if (busy > max_busy[i])
  96                                 swap(busy, max_busy[i]);
  97                 }
  98         }
  99         last = rps->pm_timestamp;
 100         rps->pm_timestamp = timestamp;
 101
 102         if (intel_rps_is_active(rps)) {
 103                 s64 busy;
 104                 int i;
 105
 106                 dt = ktime_sub(timestamp, last);
 107
 108                 /*
 109                  * Our goal is to evaluate each engine independently, so we run
 110                  * at the lowest clocks required to sustain the heaviest
 111                  * workload. However, a task may be split into sequential
 112                  * dependent operations across a set of engines, such that
 113                  * the independent contributions do not account for high load,
 114                  * but overall the task is GPU bound. For example, consider
 115                  * video decode on vcs followed by colour post-processing
 116                  * on vecs, followed by general post-processing on rcs.
 117                  * Since multi-engines being active does imply a single
 118                  * continuous workload across all engines, we hedge our
 119                  * bets by only contributing a factor of the distributed
 120                  * load into our busyness calculation.
 121                  */
 122                 busy = max_busy[0];
 123                 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
 124                         if (!max_busy[i])
 125                                 break;
 126
 127                         busy += div_u64(max_busy[i], 1 << i);
 128                 }
 129                 GT_TRACE(gt,
 130                          "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
 131                          busy, (int)div64_u64(100 * busy, dt),
 132                          max_busy[0], max_busy[1], max_busy[2],
 133                          rps->pm_interval);
 134
 135                 if (100 * busy > rps->power.up_threshold * dt &&
 136                     rps->cur_freq < rps->max_freq_softlimit) {
 137                         rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
 138                         rps->pm_interval = 1;
 139                         queue_work(gt->i915->unordered_wq, &rps->work);
 140                 } else if (100 * busy < rps->power.down_threshold * dt &&
 141                            rps->cur_freq > rps->min_freq_softlimit) {
 142                         rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
 143                         rps->pm_interval = 1;
 144                         queue_work(gt->i915->unordered_wq, &rps->work);
 145                 } else {
 146                         rps->last_adj = 0;
 147                 }
 148
 149                 mod_timer(&rps->timer,
 150                           jiffies + msecs_to_jiffies(rps->pm_interval));
 151                 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
 152         }
 153 }
 154
 155 static void rps_start_timer(struct intel_rps *rps)
 156 {
 157         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 158         rps->pm_interval = 1;
 159         mod_timer(&rps->timer, jiffies + 1);
 160 }
 161
 162 static void rps_stop_timer(struct intel_rps *rps)
 163 {
 164         del_timer_sync(&rps->timer);
 165         rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
 166         cancel_work_sync(&rps->work);
 167 }
 168
 169 static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
 170 {
 171         u32 mask = 0;
 172
 173         /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
 174         if (val > rps->min_freq_softlimit)
 175                 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
 176                          GEN6_PM_RP_DOWN_THRESHOLD |
 177                          GEN6_PM_RP_DOWN_TIMEOUT);
 178
 179         if (val < rps->max_freq_softlimit)
 180                 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
 181
 182         mask &= rps->pm_events;
 183
 184         return rps_pm_sanitize_mask(rps, ~mask);
 185 }
 186
 187 static void rps_reset_ei(struct intel_rps *rps)
 188 {
 189         memset(&rps->ei, 0, sizeof(rps->ei));
 190 }
 191
 192 static void rps_enable_interrupts(struct intel_rps *rps)
 193 {
 194         struct intel_gt *gt = rps_to_gt(rps);
 195
 196         GEM_BUG_ON(rps_uses_slpc(rps));
 197
 198         GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
 199                  rps->pm_events, rps_pm_mask(rps, rps->last_freq));
 200
 201         rps_reset_ei(rps);
 202
 203         spin_lock_irq(gt->irq_lock);
 204         gen6_gt_pm_enable_irq(gt, rps->pm_events);
 205         spin_unlock_irq(gt->irq_lock);
 206
 207         intel_uncore_write(gt->uncore,
 208                            GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
 209 }
 210
 211 static void gen6_rps_reset_interrupts(struct intel_rps *rps)
 212 {
 213         gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
 214 }
 215
 216 static void gen11_rps_reset_interrupts(struct intel_rps *rps)
 217 {
 218         while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
 219                 ;
 220 }
 221
 222 static void rps_reset_interrupts(struct intel_rps *rps)
 223 {
 224         struct intel_gt *gt = rps_to_gt(rps);
 225
 226         spin_lock_irq(gt->irq_lock);
 227         if (GRAPHICS_VER(gt->i915) >= 11)
 228                 gen11_rps_reset_interrupts(rps);
 229         else
 230                 gen6_rps_reset_interrupts(rps);
 231
 232         rps->pm_iir = 0;
 233         spin_unlock_irq(gt->irq_lock);
 234 }
 235
 236 static void rps_disable_interrupts(struct intel_rps *rps)
 237 {
 238         struct intel_gt *gt = rps_to_gt(rps);
 239
 240         intel_uncore_write(gt->uncore,
 241                            GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
 242
 243         spin_lock_irq(gt->irq_lock);
 244         gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
 245         spin_unlock_irq(gt->irq_lock);
 246
 247         intel_synchronize_irq(gt->i915);
 248
 249         /*
 250          * Now that we will not be generating any more work, flush any
 251          * outstanding tasks. As we are called on the RPS idle path,
 252          * we will reset the GPU to minimum frequencies, so the current
 253          * state of the worker can be discarded.
 254          */
 255         cancel_work_sync(&rps->work);
 256
 257         rps_reset_interrupts(rps);
 258         GT_TRACE(gt, "interrupts:off\n");
 259 }
 260
 261 static const struct cparams {
 262         u16 i;
 263         u16 t;
 264         u16 m;
 265         u16 c;
 266 } cparams[] = {
 267         { 1, 1333, 301, 28664 },
 268         { 1, 1066, 294, 24460 },
 269         { 1, 800, 294, 25192 },
 270         { 0, 1333, 276, 27605 },
 271         { 0, 1066, 276, 27605 },
 272         { 0, 800, 231, 23784 },
 273 };
 274
 275 static void gen5_rps_init(struct intel_rps *rps)
 276 {
 277         struct drm_i915_private *i915 = rps_to_i915(rps);
 278         struct intel_uncore *uncore = rps_to_uncore(rps);
 279         u8 fmax, fmin, fstart;
 280         u32 rgvmodectl;
 281         int c_m, i;
 282
 283         if (i915->fsb_freq <= 3200)
 284                 c_m = 0;
 285         else if (i915->fsb_freq <= 4800)
 286                 c_m = 1;
 287         else
 288                 c_m = 2;
 289
 290         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
 291                 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
 292                         rps->ips.m = cparams[i].m;
 293                         rps->ips.c = cparams[i].c;
 294                         break;
 295                 }
 296         }
 297
 298         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 299
 300         /* Set up min, max, and cur for interrupt handling */
 301         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
 302         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
 303         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 304                 MEMMODE_FSTART_SHIFT;
 305         drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
 306                 fmax, fmin, fstart);
 307
 308         rps->min_freq = fmax;
 309         rps->efficient_freq = fstart;
 310         rps->max_freq = fmin;
 311 }
 312
 313 static unsigned long
 314 __ips_chipset_val(struct intel_ips *ips)
 315 {
 316         struct intel_uncore *uncore =
 317                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 318         unsigned long now = jiffies_to_msecs(jiffies), dt;
 319         unsigned long result;
 320         u64 total, delta;
 321
 322         lockdep_assert_held(&mchdev_lock);
 323
 324         /*
 325          * Prevent division-by-zero if we are asking too fast.
 326          * Also, we don't get interesting results if we are polling
 327          * faster than once in 10ms, so just return the saved value
 328          * in such cases.
 329          */
 330         dt = now - ips->last_time1;
 331         if (dt <= 10)
 332                 return ips->chipset_power;
 333
 334         /* FIXME: handle per-counter overflow */
 335         total = intel_uncore_read(uncore, DMIEC);
 336         total += intel_uncore_read(uncore, DDREC);
 337         total += intel_uncore_read(uncore, CSIEC);
 338
 339         delta = total - ips->last_count1;
 340
 341         result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
 342
 343         ips->last_count1 = total;
 344         ips->last_time1 = now;
 345
 346         ips->chipset_power = result;
 347
 348         return result;
 349 }
 350
 351 static unsigned long ips_mch_val(struct intel_uncore *uncore)
 352 {
 353         unsigned int m, x, b;
 354         u32 tsfs;
 355
 356         tsfs = intel_uncore_read(uncore, TSFS);
 357         x = intel_uncore_read8(uncore, TR1);
 358
 359         b = tsfs & TSFS_INTR_MASK;
 360         m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
 361
 362         return m * x / 127 - b;
 363 }
 364
 365 static int _pxvid_to_vd(u8 pxvid)
 366 {
 367         if (pxvid == 0)
 368                 return 0;
 369
 370         if (pxvid >= 8 && pxvid < 31)
 371                 pxvid = 31;
 372
 373         return (pxvid + 2) * 125;
 374 }
 375
 376 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
 377 {
 378         const int vd = _pxvid_to_vd(pxvid);
 379
 380         if (INTEL_INFO(i915)->is_mobile)
 381                 return max(vd - 1125, 0);
 382
 383         return vd;
 384 }
 385
 386 static void __gen5_ips_update(struct intel_ips *ips)
 387 {
 388         struct intel_uncore *uncore =
 389                 rps_to_uncore(container_of(ips, struct intel_rps, ips));
 390         u64 now, delta, dt;
 391         u32 count;
 392
 393         lockdep_assert_held(&mchdev_lock);
 394
 395         now = ktime_get_raw_ns();
 396         dt = now - ips->last_time2;
 397         do_div(dt, NSEC_PER_MSEC);
 398
 399         /* Don't divide by 0 */
 400         if (dt <= 10)
 401                 return;
 402
 403         count = intel_uncore_read(uncore, GFXEC);
 404         delta = count - ips->last_count2;
 405
 406         ips->last_count2 = count;
 407         ips->last_time2 = now;
 408
 409         /* More magic constants... */
 410         ips->gfx_power = div_u64(delta * 1181, dt * 10);
 411 }
 412
 413 static void gen5_rps_update(struct intel_rps *rps)
 414 {
 415         spin_lock_irq(&mchdev_lock);
 416         __gen5_ips_update(&rps->ips);
 417         spin_unlock_irq(&mchdev_lock);
 418 }
 419
 420 static unsigned int gen5_invert_freq(struct intel_rps *rps,
 421                                      unsigned int val)
 422 {
 423         /* Invert the frequency bin into an ips delay */
 424         val = rps->max_freq - val;
 425         val = rps->min_freq + val;
 426
 427         return val;
 428 }
 429
 430 static int __gen5_rps_set(struct intel_rps *rps, u8 val)
 431 {
 432         struct intel_uncore *uncore = rps_to_uncore(rps);
 433         u16 rgvswctl;
 434
 435         lockdep_assert_held(&mchdev_lock);
 436
 437         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 438         if (rgvswctl & MEMCTL_CMD_STS) {
 439                 drm_dbg(&rps_to_i915(rps)->drm,
 440                         "gpu busy, RCS change rejected\n");
 441                 return -EBUSY; /* still busy with another command */
 442         }
 443
 444         /* Invert the frequency bin into an ips delay */
 445         val = gen5_invert_freq(rps, val);
 446
 447         rgvswctl =
 448                 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
 449                 (val << MEMCTL_FREQ_SHIFT) |
 450                 MEMCTL_SFCAVM;
 451         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 452         intel_uncore_posting_read16(uncore, MEMSWCTL);
 453
 454         rgvswctl |= MEMCTL_CMD_STS;
 455         intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
 456
 457         return 0;
 458 }
 459
 460 static int gen5_rps_set(struct intel_rps *rps, u8 val)
 461 {
 462         int err;
 463
 464         spin_lock_irq(&mchdev_lock);
 465         err = __gen5_rps_set(rps, val);
 466         spin_unlock_irq(&mchdev_lock);
 467
 468         return err;
 469 }
 470
 471 static unsigned long intel_pxfreq(u32 vidfreq)
 472 {
 473         int div = (vidfreq & 0x3f0000) >> 16;
 474         int post = (vidfreq & 0x3000) >> 12;
 475         int pre = (vidfreq & 0x7);
 476
 477         if (!pre)
 478                 return 0;
 479
 480         return div * 133333 / (pre << post);
 481 }
 482
 483 static unsigned int init_emon(struct intel_uncore *uncore)
 484 {
 485         u8 pxw[16];
 486         int i;
 487
 488         /* Disable to program */
 489         intel_uncore_write(uncore, ECR, 0);
 490         intel_uncore_posting_read(uncore, ECR);
 491
 492         /* Program energy weights for various events */
 493         intel_uncore_write(uncore, SDEW, 0x15040d00);
 494         intel_uncore_write(uncore, CSIEW0, 0x007f0000);
 495         intel_uncore_write(uncore, CSIEW1, 0x1e220004);
 496         intel_uncore_write(uncore, CSIEW2, 0x04000004);
 497
 498         for (i = 0; i < 5; i++)
 499                 intel_uncore_write(uncore, PEW(i), 0);
 500         for (i = 0; i < 3; i++)
 501                 intel_uncore_write(uncore, DEW(i), 0);
 502
 503         /* Program P-state weights to account for frequency power adjustment */
 504         for (i = 0; i < 16; i++) {
 505                 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
 506                 unsigned int freq = intel_pxfreq(pxvidfreq);
 507                 unsigned int vid =
 508                         (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 509                 unsigned int val;
 510
 511                 val = vid * vid * freq / 1000 * 255;
 512                 val /= 127 * 127 * 900;
 513
 514                 pxw[i] = val;
 515         }
 516         /* Render standby states get 0 weight */
 517         pxw[14] = 0;
 518         pxw[15] = 0;
 519
 520         for (i = 0; i < 4; i++) {
 521                 intel_uncore_write(uncore, PXW(i),
 522                                    pxw[i * 4 + 0] << 24 |
 523                                    pxw[i * 4 + 1] << 16 |
 524                                    pxw[i * 4 + 2] <<  8 |
 525                                    pxw[i * 4 + 3] <<  0);
 526         }
 527
 528         /* Adjust magic regs to magic values (more experimental results) */
 529         intel_uncore_write(uncore, OGW0, 0);
 530         intel_uncore_write(uncore, OGW1, 0);
 531         intel_uncore_write(uncore, EG0, 0x00007f00);
 532         intel_uncore_write(uncore, EG1, 0x0000000e);
 533         intel_uncore_write(uncore, EG2, 0x000e0000);
 534         intel_uncore_write(uncore, EG3, 0x68000300);
 535         intel_uncore_write(uncore, EG4, 0x42000000);
 536         intel_uncore_write(uncore, EG5, 0x00140031);
 537         intel_uncore_write(uncore, EG6, 0);
 538         intel_uncore_write(uncore, EG7, 0);
 539
 540         for (i = 0; i < 8; i++)
 541                 intel_uncore_write(uncore, PXWL(i), 0);
 542
 543         /* Enable PMON + select events */
 544         intel_uncore_write(uncore, ECR, 0x80000019);
 545
 546         return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
 547 }
 548
 549 static bool gen5_rps_enable(struct intel_rps *rps)
 550 {
 551         struct drm_i915_private *i915 = rps_to_i915(rps);
 552         struct intel_uncore *uncore = rps_to_uncore(rps);
 553         u8 fstart, vstart;
 554         u32 rgvmodectl;
 555
 556         spin_lock_irq(&mchdev_lock);
 557
 558         rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
 559
 560         /* Enable temp reporting */
 561         intel_uncore_write16(uncore, PMMISC,
 562                              intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
 563         intel_uncore_write16(uncore, TSC1,
 564                              intel_uncore_read16(uncore, TSC1) | TSE);
 565
 566         /* 100ms RC evaluation intervals */
 567         intel_uncore_write(uncore, RCUPEI, 100000);
 568         intel_uncore_write(uncore, RCDNEI, 100000);
 569
 570         /* Set max/min thresholds to 90ms and 80ms respectively */
 571         intel_uncore_write(uncore, RCBMAXAVG, 90000);
 572         intel_uncore_write(uncore, RCBMINAVG, 80000);
 573
 574         intel_uncore_write(uncore, MEMIHYST, 1);
 575
 576         /* Set up min, max, and cur for interrupt handling */
 577         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
 578                 MEMMODE_FSTART_SHIFT;
 579
 580         vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
 581                   PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
 582
 583         intel_uncore_write(uncore,
 584                            MEMINTREN,
 585                            MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
 586
 587         intel_uncore_write(uncore, VIDSTART, vstart);
 588         intel_uncore_posting_read(uncore, VIDSTART);
 589
 590         rgvmodectl |= MEMMODE_SWMODE_EN;
 591         intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
 592
 593         if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
 594                              MEMCTL_CMD_STS) == 0, 10))
 595                 drm_err(&uncore->i915->drm,
 596                         "stuck trying to change perf mode\n");
 597         mdelay(1);
 598
 599         __gen5_rps_set(rps, rps->cur_freq);
 600
 601         rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
 602         rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
 603         rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
 604         rps->ips.last_time1 = jiffies_to_msecs(jiffies);
 605
 606         rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
 607         rps->ips.last_time2 = ktime_get_raw_ns();
 608
 609         spin_lock(&i915->irq_lock);
 610         ilk_enable_display_irq(i915, DE_PCU_EVENT);
 611         spin_unlock(&i915->irq_lock);
 612
 613         spin_unlock_irq(&mchdev_lock);
 614
 615         rps->ips.corr = init_emon(uncore);
 616
 617         return true;
 618 }
 619
 620 static void gen5_rps_disable(struct intel_rps *rps)
 621 {
 622         struct drm_i915_private *i915 = rps_to_i915(rps);
 623         struct intel_uncore *uncore = rps_to_uncore(rps);
 624         u16 rgvswctl;
 625
 626         spin_lock_irq(&mchdev_lock);
 627
 628         spin_lock(&i915->irq_lock);
 629         ilk_disable_display_irq(i915, DE_PCU_EVENT);
 630         spin_unlock(&i915->irq_lock);
 631
 632         rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
 633
 634         /* Ack interrupts, disable EFC interrupt */
 635         intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0);
 636         intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
 637
 638         /* Go back to the starting frequency */
 639         __gen5_rps_set(rps, rps->idle_freq);
 640         mdelay(1);
 641         rgvswctl |= MEMCTL_CMD_STS;
 642         intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
 643         mdelay(1);
 644
 645         spin_unlock_irq(&mchdev_lock);
 646 }
 647
 648 static u32 rps_limits(struct intel_rps *rps, u8 val)
 649 {
 650         u32 limits;
 651
 652         /*
 653          * Only set the down limit when we've reached the lowest level to avoid
 654          * getting more interrupts, otherwise leave this clear. This prevents a
 655          * race in the hw when coming out of rc6: There's a tiny window where
 656          * the hw runs at the minimal clock before selecting the desired
 657          * frequency, if the down threshold expires in that window we will not
 658          * receive a down interrupt.
 659          */
 660         if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
 661                 limits = rps->max_freq_softlimit << 23;
 662                 if (val <= rps->min_freq_softlimit)
 663                         limits |= rps->min_freq_softlimit << 14;
 664         } else {
 665                 limits = rps->max_freq_softlimit << 24;
 666                 if (val <= rps->min_freq_softlimit)
 667                         limits |= rps->min_freq_softlimit << 16;
 668         }
 669
 670         return limits;
 671 }
 672
 673 static void rps_set_power(struct intel_rps *rps, int new_power)
 674 {
 675         struct intel_gt *gt = rps_to_gt(rps);
 676         struct intel_uncore *uncore = gt->uncore;
 677         u32 ei_up = 0, ei_down = 0;
 678
 679         lockdep_assert_held(&rps->power.mutex);
 680
 681         if (new_power == rps->power.mode)
 682                 return;
 683
 684         /* Note the units here are not exactly 1us, but 1280ns. */
 685         switch (new_power) {
 686         case LOW_POWER:
 687                 ei_up = 16000;
 688                 ei_down = 32000;
 689                 break;
 690
 691         case BETWEEN:
 692                 ei_up = 13000;
 693                 ei_down = 32000;
 694                 break;
 695
 696         case HIGH_POWER:
 697                 ei_up = 10000;
 698                 ei_down = 32000;
 699                 break;
 700         }
 701
 702         /* When byt can survive without system hang with dynamic
 703          * sw freq adjustments, this restriction can be lifted.
 704          */
 705         if (IS_VALLEYVIEW(gt->i915))
 706                 goto skip_hw_write;
 707
 708         GT_TRACE(gt,
 709                  "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
 710                  new_power,
 711                  rps->power.up_threshold, ei_up,
 712                  rps->power.down_threshold, ei_down);
 713
 714         set(uncore, GEN6_RP_UP_EI,
 715             intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
 716         set(uncore, GEN6_RP_UP_THRESHOLD,
 717             intel_gt_ns_to_pm_interval(gt,
 718                                        ei_up * rps->power.up_threshold * 10));
 719
 720         set(uncore, GEN6_RP_DOWN_EI,
 721             intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
 722         set(uncore, GEN6_RP_DOWN_THRESHOLD,
 723             intel_gt_ns_to_pm_interval(gt,
 724                                        ei_down *
 725                                        rps->power.down_threshold * 10));
 726
 727         set(uncore, GEN6_RP_CONTROL,
 728             (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
 729             GEN6_RP_MEDIA_HW_NORMAL_MODE |
 730             GEN6_RP_MEDIA_IS_GFX |
 731             GEN6_RP_ENABLE |
 732             GEN6_RP_UP_BUSY_AVG |
 733             GEN6_RP_DOWN_IDLE_AVG);
 734
 735 skip_hw_write:
 736         rps->power.mode = new_power;
 737 }
 738
 739 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
 740 {
 741         int new_power;
 742
 743         new_power = rps->power.mode;
 744         switch (rps->power.mode) {
 745         case LOW_POWER:
 746                 if (val > rps->efficient_freq + 1 &&
 747                     val > rps->cur_freq)
 748                         new_power = BETWEEN;
 749                 break;
 750
 751         case BETWEEN:
 752                 if (val <= rps->efficient_freq &&
 753                     val < rps->cur_freq)
 754                         new_power = LOW_POWER;
 755                 else if (val >= rps->rp0_freq &&
 756                          val > rps->cur_freq)
 757                         new_power = HIGH_POWER;
 758                 break;
 759
 760         case HIGH_POWER:
 761                 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
 762                     val < rps->cur_freq)
 763                         new_power = BETWEEN;
 764                 break;
 765         }
 766         /* Max/min bins are special */
 767         if (val <= rps->min_freq_softlimit)
 768                 new_power = LOW_POWER;
 769         if (val >= rps->max_freq_softlimit)
 770                 new_power = HIGH_POWER;
 771
 772         mutex_lock(&rps->power.mutex);
 773         if (rps->power.interactive)
 774                 new_power = HIGH_POWER;
 775         rps_set_power(rps, new_power);
 776         mutex_unlock(&rps->power.mutex);
 777 }
 778
 779 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
 780 {
 781         GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
 782                  str_yes_no(interactive));
 783
 784         mutex_lock(&rps->power.mutex);
 785         if (interactive) {
 786                 if (!rps->power.interactive++ && intel_rps_is_active(rps))
 787                         rps_set_power(rps, HIGH_POWER);
 788         } else {
 789                 GEM_BUG_ON(!rps->power.interactive);
 790                 rps->power.interactive--;
 791         }
 792         mutex_unlock(&rps->power.mutex);
 793 }
 794
 795 static int gen6_rps_set(struct intel_rps *rps, u8 val)
 796 {
 797         struct intel_uncore *uncore = rps_to_uncore(rps);
 798         struct drm_i915_private *i915 = rps_to_i915(rps);
 799         u32 swreq;
 800
 801         GEM_BUG_ON(rps_uses_slpc(rps));
 802
 803         if (GRAPHICS_VER(i915) >= 9)
 804                 swreq = GEN9_FREQUENCY(val);
 805         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
 806                 swreq = HSW_FREQUENCY(val);
 807         else
 808                 swreq = (GEN6_FREQUENCY(val) |
 809                          GEN6_OFFSET(0) |
 810                          GEN6_AGGRESSIVE_TURBO);
 811         set(uncore, GEN6_RPNSWREQ, swreq);
 812
 813         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
 814                  val, intel_gpu_freq(rps, val), swreq);
 815
 816         return 0;
 817 }
 818
 819 static int vlv_rps_set(struct intel_rps *rps, u8 val)
 820 {
 821         struct drm_i915_private *i915 = rps_to_i915(rps);
 822         int err;
 823
 824         vlv_punit_get(i915);
 825         err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
 826         vlv_punit_put(i915);
 827
 828         GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
 829                  val, intel_gpu_freq(rps, val));
 830
 831         return err;
 832 }
 833
 834 static int rps_set(struct intel_rps *rps, u8 val, bool update)
 835 {
 836         struct drm_i915_private *i915 = rps_to_i915(rps);
 837         int err;
 838
 839         if (val == rps->last_freq)
 840                 return 0;
 841
 842         if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 843                 err = vlv_rps_set(rps, val);
 844         else if (GRAPHICS_VER(i915) >= 6)
 845                 err = gen6_rps_set(rps, val);
 846         else
 847                 err = gen5_rps_set(rps, val);
 848         if (err)
 849                 return err;
 850
 851         if (update && GRAPHICS_VER(i915) >= 6)
 852                 gen6_rps_set_thresholds(rps, val);
 853         rps->last_freq = val;
 854
 855         return 0;
 856 }
 857
 858 void intel_rps_unpark(struct intel_rps *rps)
 859 {
 860         if (!intel_rps_is_enabled(rps))
 861                 return;
 862
 863         GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
 864
 865         /*
 866          * Use the user's desired frequency as a guide, but for better
 867          * performance, jump directly to RPe as our starting frequency.
 868          */
 869         mutex_lock(&rps->lock);
 870
 871         intel_rps_set_active(rps);
 872         intel_rps_set(rps,
 873                       clamp(rps->cur_freq,
 874                             rps->min_freq_softlimit,
 875                             rps->max_freq_softlimit));
 876
 877         mutex_unlock(&rps->lock);
 878
 879         rps->pm_iir = 0;
 880         if (intel_rps_has_interrupts(rps))
 881                 rps_enable_interrupts(rps);
 882         if (intel_rps_uses_timer(rps))
 883                 rps_start_timer(rps);
 884
 885         if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
 886                 gen5_rps_update(rps);
 887 }
 888
 889 void intel_rps_park(struct intel_rps *rps)
 890 {
 891         int adj;
 892
 893         if (!intel_rps_is_enabled(rps))
 894                 return;
 895
 896         if (!intel_rps_clear_active(rps))
 897                 return;
 898
 899         if (intel_rps_uses_timer(rps))
 900                 rps_stop_timer(rps);
 901         if (intel_rps_has_interrupts(rps))
 902                 rps_disable_interrupts(rps);
 903
 904         if (rps->last_freq <= rps->idle_freq)
 905                 return;
 906
 907         /*
 908          * The punit delays the write of the frequency and voltage until it
 909          * determines the GPU is awake. During normal usage we don't want to
 910          * waste power changing the frequency if the GPU is sleeping (rc6).
 911          * However, the GPU and driver is now idle and we do not want to delay
 912          * switching to minimum voltage (reducing power whilst idle) as we do
 913          * not expect to be woken in the near future and so must flush the
 914          * change by waking the device.
 915          *
 916          * We choose to take the media powerwell (either would do to trick the
 917          * punit into committing the voltage change) as that takes a lot less
 918          * power than the render powerwell.
 919          */
 920         intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 921         rps_set(rps, rps->idle_freq, false);
 922         intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
 923
 924         /*
 925          * Since we will try and restart from the previously requested
 926          * frequency on unparking, treat this idle point as a downclock
 927          * interrupt and reduce the frequency for resume. If we park/unpark
 928          * more frequently than the rps worker can run, we will not respond
 929          * to any EI and never see a change in frequency.
 930          *
 931          * (Note we accommodate Cherryview's limitation of only using an
 932          * even bin by applying it to all.)
 933          */
 934         adj = rps->last_adj;
 935         if (adj < 0)
 936                 adj *= 2;
 937         else /* CHV needs even encode values */
 938                 adj = -2;
 939         rps->last_adj = adj;
 940         rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
 941         if (rps->cur_freq < rps->efficient_freq) {
 942                 rps->cur_freq = rps->efficient_freq;
 943                 rps->last_adj = 0;
 944         }
 945
 946         GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
 947 }
 948
 949 u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
 950 {
 951         struct intel_guc_slpc *slpc;
 952
 953         if (rps_uses_slpc(rps)) {
 954                 slpc = rps_to_slpc(rps);
 955
 956                 return slpc->boost_freq;
 957         } else {
 958                 return intel_gpu_freq(rps, rps->boost_freq);
 959         }
 960 }
 961
 962 static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
 963 {
 964         bool boost = false;
 965
 966         /* Validate against (static) hardware limits */
 967         val = intel_freq_opcode(rps, val);
 968         if (val < rps->min_freq || val > rps->max_freq)
 969                 return -EINVAL;
 970
 971         mutex_lock(&rps->lock);
 972         if (val != rps->boost_freq) {
 973                 rps->boost_freq = val;
 974                 boost = atomic_read(&rps->num_waiters);
 975         }
 976         mutex_unlock(&rps->lock);
 977         if (boost)
 978                 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);
 979
 980         return 0;
 981 }
 982
 983 int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
 984 {
 985         struct intel_guc_slpc *slpc;
 986
 987         if (rps_uses_slpc(rps)) {
 988                 slpc = rps_to_slpc(rps);
 989
 990                 return intel_guc_slpc_set_boost_freq(slpc, freq);
 991         } else {
 992                 return rps_set_boost_freq(rps, freq);
 993         }
 994 }
 995
 996 void intel_rps_dec_waiters(struct intel_rps *rps)
 997 {
 998         struct intel_guc_slpc *slpc;
 999
1000         if (rps_uses_slpc(rps)) {
1001                 slpc = rps_to_slpc(rps);
1002
1003                 intel_guc_slpc_dec_waiters(slpc);
1004         } else {
1005                 atomic_dec(&rps->num_waiters);
1006         }
1007 }
1008
1009 void intel_rps_boost(struct i915_request *rq)
1010 {
1011         struct intel_guc_slpc *slpc;
1012
1013         if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1014                 return;
1015
1016         /* Waitboost is not needed for contexts marked with a Freq hint */
1017         if (test_bit(CONTEXT_LOW_LATENCY, &rq->context->flags))
1018                 return;
1019
1020         /* Serializes with i915_request_retire() */
1021         if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1022                 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1023
1024                 if (rps_uses_slpc(rps)) {
1025                         slpc = rps_to_slpc(rps);
1026
1027                         if (slpc->min_freq_softlimit >= slpc->boost_freq)
1028                                 return;
1029
1030                         /* Return if old value is non zero */
1031                         if (!atomic_fetch_inc(&slpc->num_waiters)) {
1032                                 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1033                                          rq->fence.context, rq->fence.seqno);
1034                                 queue_work(rps_to_gt(rps)->i915->unordered_wq,
1035                                            &slpc->boost_work);
1036                         }
1037
1038                         return;
1039                 }
1040
1041                 if (atomic_fetch_inc(&rps->num_waiters))
1042                         return;
1043
1044                 if (!intel_rps_is_active(rps))
1045                         return;
1046
1047                 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1048                          rq->fence.context, rq->fence.seqno);
1049
1050                 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1051                         queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);
1052
1053                 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1054         }
1055 }
1056
1057 int intel_rps_set(struct intel_rps *rps, u8 val)
1058 {
1059         int err;
1060
1061         lockdep_assert_held(&rps->lock);
1062         GEM_BUG_ON(val > rps->max_freq);
1063         GEM_BUG_ON(val < rps->min_freq);
1064
1065         if (intel_rps_is_active(rps)) {
1066                 err = rps_set(rps, val, true);
1067                 if (err)
1068                         return err;
1069
1070                 /*
1071                  * Make sure we continue to get interrupts
1072                  * until we hit the minimum or maximum frequencies.
1073                  */
1074                 if (intel_rps_has_interrupts(rps)) {
1075                         struct intel_uncore *uncore = rps_to_uncore(rps);
1076
1077                         set(uncore,
1078                             GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
1079
1080                         set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
1081                 }
1082         }
1083
1084         rps->cur_freq = val;
1085         return 0;
1086 }
1087
1088 static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1089 {
1090         struct drm_i915_private *i915 = rps_to_i915(rps);
1091         struct intel_uncore *uncore = rps_to_uncore(rps);
1092
1093         if (IS_GEN9_LP(i915))
1094                 return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1095         else
1096                 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1097 }
1098
1099 static void
1100 mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1101 {
1102         struct intel_uncore *uncore = rps_to_uncore(rps);
1103         u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ?
1104                                 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) :
1105                                 intel_uncore_read(uncore, MTL_RP_STATE_CAP);
1106         u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ?
1107                         intel_uncore_read(uncore, MTL_MPE_FREQUENCY) :
1108                         intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY);
1109
1110         /* MTL values are in units of 16.67 MHz */
1111         caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap);
1112         caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap);
1113         caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe);
1114 }
1115
1116 static void
1117 __gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1118 {
1119         struct drm_i915_private *i915 = rps_to_i915(rps);
1120         u32 rp_state_cap;
1121
1122         rp_state_cap = intel_rps_read_state_cap(rps);
1123
1124         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
1125         if (IS_GEN9_LP(i915)) {
1126                 caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1127                 caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1128                 caps->min_freq = (rp_state_cap >>  0) & 0xff;
1129         } else {
1130                 caps->rp0_freq = (rp_state_cap >>  0) & 0xff;
1131                 if (GRAPHICS_VER(i915) >= 10)
1132                         caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1133                                                        intel_uncore_read(to_gt(i915)->uncore,
1134                                                        GEN10_FREQ_INFO_REC));
1135                 else
1136                         caps->rp1_freq = (rp_state_cap >>  8) & 0xff;
1137                 caps->min_freq = (rp_state_cap >> 16) & 0xff;
1138         }
1139
1140         if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1141                 /*
1142                  * In this case rp_state_cap register reports frequencies in
1143                  * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1144                  * units of 16.67 MHz
1145                  */
1146                 caps->rp0_freq *= GEN9_FREQ_SCALER;
1147                 caps->rp1_freq *= GEN9_FREQ_SCALER;
1148                 caps->min_freq *= GEN9_FREQ_SCALER;
1149         }
1150 }
1151
1152 /**
1153  * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1154  * @rps: the intel_rps structure
1155  * @caps: returned freq caps
1156  *
1157  * Returned "caps" frequencies should be converted to MHz using
1158  * intel_gpu_freq()
1159  */
1160 void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1161 {
1162         struct drm_i915_private *i915 = rps_to_i915(rps);
1163
1164         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1165                 return mtl_get_freq_caps(rps, caps);
1166         else
1167                 return __gen6_rps_get_freq_caps(rps, caps);
1168 }
1169
1170 static void gen6_rps_init(struct intel_rps *rps)
1171 {
1172         struct drm_i915_private *i915 = rps_to_i915(rps);
1173         struct intel_rps_freq_caps caps;
1174
1175         gen6_rps_get_freq_caps(rps, &caps);
1176         rps->rp0_freq = caps.rp0_freq;
1177         rps->rp1_freq = caps.rp1_freq;
1178         rps->min_freq = caps.min_freq;
1179
1180         /* hw_max = RP0 until we check for overclocking */
1181         rps->max_freq = rps->rp0_freq;
1182
1183         rps->efficient_freq = rps->rp1_freq;
1184         if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1185             IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1186                 u32 ddcc_status = 0;
1187                 u32 mult = 1;
1188
1189                 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1190                         mult = GEN9_FREQ_SCALER;
1191                 if (snb_pcode_read(rps_to_gt(rps)->uncore,
1192                                    HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1193                                    &ddcc_status, NULL) == 0)
1194                         rps->efficient_freq =
1195                                 clamp_t(u32,
1196                                         ((ddcc_status >> 8) & 0xff) * mult,
1197                                         rps->min_freq,
1198                                         rps->max_freq);
1199         }
1200 }
1201
1202 static bool rps_reset(struct intel_rps *rps)
1203 {
1204         struct drm_i915_private *i915 = rps_to_i915(rps);
1205
1206         /* force a reset */
1207         rps->power.mode = -1;
1208         rps->last_freq = -1;
1209
1210         if (rps_set(rps, rps->min_freq, true)) {
1211                 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1212                 return false;
1213         }
1214
1215         rps->cur_freq = rps->min_freq;
1216         return true;
1217 }
1218
1219 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
1220 static bool gen9_rps_enable(struct intel_rps *rps)
1221 {
1222         struct intel_gt *gt = rps_to_gt(rps);
1223         struct intel_uncore *uncore = gt->uncore;
1224
1225         /* Program defaults and thresholds for RPS */
1226         if (GRAPHICS_VER(gt->i915) == 9)
1227                 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1228                                       GEN9_FREQUENCY(rps->rp1_freq));
1229
1230         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1231
1232         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1233
1234         return rps_reset(rps);
1235 }
1236
1237 static bool gen8_rps_enable(struct intel_rps *rps)
1238 {
1239         struct intel_uncore *uncore = rps_to_uncore(rps);
1240
1241         intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1242                               HSW_FREQUENCY(rps->rp1_freq));
1243
1244         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1245
1246         rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1247
1248         return rps_reset(rps);
1249 }
1250
1251 static bool gen6_rps_enable(struct intel_rps *rps)
1252 {
1253         struct intel_uncore *uncore = rps_to_uncore(rps);
1254
1255         /* Power down if completely idle for over 50ms */
1256         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1257         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1258
1259         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1260                           GEN6_PM_RP_DOWN_THRESHOLD |
1261                           GEN6_PM_RP_DOWN_TIMEOUT);
1262
1263         return rps_reset(rps);
1264 }
1265
1266 static int chv_rps_max_freq(struct intel_rps *rps)
1267 {
1268         struct drm_i915_private *i915 = rps_to_i915(rps);
1269         struct intel_gt *gt = rps_to_gt(rps);
1270         u32 val;
1271
1272         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1273
1274         switch (gt->info.sseu.eu_total) {
1275         case 8:
1276                 /* (2 * 4) config */
1277                 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1278                 break;
1279         case 12:
1280                 /* (2 * 6) config */
1281                 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1282                 break;
1283         case 16:
1284                 /* (2 * 8) config */
1285         default:
1286                 /* Setting (2 * 8) Min RP0 for any other combination */
1287                 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1288                 break;
1289         }
1290
1291         return val & FB_GFX_FREQ_FUSE_MASK;
1292 }
1293
1294 static int chv_rps_rpe_freq(struct intel_rps *rps)
1295 {
1296         struct drm_i915_private *i915 = rps_to_i915(rps);
1297         u32 val;
1298
1299         val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1300         val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1301
1302         return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1303 }
1304
1305 static int chv_rps_guar_freq(struct intel_rps *rps)
1306 {
1307         struct drm_i915_private *i915 = rps_to_i915(rps);
1308         u32 val;
1309
1310         val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1311
1312         return val & FB_GFX_FREQ_FUSE_MASK;
1313 }
1314
1315 static u32 chv_rps_min_freq(struct intel_rps *rps)
1316 {
1317         struct drm_i915_private *i915 = rps_to_i915(rps);
1318         u32 val;
1319
1320         val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1321         val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1322
1323         return val & FB_GFX_FREQ_FUSE_MASK;
1324 }
1325
1326 static bool chv_rps_enable(struct intel_rps *rps)
1327 {
1328         struct intel_uncore *uncore = rps_to_uncore(rps);
1329         struct drm_i915_private *i915 = rps_to_i915(rps);
1330         u32 val;
1331
1332         /* 1: Program defaults and thresholds for RPS*/
1333         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1334         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1335         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1336         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1337         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1338
1339         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1340
1341         /* 2: Enable RPS */
1342         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1343                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1344                               GEN6_RP_MEDIA_IS_GFX |
1345                               GEN6_RP_ENABLE |
1346                               GEN6_RP_UP_BUSY_AVG |
1347                               GEN6_RP_DOWN_IDLE_AVG);
1348
1349         rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1350                           GEN6_PM_RP_DOWN_THRESHOLD |
1351                           GEN6_PM_RP_DOWN_TIMEOUT);
1352
1353         /* Setting Fixed Bias */
1354         vlv_punit_get(i915);
1355
1356         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1357         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1358
1359         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1360
1361         vlv_punit_put(i915);
1362
1363         /* RPS code assumes GPLL is used */
1364         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1365                       "GPLL not enabled\n");
1366
1367         drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1368                 str_yes_no(val & GPLLENABLE));
1369         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1370
1371         return rps_reset(rps);
1372 }
1373
1374 static int vlv_rps_guar_freq(struct intel_rps *rps)
1375 {
1376         struct drm_i915_private *i915 = rps_to_i915(rps);
1377         u32 val, rp1;
1378
1379         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1380
1381         rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1382         rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1383
1384         return rp1;
1385 }
1386
1387 static int vlv_rps_max_freq(struct intel_rps *rps)
1388 {
1389         struct drm_i915_private *i915 = rps_to_i915(rps);
1390         u32 val, rp0;
1391
1392         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1393
1394         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1395         /* Clamp to max */
1396         rp0 = min_t(u32, rp0, 0xea);
1397
1398         return rp0;
1399 }
1400
1401 static int vlv_rps_rpe_freq(struct intel_rps *rps)
1402 {
1403         struct drm_i915_private *i915 = rps_to_i915(rps);
1404         u32 val, rpe;
1405
1406         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1407         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1408         val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1409         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1410
1411         return rpe;
1412 }
1413
1414 static int vlv_rps_min_freq(struct intel_rps *rps)
1415 {
1416         struct drm_i915_private *i915 = rps_to_i915(rps);
1417         u32 val;
1418
1419         val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1420         /*
1421          * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1422          * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1423          * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1424          * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1425          * to make sure it matches what Punit accepts.
1426          */
1427         return max_t(u32, val, 0xc0);
1428 }
1429
1430 static bool vlv_rps_enable(struct intel_rps *rps)
1431 {
1432         struct intel_uncore *uncore = rps_to_uncore(rps);
1433         struct drm_i915_private *i915 = rps_to_i915(rps);
1434         u32 val;
1435
1436         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1437         intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1438         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1439         intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1440         intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1441
1442         intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1443
1444         intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1445                               GEN6_RP_MEDIA_TURBO |
1446                               GEN6_RP_MEDIA_HW_NORMAL_MODE |
1447                               GEN6_RP_MEDIA_IS_GFX |
1448                               GEN6_RP_ENABLE |
1449                               GEN6_RP_UP_BUSY_AVG |
1450                               GEN6_RP_DOWN_IDLE_CONT);
1451
1452         /* WaGsvRC0ResidencyMethod:vlv */
1453         rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1454
1455         vlv_punit_get(i915);
1456
1457         /* Setting Fixed Bias */
1458         val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1459         vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1460
1461         val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1462
1463         vlv_punit_put(i915);
1464
1465         /* RPS code assumes GPLL is used */
1466         drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1467                       "GPLL not enabled\n");
1468
1469         drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1470                 str_yes_no(val & GPLLENABLE));
1471         drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1472
1473         return rps_reset(rps);
1474 }
1475
1476 static unsigned long __ips_gfx_val(struct intel_ips *ips)
1477 {
1478         struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1479         struct intel_uncore *uncore = rps_to_uncore(rps);
1480         unsigned int t, state1, state2;
1481         u32 pxvid, ext_v;
1482         u64 corr, corr2;
1483
1484         lockdep_assert_held(&mchdev_lock);
1485
1486         pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1487         pxvid = (pxvid >> 24) & 0x7f;
1488         ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1489
1490         state1 = ext_v;
1491
1492         /* Revel in the empirically derived constants */
1493
1494         /* Correction factor in 1/100000 units */
1495         t = ips_mch_val(uncore);
1496         if (t > 80)
1497                 corr = t * 2349 + 135940;
1498         else if (t >= 50)
1499                 corr = t * 964 + 29317;
1500         else /* < 50 */
1501                 corr = t * 301 + 1004;
1502
1503         corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1504         corr2 = div_u64(corr, 100000) * ips->corr;
1505
1506         state2 = div_u64(corr2 * state1, 10000);
1507         state2 /= 100; /* convert to mW */
1508
1509         __gen5_ips_update(ips);
1510
1511         return ips->gfx_power + state2;
1512 }
1513
1514 static bool has_busy_stats(struct intel_rps *rps)
1515 {
1516         struct intel_engine_cs *engine;
1517         enum intel_engine_id id;
1518
1519         for_each_engine(engine, rps_to_gt(rps), id) {
1520                 if (!intel_engine_supports_stats(engine))
1521                         return false;
1522         }
1523
1524         return true;
1525 }
1526
1527 void intel_rps_enable(struct intel_rps *rps)
1528 {
1529         struct drm_i915_private *i915 = rps_to_i915(rps);
1530         struct intel_uncore *uncore = rps_to_uncore(rps);
1531         bool enabled = false;
1532
1533         if (!HAS_RPS(i915))
1534                 return;
1535
1536         if (rps_uses_slpc(rps))
1537                 return;
1538
1539         intel_gt_check_clock_frequency(rps_to_gt(rps));
1540
1541         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1542         if (rps->max_freq <= rps->min_freq)
1543                 /* leave disabled, no room for dynamic reclocking */;
1544         else if (IS_CHERRYVIEW(i915))
1545                 enabled = chv_rps_enable(rps);
1546         else if (IS_VALLEYVIEW(i915))
1547                 enabled = vlv_rps_enable(rps);
1548         else if (GRAPHICS_VER(i915) >= 9)
1549                 enabled = gen9_rps_enable(rps);
1550         else if (GRAPHICS_VER(i915) >= 8)
1551                 enabled = gen8_rps_enable(rps);
1552         else if (GRAPHICS_VER(i915) >= 6)
1553                 enabled = gen6_rps_enable(rps);
1554         else if (IS_IRONLAKE_M(i915))
1555                 enabled = gen5_rps_enable(rps);
1556         else
1557                 MISSING_CASE(GRAPHICS_VER(i915));
1558         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1559         if (!enabled)
1560                 return;
1561
1562         GT_TRACE(rps_to_gt(rps),
1563                  "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n",
1564                  rps->min_freq, rps->max_freq,
1565                  intel_gpu_freq(rps, rps->min_freq),
1566                  intel_gpu_freq(rps, rps->max_freq),
1567                  rps->power.up_threshold,
1568                  rps->power.down_threshold);
1569
1570         GEM_BUG_ON(rps->max_freq < rps->min_freq);
1571         GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1572
1573         GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1574         GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1575
1576         if (has_busy_stats(rps))
1577                 intel_rps_set_timer(rps);
1578         else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1579                 intel_rps_set_interrupts(rps);
1580         else
1581                 /* Ironlake currently uses intel_ips.ko */ {}
1582
1583         intel_rps_set_enabled(rps);
1584 }
1585
1586 static void gen6_rps_disable(struct intel_rps *rps)
1587 {
1588         set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1589 }
1590
1591 void intel_rps_disable(struct intel_rps *rps)
1592 {
1593         struct drm_i915_private *i915 = rps_to_i915(rps);
1594
1595         if (!intel_rps_is_enabled(rps))
1596                 return;
1597
1598         intel_rps_clear_enabled(rps);
1599         intel_rps_clear_interrupts(rps);
1600         intel_rps_clear_timer(rps);
1601
1602         if (GRAPHICS_VER(i915) >= 6)
1603                 gen6_rps_disable(rps);
1604         else if (IS_IRONLAKE_M(i915))
1605                 gen5_rps_disable(rps);
1606 }
1607
1608 static int byt_gpu_freq(struct intel_rps *rps, int val)
1609 {
1610         /*
1611          * N = val - 0xb7
1612          * Slow = Fast = GPLL ref * N
1613          */
1614         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1615 }
1616
1617 static int byt_freq_opcode(struct intel_rps *rps, int val)
1618 {
1619         return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1620 }
1621
1622 static int chv_gpu_freq(struct intel_rps *rps, int val)
1623 {
1624         /*
1625          * N = val / 2
1626          * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1627          */
1628         return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1629 }
1630
1631 static int chv_freq_opcode(struct intel_rps *rps, int val)
1632 {
1633         /* CHV needs even values */
1634         return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1635 }
1636
1637 int intel_gpu_freq(struct intel_rps *rps, int val)
1638 {
1639         struct drm_i915_private *i915 = rps_to_i915(rps);
1640
1641         if (GRAPHICS_VER(i915) >= 9)
1642                 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1643                                          GEN9_FREQ_SCALER);
1644         else if (IS_CHERRYVIEW(i915))
1645                 return chv_gpu_freq(rps, val);
1646         else if (IS_VALLEYVIEW(i915))
1647                 return byt_gpu_freq(rps, val);
1648         else if (GRAPHICS_VER(i915) >= 6)
1649                 return val * GT_FREQUENCY_MULTIPLIER;
1650         else
1651                 return val;
1652 }
1653
1654 int intel_freq_opcode(struct intel_rps *rps, int val)
1655 {
1656         struct drm_i915_private *i915 = rps_to_i915(rps);
1657
1658         if (GRAPHICS_VER(i915) >= 9)
1659                 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1660                                          GT_FREQUENCY_MULTIPLIER);
1661         else if (IS_CHERRYVIEW(i915))
1662                 return chv_freq_opcode(rps, val);
1663         else if (IS_VALLEYVIEW(i915))
1664                 return byt_freq_opcode(rps, val);
1665         else if (GRAPHICS_VER(i915) >= 6)
1666                 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1667         else
1668                 return val;
1669 }
1670
1671 static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1672 {
1673         struct drm_i915_private *i915 = rps_to_i915(rps);
1674
1675         rps->gpll_ref_freq =
1676                 vlv_get_cck_clock(i915, "GPLL ref",
1677                                   CCK_GPLL_CLOCK_CONTROL,
1678                                   i915->czclk_freq);
1679
1680         drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1681                 rps->gpll_ref_freq);
1682 }
1683
1684 static void vlv_rps_init(struct intel_rps *rps)
1685 {
1686         struct drm_i915_private *i915 = rps_to_i915(rps);
1687
1688         vlv_iosf_sb_get(i915,
1689                         BIT(VLV_IOSF_SB_PUNIT) |
1690                         BIT(VLV_IOSF_SB_NC) |
1691                         BIT(VLV_IOSF_SB_CCK));
1692
1693         vlv_init_gpll_ref_freq(rps);
1694
1695         rps->max_freq = vlv_rps_max_freq(rps);
1696         rps->rp0_freq = rps->max_freq;
1697         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1698                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1699
1700         rps->efficient_freq = vlv_rps_rpe_freq(rps);
1701         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1702                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1703
1704         rps->rp1_freq = vlv_rps_guar_freq(rps);
1705         drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1706                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1707
1708         rps->min_freq = vlv_rps_min_freq(rps);
1709         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1710                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1711
1712         vlv_iosf_sb_put(i915,
1713                         BIT(VLV_IOSF_SB_PUNIT) |
1714                         BIT(VLV_IOSF_SB_NC) |
1715                         BIT(VLV_IOSF_SB_CCK));
1716 }
1717
1718 static void chv_rps_init(struct intel_rps *rps)
1719 {
1720         struct drm_i915_private *i915 = rps_to_i915(rps);
1721
1722         vlv_iosf_sb_get(i915,
1723                         BIT(VLV_IOSF_SB_PUNIT) |
1724                         BIT(VLV_IOSF_SB_NC) |
1725                         BIT(VLV_IOSF_SB_CCK));
1726
1727         vlv_init_gpll_ref_freq(rps);
1728
1729         rps->max_freq = chv_rps_max_freq(rps);
1730         rps->rp0_freq = rps->max_freq;
1731         drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1732                 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1733
1734         rps->efficient_freq = chv_rps_rpe_freq(rps);
1735         drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1736                 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1737
1738         rps->rp1_freq = chv_rps_guar_freq(rps);
1739         drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1740                 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1741
1742         rps->min_freq = chv_rps_min_freq(rps);
1743         drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1744                 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1745
1746         vlv_iosf_sb_put(i915,
1747                         BIT(VLV_IOSF_SB_PUNIT) |
1748                         BIT(VLV_IOSF_SB_NC) |
1749                         BIT(VLV_IOSF_SB_CCK));
1750
1751         drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1752                                    rps->rp1_freq | rps->min_freq) & 1,
1753                       "Odd GPU freq values\n");
1754 }
1755
1756 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1757 {
1758         ei->ktime = ktime_get_raw();
1759         ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1760         ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1761 }
1762
1763 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1764 {
1765         struct intel_uncore *uncore = rps_to_uncore(rps);
1766         const struct intel_rps_ei *prev = &rps->ei;
1767         struct intel_rps_ei now;
1768         u32 events = 0;
1769
1770         if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1771                 return 0;
1772
1773         vlv_c0_read(uncore, &now);
1774
1775         if (prev->ktime) {
1776                 u64 time, c0;
1777                 u32 render, media;
1778
1779                 time = ktime_us_delta(now.ktime, prev->ktime);
1780
1781                 time *= rps_to_i915(rps)->czclk_freq;
1782
1783                 /* Workload can be split between render + media,
1784                  * e.g. SwapBuffers being blitted in X after being rendered in
1785                  * mesa. To account for this we need to combine both engines
1786                  * into our activity counter.
1787                  */
1788                 render = now.render_c0 - prev->render_c0;
1789                 media = now.media_c0 - prev->media_c0;
1790                 c0 = max(render, media);
1791                 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1792
1793                 if (c0 > time * rps->power.up_threshold)
1794                         events = GEN6_PM_RP_UP_THRESHOLD;
1795                 else if (c0 < time * rps->power.down_threshold)
1796                         events = GEN6_PM_RP_DOWN_THRESHOLD;
1797         }
1798
1799         rps->ei = now;
1800         return events;
1801 }
1802
1803 static void rps_work(struct work_struct *work)
1804 {
1805         struct intel_rps *rps = container_of(work, typeof(*rps), work);
1806         struct intel_gt *gt = rps_to_gt(rps);
1807         struct drm_i915_private *i915 = rps_to_i915(rps);
1808         bool client_boost = false;
1809         int new_freq, adj, min, max;
1810         u32 pm_iir = 0;
1811
1812         spin_lock_irq(gt->irq_lock);
1813         pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1814         client_boost = atomic_read(&rps->num_waiters);
1815         spin_unlock_irq(gt->irq_lock);
1816
1817         /* Make sure we didn't queue anything we're not going to process. */
1818         if (!pm_iir && !client_boost)
1819                 goto out;
1820
1821         mutex_lock(&rps->lock);
1822         if (!intel_rps_is_active(rps)) {
1823                 mutex_unlock(&rps->lock);
1824                 return;
1825         }
1826
1827         pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1828
1829         adj = rps->last_adj;
1830         new_freq = rps->cur_freq;
1831         min = rps->min_freq_softlimit;
1832         max = rps->max_freq_softlimit;
1833         if (client_boost)
1834                 max = rps->max_freq;
1835
1836         GT_TRACE(gt,
1837                  "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1838                  pm_iir, str_yes_no(client_boost),
1839                  adj, new_freq, min, max);
1840
1841         if (client_boost && new_freq < rps->boost_freq) {
1842                 new_freq = rps->boost_freq;
1843                 adj = 0;
1844         } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1845                 if (adj > 0)
1846                         adj *= 2;
1847                 else /* CHV needs even encode values */
1848                         adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1849
1850                 if (new_freq >= rps->max_freq_softlimit)
1851                         adj = 0;
1852         } else if (client_boost) {
1853                 adj = 0;
1854         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1855                 if (rps->cur_freq > rps->efficient_freq)
1856                         new_freq = rps->efficient_freq;
1857                 else if (rps->cur_freq > rps->min_freq_softlimit)
1858                         new_freq = rps->min_freq_softlimit;
1859                 adj = 0;
1860         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1861                 if (adj < 0)
1862                         adj *= 2;
1863                 else /* CHV needs even encode values */
1864                         adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1865
1866                 if (new_freq <= rps->min_freq_softlimit)
1867                         adj = 0;
1868         } else { /* unknown event */
1869                 adj = 0;
1870         }
1871
1872         /*
1873          * sysfs frequency limits may have snuck in while
1874          * servicing the interrupt
1875          */
1876         new_freq += adj;
1877         new_freq = clamp_t(int, new_freq, min, max);
1878
1879         if (intel_rps_set(rps, new_freq)) {
1880                 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1881                 adj = 0;
1882         }
1883         rps->last_adj = adj;
1884
1885         mutex_unlock(&rps->lock);
1886
1887 out:
1888         spin_lock_irq(gt->irq_lock);
1889         gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1890         spin_unlock_irq(gt->irq_lock);
1891 }
1892
1893 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1894 {
1895         struct intel_gt *gt = rps_to_gt(rps);
1896         const u32 events = rps->pm_events & pm_iir;
1897
1898         lockdep_assert_held(gt->irq_lock);
1899
1900         if (unlikely(!events))
1901                 return;
1902
1903         GT_TRACE(gt, "irq events:%x\n", events);
1904
1905         gen6_gt_pm_mask_irq(gt, events);
1906
1907         rps->pm_iir |= events;
1908         queue_work(gt->i915->unordered_wq, &rps->work);
1909 }
1910
1911 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1912 {
1913         struct intel_gt *gt = rps_to_gt(rps);
1914         u32 events;
1915
1916         events = pm_iir & rps->pm_events;
1917         if (events) {
1918                 spin_lock(gt->irq_lock);
1919
1920                 GT_TRACE(gt, "irq events:%x\n", events);
1921
1922                 gen6_gt_pm_mask_irq(gt, events);
1923                 rps->pm_iir |= events;
1924
1925                 queue_work(gt->i915->unordered_wq, &rps->work);
1926                 spin_unlock(gt->irq_lock);
1927         }
1928
1929         if (GRAPHICS_VER(gt->i915) >= 8)
1930                 return;
1931
1932         if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1933                 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1934
1935         if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1936                 drm_dbg(&rps_to_i915(rps)->drm,
1937                         "Command parser error, pm_iir 0x%08x\n", pm_iir);
1938 }
1939
1940 void gen5_rps_irq_handler(struct intel_rps *rps)
1941 {
1942         struct intel_uncore *uncore = rps_to_uncore(rps);
1943         u32 busy_up, busy_down, max_avg, min_avg;
1944         u8 new_freq;
1945
1946         spin_lock(&mchdev_lock);
1947
1948         intel_uncore_write16(uncore,
1949                              MEMINTRSTS,
1950                              intel_uncore_read(uncore, MEMINTRSTS));
1951
1952         intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1953         busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1954         busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1955         max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1956         min_avg = intel_uncore_read(uncore, RCBMINAVG);
1957
1958         /* Handle RCS change request from hw */
1959         new_freq = rps->cur_freq;
1960         if (busy_up > max_avg)
1961                 new_freq++;
1962         else if (busy_down < min_avg)
1963                 new_freq--;
1964         new_freq = clamp(new_freq,
1965                          rps->min_freq_softlimit,
1966                          rps->max_freq_softlimit);
1967
1968         if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1969                 rps->cur_freq = new_freq;
1970
1971         spin_unlock(&mchdev_lock);
1972 }
1973
1974 void intel_rps_init_early(struct intel_rps *rps)
1975 {
1976         mutex_init(&rps->lock);
1977         mutex_init(&rps->power.mutex);
1978
1979         INIT_WORK(&rps->work, rps_work);
1980         timer_setup(&rps->timer, rps_timer, 0);
1981
1982         atomic_set(&rps->num_waiters, 0);
1983 }
1984
1985 void intel_rps_init(struct intel_rps *rps)
1986 {
1987         struct drm_i915_private *i915 = rps_to_i915(rps);
1988
1989         if (rps_uses_slpc(rps))
1990                 return;
1991
1992         if (IS_CHERRYVIEW(i915))
1993                 chv_rps_init(rps);
1994         else if (IS_VALLEYVIEW(i915))
1995                 vlv_rps_init(rps);
1996         else if (GRAPHICS_VER(i915) >= 6)
1997                 gen6_rps_init(rps);
1998         else if (IS_IRONLAKE_M(i915))
1999                 gen5_rps_init(rps);
2000
2001         /* Derive initial user preferences/limits from the hardware limits */
2002         rps->max_freq_softlimit = rps->max_freq;
2003         rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
2004         rps->min_freq_softlimit = rps->min_freq;
2005         rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
2006
2007         /* After setting max-softlimit, find the overclock max freq */
2008         if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
2009                 u32 params = 0;
2010
2011                 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, &params, NULL);
2012                 if (params & BIT(31)) { /* OC supported */
2013                         drm_dbg(&i915->drm,
2014                                 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2015                                 (rps->max_freq & 0xff) * 50,
2016                                 (params & 0xff) * 50);
2017                         rps->max_freq = params & 0xff;
2018                 }
2019         }
2020
2021         /* Set default thresholds in % */
2022         rps->power.up_threshold = 95;
2023         rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold;
2024         rps->power.down_threshold = 85;
2025         rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold;
2026
2027         /* Finally allow us to boost to max by default */
2028         rps->boost_freq = rps->max_freq;
2029         rps->idle_freq = rps->min_freq;
2030
2031         /* Start in the middle, from here we will autotune based on workload */
2032         rps->cur_freq = rps->efficient_freq;
2033
2034         rps->pm_intrmsk_mbz = 0;
2035
2036         /*
2037          * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2038          * if GEN6_PM_UP_EI_EXPIRED is masked.
2039          *
2040          * TODO: verify if this can be reproduced on VLV,CHV.
2041          */
2042         if (GRAPHICS_VER(i915) <= 7)
2043                 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2044
2045         if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2046                 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2047
2048         /* GuC needs ARAT expired interrupt unmasked */
2049         if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2050                 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2051 }
2052
2053 void intel_rps_sanitize(struct intel_rps *rps)
2054 {
2055         if (rps_uses_slpc(rps))
2056                 return;
2057
2058         if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2059                 rps_disable_interrupts(rps);
2060 }
2061
2062 u32 intel_rps_read_rpstat(struct intel_rps *rps)
2063 {
2064         struct drm_i915_private *i915 = rps_to_i915(rps);
2065         i915_reg_t rpstat;
2066
2067         rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1;
2068
2069         return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat);
2070 }
2071
2072 static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2073 {
2074         struct drm_i915_private *i915 = rps_to_i915(rps);
2075         u32 cagf;
2076
2077         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2078                 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat);
2079         else if (GRAPHICS_VER(i915) >= 12)
2080                 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat);
2081         else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2082                 cagf = REG_FIELD_GET(RPE_MASK, rpstat);
2083         else if (GRAPHICS_VER(i915) >= 9)
2084                 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat);
2085         else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2086                 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat);
2087         else if (GRAPHICS_VER(i915) >= 6)
2088                 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat);
2089         else
2090                 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat));
2091
2092         return cagf;
2093 }
2094
2095 static u32 __read_cagf(struct intel_rps *rps, bool take_fw)
2096 {
2097         struct drm_i915_private *i915 = rps_to_i915(rps);
2098         struct intel_uncore *uncore = rps_to_uncore(rps);
2099         i915_reg_t r = INVALID_MMIO_REG;
2100         u32 freq;
2101
2102         /*
2103          * For Gen12+ reading freq from HW does not need a forcewake and
2104          * registers will return 0 freq when GT is in RC6
2105          */
2106         if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2107                 r = MTL_MIRROR_TARGET_WP1;
2108         } else if (GRAPHICS_VER(i915) >= 12) {
2109                 r = GEN12_RPSTAT1;
2110         } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2111                 vlv_punit_get(i915);
2112                 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2113                 vlv_punit_put(i915);
2114         } else if (GRAPHICS_VER(i915) >= 6) {
2115                 r = GEN6_RPSTAT1;
2116         } else {
2117                 r = MEMSTAT_ILK;
2118         }
2119
2120         if (i915_mmio_reg_valid(r))
2121                 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r);
2122
2123         return intel_rps_get_cagf(rps, freq);
2124 }
2125
2126 static u32 read_cagf(struct intel_rps *rps)
2127 {
2128         return __read_cagf(rps, true);
2129 }
2130
2131 u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2132 {
2133         struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2134         intel_wakeref_t wakeref;
2135         u32 freq = 0;
2136
2137         with_intel_runtime_pm_if_in_use(rpm, wakeref)
2138                 freq = intel_gpu_freq(rps, read_cagf(rps));
2139
2140         return freq;
2141 }
2142
2143 u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)
2144 {
2145         return intel_gpu_freq(rps, __read_cagf(rps, false));
2146 }
2147
2148 static u32 intel_rps_read_punit_req(struct intel_rps *rps)
2149 {
2150         struct intel_uncore *uncore = rps_to_uncore(rps);
2151         struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2152         intel_wakeref_t wakeref;
2153         u32 freq = 0;
2154
2155         with_intel_runtime_pm_if_in_use(rpm, wakeref)
2156                 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2157
2158         return freq;
2159 }
2160
2161 static u32 intel_rps_get_req(u32 pureq)
2162 {
2163         u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2164
2165         return req;
2166 }
2167
2168 u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2169 {
2170         u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2171
2172         return intel_gpu_freq(rps, freq);
2173 }
2174
2175 u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2176 {
2177         if (rps_uses_slpc(rps))
2178                 return intel_rps_read_punit_req_frequency(rps);
2179         else
2180                 return intel_gpu_freq(rps, rps->cur_freq);
2181 }
2182
2183 u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2184 {
2185         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2186
2187         if (rps_uses_slpc(rps))
2188                 return slpc->max_freq_softlimit;
2189         else
2190                 return intel_gpu_freq(rps, rps->max_freq_softlimit);
2191 }
2192
2193 /**
2194  * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2195  * @rps: the intel_rps structure
2196  *
2197  * Returns the max frequency in a raw format. In newer platforms raw is in
2198  * units of 50 MHz.
2199  */
2200 u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2201 {
2202         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2203         u32 freq;
2204
2205         if (rps_uses_slpc(rps)) {
2206                 return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2207                                          GT_FREQUENCY_MULTIPLIER);
2208         } else {
2209                 freq = rps->max_freq;
2210                 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2211                         /* Convert GT frequency to 50 MHz units */
2212                         freq /= GEN9_FREQ_SCALER;
2213                 }
2214                 return freq;
2215         }
2216 }
2217
2218 u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2219 {
2220         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2221
2222         if (rps_uses_slpc(rps))
2223                 return slpc->rp0_freq;
2224         else
2225                 return intel_gpu_freq(rps, rps->rp0_freq);
2226 }
2227
2228 u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2229 {
2230         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2231
2232         if (rps_uses_slpc(rps))
2233                 return slpc->rp1_freq;
2234         else
2235                 return intel_gpu_freq(rps, rps->rp1_freq);
2236 }
2237
2238 u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2239 {
2240         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2241
2242         if (rps_uses_slpc(rps))
2243                 return slpc->min_freq;
2244         else
2245                 return intel_gpu_freq(rps, rps->min_freq);
2246 }
2247
2248 static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2249 {
2250         struct intel_gt *gt = rps_to_gt(rps);
2251         struct drm_i915_private *i915 = gt->i915;
2252         struct intel_uncore *uncore = gt->uncore;
2253         struct intel_rps_freq_caps caps;
2254         u32 rp_state_limits;
2255         u32 gt_perf_status;
2256         u32 rpmodectl, rpinclimit, rpdeclimit;
2257         u32 rpstat, cagf, reqf;
2258         u32 rpcurupei, rpcurup, rpprevup;
2259         u32 rpcurdownei, rpcurdown, rpprevdown;
2260         u32 rpupei, rpupt, rpdownei, rpdownt;
2261         u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
2262
2263         rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS);
2264         gen6_rps_get_freq_caps(rps, &caps);
2265         if (IS_GEN9_LP(i915))
2266                 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS);
2267         else
2268                 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS);
2269
2270         /* RPSTAT1 is in the GT power well */
2271         intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
2272
2273         reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2274         if (GRAPHICS_VER(i915) >= 9) {
2275                 reqf >>= 23;
2276         } else {
2277                 reqf &= ~GEN6_TURBO_DISABLE;
2278                 if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2279                         reqf >>= 24;
2280                 else
2281                         reqf >>= 25;
2282         }
2283         reqf = intel_gpu_freq(rps, reqf);
2284
2285         rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL);
2286         rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2287         rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2288
2289         rpstat = intel_rps_read_rpstat(rps);
2290         rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK;
2291         rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK;
2292         rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK;
2293         rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
2294         rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
2295         rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
2296
2297         rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI);
2298         rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2299
2300         rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI);
2301         rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2302
2303         cagf = intel_rps_read_actual_frequency(rps);
2304
2305         intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
2306
2307         if (GRAPHICS_VER(i915) >= 11) {
2308                 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE);
2309                 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK);
2310                 /*
2311                  * The equivalent to the PM ISR & IIR cannot be read
2312                  * without affecting the current state of the system
2313                  */
2314                 pm_isr = 0;
2315                 pm_iir = 0;
2316         } else if (GRAPHICS_VER(i915) >= 8) {
2317                 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2));
2318                 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2));
2319                 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2));
2320                 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2));
2321         } else {
2322                 pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
2323                 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
2324                 pm_isr = intel_uncore_read(uncore, GEN6_PMISR);
2325                 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR);
2326         }
2327         pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2328
2329         drm_printf(p, "Video Turbo Mode: %s\n",
2330                    str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO));
2331         drm_printf(p, "HW control enabled: %s\n",
2332                    str_yes_no(rpmodectl & GEN6_RP_ENABLE));
2333         drm_printf(p, "SW control enabled: %s\n",
2334                    str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE));
2335
2336         drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n",
2337                    pm_ier, pm_imr, pm_mask);
2338         if (GRAPHICS_VER(i915) <= 10)
2339                 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n",
2340                            pm_isr, pm_iir);
2341         drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2342                    rps->pm_intrmsk_mbz);
2343         drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
2344         drm_printf(p, "Render p-state ratio: %d\n",
2345                    (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8);
2346         drm_printf(p, "Render p-state VID: %d\n",
2347                    gt_perf_status & 0xff);
2348         drm_printf(p, "Render p-state limit: %d\n",
2349                    rp_state_limits & 0xff);
2350         drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat);
2351         drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl);
2352         drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit);
2353         drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit);
2354         drm_printf(p, "RPNSWREQ: %dMHz\n", reqf);
2355         drm_printf(p, "CAGF: %dMHz\n", cagf);
2356         drm_printf(p, "RP CUR UP EI: %d (%lldns)\n",
2357                    rpcurupei,
2358                    intel_gt_pm_interval_to_ns(gt, rpcurupei));
2359         drm_printf(p, "RP CUR UP: %d (%lldns)\n",
2360                    rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup));
2361         drm_printf(p, "RP PREV UP: %d (%lldns)\n",
2362                    rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup));
2363         drm_printf(p, "Up threshold: %d%%\n",
2364                    rps->power.up_threshold);
2365         drm_printf(p, "RP UP EI: %d (%lldns)\n",
2366                    rpupei, intel_gt_pm_interval_to_ns(gt, rpupei));
2367         drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n",
2368                    rpupt, intel_gt_pm_interval_to_ns(gt, rpupt));
2369
2370         drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n",
2371                    rpcurdownei,
2372                    intel_gt_pm_interval_to_ns(gt, rpcurdownei));
2373         drm_printf(p, "RP CUR DOWN: %d (%lldns)\n",
2374                    rpcurdown,
2375                    intel_gt_pm_interval_to_ns(gt, rpcurdown));
2376         drm_printf(p, "RP PREV DOWN: %d (%lldns)\n",
2377                    rpprevdown,
2378                    intel_gt_pm_interval_to_ns(gt, rpprevdown));
2379         drm_printf(p, "Down threshold: %d%%\n",
2380                    rps->power.down_threshold);
2381         drm_printf(p, "RP DOWN EI: %d (%lldns)\n",
2382                    rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei));
2383         drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n",
2384                    rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt));
2385
2386         drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2387                    intel_gpu_freq(rps, caps.min_freq));
2388         drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2389                    intel_gpu_freq(rps, caps.rp1_freq));
2390         drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2391                    intel_gpu_freq(rps, caps.rp0_freq));
2392         drm_printf(p, "Max overclocked frequency: %dMHz\n",
2393                    intel_gpu_freq(rps, rps->max_freq));
2394
2395         drm_printf(p, "Current freq: %d MHz\n",
2396                    intel_gpu_freq(rps, rps->cur_freq));
2397         drm_printf(p, "Actual freq: %d MHz\n", cagf);
2398         drm_printf(p, "Idle freq: %d MHz\n",
2399                    intel_gpu_freq(rps, rps->idle_freq));
2400         drm_printf(p, "Min freq: %d MHz\n",
2401                    intel_gpu_freq(rps, rps->min_freq));
2402         drm_printf(p, "Boost freq: %d MHz\n",
2403                    intel_gpu_freq(rps, rps->boost_freq));
2404         drm_printf(p, "Max freq: %d MHz\n",
2405                    intel_gpu_freq(rps, rps->max_freq));
2406         drm_printf(p,
2407                    "efficient (RPe) frequency: %d MHz\n",
2408                    intel_gpu_freq(rps, rps->efficient_freq));
2409 }
2410
2411 static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2412 {
2413         struct intel_gt *gt = rps_to_gt(rps);
2414         struct intel_uncore *uncore = gt->uncore;
2415         struct intel_rps_freq_caps caps;
2416         u32 pm_mask;
2417
2418         gen6_rps_get_freq_caps(rps, &caps);
2419         pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2420
2421         drm_printf(p, "PM MASK=0x%08x\n", pm_mask);
2422         drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2423                    rps->pm_intrmsk_mbz);
2424         drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps));
2425         drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps));
2426         drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2427                    intel_gpu_freq(rps, caps.min_freq));
2428         drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2429                    intel_gpu_freq(rps, caps.rp1_freq));
2430         drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2431                    intel_gpu_freq(rps, caps.rp0_freq));
2432         drm_printf(p, "Current freq: %d MHz\n",
2433                    intel_rps_get_requested_frequency(rps));
2434         drm_printf(p, "Actual freq: %d MHz\n",
2435                    intel_rps_read_actual_frequency(rps));
2436         drm_printf(p, "Min freq: %d MHz\n",
2437                    intel_rps_get_min_frequency(rps));
2438         drm_printf(p, "Boost freq: %d MHz\n",
2439                    intel_rps_get_boost_frequency(rps));
2440         drm_printf(p, "Max freq: %d MHz\n",
2441                    intel_rps_get_max_frequency(rps));
2442         drm_printf(p,
2443                    "efficient (RPe) frequency: %d MHz\n",
2444                    intel_gpu_freq(rps, caps.rp1_freq));
2445 }
2446
2447 void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2448 {
2449         if (rps_uses_slpc(rps))
2450                 return slpc_frequency_dump(rps, p);
2451         else
2452                 return rps_frequency_dump(rps, p);
2453 }
2454
2455 static int set_max_freq(struct intel_rps *rps, u32 val)
2456 {
2457         struct drm_i915_private *i915 = rps_to_i915(rps);
2458         int ret = 0;
2459
2460         mutex_lock(&rps->lock);
2461
2462         val = intel_freq_opcode(rps, val);
2463         if (val < rps->min_freq ||
2464             val > rps->max_freq ||
2465             val < rps->min_freq_softlimit) {
2466                 ret = -EINVAL;
2467                 goto unlock;
2468         }
2469
2470         if (val > rps->rp0_freq)
2471                 drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2472                         intel_gpu_freq(rps, val));
2473
2474         rps->max_freq_softlimit = val;
2475
2476         val = clamp_t(int, rps->cur_freq,
2477                       rps->min_freq_softlimit,
2478                       rps->max_freq_softlimit);
2479
2480         /*
2481          * We still need *_set_rps to process the new max_delay and
2482          * update the interrupt limits and PMINTRMSK even though
2483          * frequency request may be unchanged.
2484          */
2485         intel_rps_set(rps, val);
2486
2487 unlock:
2488         mutex_unlock(&rps->lock);
2489
2490         return ret;
2491 }
2492
2493 int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2494 {
2495         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2496
2497         if (rps_uses_slpc(rps))
2498                 return intel_guc_slpc_set_max_freq(slpc, val);
2499         else
2500                 return set_max_freq(rps, val);
2501 }
2502
2503 u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2504 {
2505         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2506
2507         if (rps_uses_slpc(rps))
2508                 return slpc->min_freq_softlimit;
2509         else
2510                 return intel_gpu_freq(rps, rps->min_freq_softlimit);
2511 }
2512
2513 /**
2514  * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2515  * @rps: the intel_rps structure
2516  *
2517  * Returns the min frequency in a raw format. In newer platforms raw is in
2518  * units of 50 MHz.
2519  */
2520 u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2521 {
2522         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2523         u32 freq;
2524
2525         if (rps_uses_slpc(rps)) {
2526                 return DIV_ROUND_CLOSEST(slpc->min_freq,
2527                                          GT_FREQUENCY_MULTIPLIER);
2528         } else {
2529                 freq = rps->min_freq;
2530                 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2531                         /* Convert GT frequency to 50 MHz units */
2532                         freq /= GEN9_FREQ_SCALER;
2533                 }
2534                 return freq;
2535         }
2536 }
2537
2538 static int set_min_freq(struct intel_rps *rps, u32 val)
2539 {
2540         int ret = 0;
2541
2542         mutex_lock(&rps->lock);
2543
2544         val = intel_freq_opcode(rps, val);
2545         if (val < rps->min_freq ||
2546             val > rps->max_freq ||
2547             val > rps->max_freq_softlimit) {
2548                 ret = -EINVAL;
2549                 goto unlock;
2550         }
2551
2552         rps->min_freq_softlimit = val;
2553
2554         val = clamp_t(int, rps->cur_freq,
2555                       rps->min_freq_softlimit,
2556                       rps->max_freq_softlimit);
2557
2558         /*
2559          * We still need *_set_rps to process the new min_delay and
2560          * update the interrupt limits and PMINTRMSK even though
2561          * frequency request may be unchanged.
2562          */
2563         intel_rps_set(rps, val);
2564
2565 unlock:
2566         mutex_unlock(&rps->lock);
2567
2568         return ret;
2569 }
2570
2571 int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2572 {
2573         struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2574
2575         if (rps_uses_slpc(rps))
2576                 return intel_guc_slpc_set_min_freq(slpc, val);
2577         else
2578                 return set_min_freq(rps, val);
2579 }
2580
2581 u8 intel_rps_get_up_threshold(struct intel_rps *rps)
2582 {
2583         return rps->power.up_threshold;
2584 }
2585
2586 static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val)
2587 {
2588         int ret;
2589
2590         if (val > 100)
2591                 return -EINVAL;
2592
2593         ret = mutex_lock_interruptible(&rps->lock);
2594         if (ret)
2595                 return ret;
2596
2597         if (*threshold == val)
2598                 goto out_unlock;
2599
2600         *threshold = val;
2601
2602         /* Force reset. */
2603         rps->last_freq = -1;
2604         mutex_lock(&rps->power.mutex);
2605         rps->power.mode = -1;
2606         mutex_unlock(&rps->power.mutex);
2607
2608         intel_rps_set(rps, clamp(rps->cur_freq,
2609                                  rps->min_freq_softlimit,
2610                                  rps->max_freq_softlimit));
2611
2612 out_unlock:
2613         mutex_unlock(&rps->lock);
2614
2615         return ret;
2616 }
2617
2618 int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold)
2619 {
2620         return rps_set_threshold(rps, &rps->power.up_threshold, threshold);
2621 }
2622
2623 u8 intel_rps_get_down_threshold(struct intel_rps *rps)
2624 {
2625         return rps->power.down_threshold;
2626 }
2627
2628 int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold)
2629 {
2630         return rps_set_threshold(rps, &rps->power.down_threshold, threshold);
2631 }
2632
2633 static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2634 {
2635         struct intel_uncore *uncore = rps_to_uncore(rps);
2636         u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2637
2638         /* Allow punit to process software requests */
2639         intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2640 }
2641
2642 void intel_rps_raise_unslice(struct intel_rps *rps)
2643 {
2644         struct intel_uncore *uncore = rps_to_uncore(rps);
2645
2646         mutex_lock(&rps->lock);
2647
2648         if (rps_uses_slpc(rps)) {
2649                 /* RP limits have not been initialized yet for SLPC path */
2650                 struct intel_rps_freq_caps caps;
2651
2652                 gen6_rps_get_freq_caps(rps, &caps);
2653
2654                 intel_rps_set_manual(rps, true);
2655                 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2656                                    ((caps.rp0_freq <<
2657                                    GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2658                                    GEN9_IGNORE_SLICE_RATIO));
2659                 intel_rps_set_manual(rps, false);
2660         } else {
2661                 intel_rps_set(rps, rps->rp0_freq);
2662         }
2663
2664         mutex_unlock(&rps->lock);
2665 }
2666
2667 void intel_rps_lower_unslice(struct intel_rps *rps)
2668 {
2669         struct intel_uncore *uncore = rps_to_uncore(rps);
2670
2671         mutex_lock(&rps->lock);
2672
2673         if (rps_uses_slpc(rps)) {
2674                 /* RP limits have not been initialized yet for SLPC path */
2675                 struct intel_rps_freq_caps caps;
2676
2677                 gen6_rps_get_freq_caps(rps, &caps);
2678
2679                 intel_rps_set_manual(rps, true);
2680                 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2681                                    ((caps.min_freq <<
2682                                    GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2683                                    GEN9_IGNORE_SLICE_RATIO));
2684                 intel_rps_set_manual(rps, false);
2685         } else {
2686                 intel_rps_set(rps, rps->min_freq);
2687         }
2688
2689         mutex_unlock(&rps->lock);
2690 }
2691
2692 static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2693 {
2694         struct intel_gt *gt = rps_to_gt(rps);
2695         intel_wakeref_t wakeref;
2696         u32 val;
2697
2698         with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2699                 val = intel_uncore_read(gt->uncore, reg32);
2700
2701         return val;
2702 }
2703
2704 bool rps_read_mask_mmio(struct intel_rps *rps,
2705                         i915_reg_t reg32, u32 mask)
2706 {
2707         return rps_read_mmio(rps, reg32) & mask;
2708 }
2709
2710 /* External interface for intel_ips.ko */
2711
2712 static struct drm_i915_private __rcu *ips_mchdev;
2713
2714 /*
2715  * Tells the intel_ips driver that the i915 driver is now loaded, if
2716  * IPS got loaded first.
2717  *
2718  * This awkward dance is so that neither module has to depend on the
2719  * other in order for IPS to do the appropriate communication of
2720  * GPU turbo limits to i915.
2721  */
2722 static void
2723 ips_ping_for_i915_load(void)
2724 {
2725         void (*link)(void);
2726
2727         link = symbol_get(ips_link_to_i915_driver);
2728         if (link) {
2729                 link();
2730                 symbol_put(ips_link_to_i915_driver);
2731         }
2732 }
2733
2734 void intel_rps_driver_register(struct intel_rps *rps)
2735 {
2736         struct intel_gt *gt = rps_to_gt(rps);
2737
2738         /*
2739          * We only register the i915 ips part with intel-ips once everything is
2740          * set up, to avoid intel-ips sneaking in and reading bogus values.
2741          */
2742         if (GRAPHICS_VER(gt->i915) == 5) {
2743                 GEM_BUG_ON(ips_mchdev);
2744                 rcu_assign_pointer(ips_mchdev, gt->i915);
2745                 ips_ping_for_i915_load();
2746         }
2747 }
2748
2749 void intel_rps_driver_unregister(struct intel_rps *rps)
2750 {
2751         if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2752                 rcu_assign_pointer(ips_mchdev, NULL);
2753 }
2754
2755 static struct drm_i915_private *mchdev_get(void)
2756 {
2757         struct drm_i915_private *i915;
2758
2759         rcu_read_lock();
2760         i915 = rcu_dereference(ips_mchdev);
2761         if (i915 && !kref_get_unless_zero(&i915->drm.ref))
2762                 i915 = NULL;
2763         rcu_read_unlock();
2764
2765         return i915;
2766 }
2767
2768 /**
2769  * i915_read_mch_val - return value for IPS use
2770  *
2771  * Calculate and return a value for the IPS driver to use when deciding whether
2772  * we have thermal and power headroom to increase CPU or GPU power budget.
2773  */
2774 unsigned long i915_read_mch_val(void)
2775 {
2776         struct drm_i915_private *i915;
2777         unsigned long chipset_val = 0;
2778         unsigned long graphics_val = 0;
2779         intel_wakeref_t wakeref;
2780
2781         i915 = mchdev_get();
2782         if (!i915)
2783                 return 0;
2784
2785         with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2786                 struct intel_ips *ips = &to_gt(i915)->rps.ips;
2787
2788                 spin_lock_irq(&mchdev_lock);
2789                 chipset_val = __ips_chipset_val(ips);
2790                 graphics_val = __ips_gfx_val(ips);
2791                 spin_unlock_irq(&mchdev_lock);
2792         }
2793
2794         drm_dev_put(&i915->drm);
2795         return chipset_val + graphics_val;
2796 }
2797 EXPORT_SYMBOL_GPL(i915_read_mch_val);
2798
2799 /**
2800  * i915_gpu_raise - raise GPU frequency limit
2801  *
2802  * Raise the limit; IPS indicates we have thermal headroom.
2803  */
2804 bool i915_gpu_raise(void)
2805 {
2806         struct drm_i915_private *i915;
2807         struct intel_rps *rps;
2808
2809         i915 = mchdev_get();
2810         if (!i915)
2811                 return false;
2812
2813         rps = &to_gt(i915)->rps;
2814
2815         spin_lock_irq(&mchdev_lock);
2816         if (rps->max_freq_softlimit < rps->max_freq)
2817                 rps->max_freq_softlimit++;
2818         spin_unlock_irq(&mchdev_lock);
2819
2820         drm_dev_put(&i915->drm);
2821         return true;
2822 }
2823 EXPORT_SYMBOL_GPL(i915_gpu_raise);
2824
2825 /**
2826  * i915_gpu_lower - lower GPU frequency limit
2827  *
2828  * IPS indicates we're close to a thermal limit, so throttle back the GPU
2829  * frequency maximum.
2830  */
2831 bool i915_gpu_lower(void)
2832 {
2833         struct drm_i915_private *i915;
2834         struct intel_rps *rps;
2835
2836         i915 = mchdev_get();
2837         if (!i915)
2838                 return false;
2839
2840         rps = &to_gt(i915)->rps;
2841
2842         spin_lock_irq(&mchdev_lock);
2843         if (rps->max_freq_softlimit > rps->min_freq)
2844                 rps->max_freq_softlimit--;
2845         spin_unlock_irq(&mchdev_lock);
2846
2847         drm_dev_put(&i915->drm);
2848         return true;
2849 }
2850 EXPORT_SYMBOL_GPL(i915_gpu_lower);
2851
2852 /**
2853  * i915_gpu_busy - indicate GPU business to IPS
2854  *
2855  * Tell the IPS driver whether or not the GPU is busy.
2856  */
2857 bool i915_gpu_busy(void)
2858 {
2859         struct drm_i915_private *i915;
2860         bool ret;
2861
2862         i915 = mchdev_get();
2863         if (!i915)
2864                 return false;
2865
2866         ret = to_gt(i915)->awake;
2867
2868         drm_dev_put(&i915->drm);
2869         return ret;
2870 }
2871 EXPORT_SYMBOL_GPL(i915_gpu_busy);
2872
2873 /**
2874  * i915_gpu_turbo_disable - disable graphics turbo
2875  *
2876  * Disable graphics turbo by resetting the max frequency and setting the
2877  * current frequency to the default.
2878  */
2879 bool i915_gpu_turbo_disable(void)
2880 {
2881         struct drm_i915_private *i915;
2882         struct intel_rps *rps;
2883         bool ret;
2884
2885         i915 = mchdev_get();
2886         if (!i915)
2887                 return false;
2888
2889         rps = &to_gt(i915)->rps;
2890
2891         spin_lock_irq(&mchdev_lock);
2892         rps->max_freq_softlimit = rps->min_freq;
2893         ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
2894         spin_unlock_irq(&mchdev_lock);
2895
2896         drm_dev_put(&i915->drm);
2897         return ret;
2898 }
2899 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2900
2901 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2902 #include "selftest_rps.c"
2903 #include "selftest_slpc.c"
2904 #endif