projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'm68k-for-v6.4-tag1' of git://git.kernel.org/pub/scm/linux/kernel/git/geert...
[linux-block.git] / drivers / gpu / drm / i915 / gt / intel_rps.c
CommitLineData
24f90d66 1// SPDX-License-Identifier: MIT
3e7abf81 2/*
3e7abf81
AS		 3 * Copyright © 2019 Intel Corporation
4 */
5
01fabda8
LDM		 6#include <linux/string_helpers.h>
7
83d2bdb6
JN		 8#include <drm/i915_drm.h>
9
acc855d3 10#include "display/intel_display.h"
3e7abf81 11#include "i915_drv.h"
80dfdeb7 12#include "i915_irq.h"
b3786b29 13#include "intel_breadcrumbs.h"
3e7abf81 14#include "intel_gt.h"
9c878557 15#include "intel_gt_clock_utils.h"
3e7abf81
AS		 16#include "intel_gt_irq.h"
17#include "intel_gt_pm_irq.h"
0d6419e9 18#include "intel_gt_regs.h"
e30e6c7b 19#include "intel_mchbar_regs.h"
4dd4375b 20#include "intel_pcode.h"
3e7abf81 21#include "intel_rps.h"
1eecf31e 22#include "vlv_sideband.h"
3e7abf81
AS		 23#include "../../../platform/x86/intel_ips.h"
24
36d516be
CW		 25#define BUSY_MAX_EI 20u /* ms */
26
3e7abf81
AS		 27/*
28 * Lock protecting IPS related data structures
29 */
30static DEFINE_SPINLOCK(mchdev_lock);
31
32static struct intel_gt *rps_to_gt(struct intel_rps *rps)
33{
34 return container_of(rps, struct intel_gt, rps);
35}
36
37static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
38{
39 return rps_to_gt(rps)->i915;
40}
41
42static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
43{
44 return rps_to_gt(rps)->uncore;
45}
46
41e5c17e
VB		 47static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
48{
49 struct intel_gt *gt = rps_to_gt(rps);
50
51 return &gt->uc.guc.slpc;
52}
53
7ba79a67
VB		 54static bool rps_uses_slpc(struct intel_rps *rps)
55{
56 struct intel_gt *gt = rps_to_gt(rps);
57
58 return intel_uc_uses_guc_slpc(&gt->uc);
59}
60
3e7abf81
AS		 61static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
62{
63 return mask & ~rps->pm_intrmsk_mbz;
64}
65
9834dfef 66static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
35cc7f32
CW		 67{
68 intel_uncore_write_fw(uncore, reg, val);
69}
70
36d516be
CW		 71static void rps_timer(struct timer_list *t)
72{
73 struct intel_rps *rps = from_timer(rps, t, timer);
74 struct intel_engine_cs *engine;
5a15550e 75 ktime_t dt, last, timestamp;
36d516be
CW		 76 enum intel_engine_id id;
77 s64 max_busy[3] = {};
36d516be 78
5a15550e 79 timestamp = 0;
36d516be
CW		 80 for_each_engine(engine, rps_to_gt(rps), id) {
81 s64 busy;
82 int i;
83
810b7ee3 84 dt = intel_engine_get_busy_time(engine, &timestamp);
36d516be
CW		 85 last = engine->stats.rps;
86 engine->stats.rps = dt;
87
88 busy = ktime_to_ns(ktime_sub(dt, last));
89 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
90 if (busy > max_busy[i])
91 swap(busy, max_busy[i]);
92 }
93 }
36d516be 94 last = rps->pm_timestamp;
810b7ee3 95 rps->pm_timestamp = timestamp;
36d516be
CW		 96
97 if (intel_rps_is_active(rps)) {
98 s64 busy;
99 int i;
100
810b7ee3 101 dt = ktime_sub(timestamp, last);
36d516be
CW		 102
103 /*
104 * Our goal is to evaluate each engine independently, so we run
105 * at the lowest clocks required to sustain the heaviest
106 * workload. However, a task may be split into sequential
107 * dependent operations across a set of engines, such that
108 * the independent contributions do not account for high load,
109 * but overall the task is GPU bound. For example, consider
110 * video decode on vcs followed by colour post-processing
111 * on vecs, followed by general post-processing on rcs.
112 * Since multi-engines being active does imply a single
113 * continuous workload across all engines, we hedge our
114 * bets by only contributing a factor of the distributed
115 * load into our busyness calculation.
116 */
117 busy = max_busy[0];
118 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
119 if (!max_busy[i])
120 break;
121
122 busy += div_u64(max_busy[i], 1 << i);
123 }
124 GT_TRACE(rps_to_gt(rps),
125 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
126 busy, (int)div64_u64(100 * busy, dt),
127 max_busy[0], max_busy[1], max_busy[2],
128 rps->pm_interval);
129
130 if (100 * busy > rps->power.up_threshold * dt &&
131 rps->cur_freq < rps->max_freq_softlimit) {
132 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
133 rps->pm_interval = 1;
134 schedule_work(&rps->work);
135 } else if (100 * busy < rps->power.down_threshold * dt &&
136 rps->cur_freq > rps->min_freq_softlimit) {
137 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
138 rps->pm_interval = 1;
139 schedule_work(&rps->work);
140 } else {
141 rps->last_adj = 0;
142 }
143
144 mod_timer(&rps->timer,
145 jiffies + msecs_to_jiffies(rps->pm_interval));
146 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
147 }
148}
149
150static void rps_start_timer(struct intel_rps *rps)
151{
152 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
153 rps->pm_interval = 1;
154 mod_timer(&rps->timer, jiffies + 1);
155}
156
157static void rps_stop_timer(struct intel_rps *rps)
158{
159 del_timer_sync(&rps->timer);
160 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
161 cancel_work_sync(&rps->work);
162}
163
3e7abf81
AS		 164static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
165{
166 u32 mask = 0;
167
168 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
169 if (val > rps->min_freq_softlimit)
170 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
171 GEN6_PM_RP_DOWN_THRESHOLD |
172 GEN6_PM_RP_DOWN_TIMEOUT);
173
174 if (val < rps->max_freq_softlimit)
175 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
176
1ebf7aaf 177 mask &= rps->pm_events;
3e7abf81
AS		 178
179 return rps_pm_sanitize_mask(rps, ~mask);
180}
181
182static void rps_reset_ei(struct intel_rps *rps)
183{
184 memset(&rps->ei, 0, sizeof(rps->ei));
185}
186
187static void rps_enable_interrupts(struct intel_rps *rps)
188{
189 struct intel_gt *gt = rps_to_gt(rps);
190
7ba79a67
VB		 191 GEM_BUG_ON(rps_uses_slpc(rps));
192
555a3224
CW		 193 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
194 rps->pm_events, rps_pm_mask(rps, rps->last_freq));
195
3e7abf81
AS		 196 rps_reset_ei(rps);
197
03d2c54d 198 spin_lock_irq(gt->irq_lock);
3e7abf81 199 gen6_gt_pm_enable_irq(gt, rps->pm_events);
03d2c54d 200 spin_unlock_irq(gt->irq_lock);
3e7abf81 201
a080bd99
CW		 202 intel_uncore_write(gt->uncore,
203 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
3e7abf81
AS		 204}
205
206static void gen6_rps_reset_interrupts(struct intel_rps *rps)
207{
208 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
209}
210
211static void gen11_rps_reset_interrupts(struct intel_rps *rps)
212{
213 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
214 ;
215}
216
217static void rps_reset_interrupts(struct intel_rps *rps)
218{
219 struct intel_gt *gt = rps_to_gt(rps);
220
03d2c54d 221 spin_lock_irq(gt->irq_lock);
c816723b 222 if (GRAPHICS_VER(gt->i915) >= 11)
3e7abf81
AS		 223 gen11_rps_reset_interrupts(rps);
224 else
225 gen6_rps_reset_interrupts(rps);
226
227 rps->pm_iir = 0;
03d2c54d 228 spin_unlock_irq(gt->irq_lock);
3e7abf81
AS		 229}
230
231static void rps_disable_interrupts(struct intel_rps *rps)
232{
233 struct intel_gt *gt = rps_to_gt(rps);
234
a080bd99
CW		 235 intel_uncore_write(gt->uncore,
236 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
3e7abf81 237
03d2c54d 238 spin_lock_irq(gt->irq_lock);
3e7abf81 239 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
03d2c54d 240 spin_unlock_irq(gt->irq_lock);
3e7abf81
AS		 241
242 intel_synchronize_irq(gt->i915);
243
244 /*
245 * Now that we will not be generating any more work, flush any
246 * outstanding tasks. As we are called on the RPS idle path,
247 * we will reset the GPU to minimum frequencies, so the current
248 * state of the worker can be discarded.
249 */
250 cancel_work_sync(&rps->work);
251
252 rps_reset_interrupts(rps);
555a3224 253 GT_TRACE(gt, "interrupts:off\n");
3e7abf81
AS		 254}
255
256static const struct cparams {
257 u16 i;
258 u16 t;
259 u16 m;
260 u16 c;
261} cparams[] = {
262 { 1, 1333, 301, 28664 },
263 { 1, 1066, 294, 24460 },
264 { 1, 800, 294, 25192 },
265 { 0, 1333, 276, 27605 },
266 { 0, 1066, 276, 27605 },
267 { 0, 800, 231, 23784 },
268};
269
270static void gen5_rps_init(struct intel_rps *rps)
271{
272 struct drm_i915_private *i915 = rps_to_i915(rps);
273 struct intel_uncore *uncore = rps_to_uncore(rps);
274 u8 fmax, fmin, fstart;
275 u32 rgvmodectl;
276 int c_m, i;
277
278 if (i915->fsb_freq <= 3200)
279 c_m = 0;
280 else if (i915->fsb_freq <= 4800)
281 c_m = 1;
282 else
283 c_m = 2;
284
285 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
286 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
287 rps->ips.m = cparams[i].m;
288 rps->ips.c = cparams[i].c;
289 break;
290 }
291 }
292
293 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
294
295 /* Set up min, max, and cur for interrupt handling */
296 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
297 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
298 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
299 MEMMODE_FSTART_SHIFT;
a8fa7c07
WK		 300 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
301 fmax, fmin, fstart);
3e7abf81 302
dd095afc 303 rps->min_freq = fmax;
043cd2d1 304 rps->efficient_freq = fstart;
dd095afc 305 rps->max_freq = fmin;
3e7abf81
AS		 306}
307
308static unsigned long
309__ips_chipset_val(struct intel_ips *ips)
310{
311 struct intel_uncore *uncore =
312 rps_to_uncore(container_of(ips, struct intel_rps, ips));
313 unsigned long now = jiffies_to_msecs(jiffies), dt;
314 unsigned long result;
315 u64 total, delta;
316
317 lockdep_assert_held(&mchdev_lock);
318
319 /*
320 * Prevent division-by-zero if we are asking too fast.
321 * Also, we don't get interesting results if we are polling
322 * faster than once in 10ms, so just return the saved value
323 * in such cases.
324 */
325 dt = now - ips->last_time1;
326 if (dt <= 10)
327 return ips->chipset_power;
328
329 /* FIXME: handle per-counter overflow */
330 total = intel_uncore_read(uncore, DMIEC);
331 total += intel_uncore_read(uncore, DDREC);
332 total += intel_uncore_read(uncore, CSIEC);
333
334 delta = total - ips->last_count1;
335
336 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
337
338 ips->last_count1 = total;
339 ips->last_time1 = now;
340
341 ips->chipset_power = result;
342
343 return result;
344}
345
346static unsigned long ips_mch_val(struct intel_uncore *uncore)
347{
348 unsigned int m, x, b;
349 u32 tsfs;
350
351 tsfs = intel_uncore_read(uncore, TSFS);
352 x = intel_uncore_read8(uncore, TR1);
353
354 b = tsfs & TSFS_INTR_MASK;
355 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
356
357 return m * x / 127 - b;
358}
359
360static int _pxvid_to_vd(u8 pxvid)
361{
362 if (pxvid == 0)
363 return 0;
364
365 if (pxvid >= 8 && pxvid < 31)
366 pxvid = 31;
367
368 return (pxvid + 2) * 125;
369}
370
371static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
372{
373 const int vd = _pxvid_to_vd(pxvid);
374
375 if (INTEL_INFO(i915)->is_mobile)
376 return max(vd - 1125, 0);
377
378 return vd;
379}
380
381static void __gen5_ips_update(struct intel_ips *ips)
382{
383 struct intel_uncore *uncore =
384 rps_to_uncore(container_of(ips, struct intel_rps, ips));
385 u64 now, delta, dt;
386 u32 count;
387
388 lockdep_assert_held(&mchdev_lock);
389
390 now = ktime_get_raw_ns();
391 dt = now - ips->last_time2;
392 do_div(dt, NSEC_PER_MSEC);
393
394 /* Don't divide by 0 */
395 if (dt <= 10)
396 return;
397
398 count = intel_uncore_read(uncore, GFXEC);
399 delta = count - ips->last_count2;
400
401 ips->last_count2 = count;
402 ips->last_time2 = now;
403
404 /* More magic constants... */
405 ips->gfx_power = div_u64(delta * 1181, dt * 10);
406}
407
408static void gen5_rps_update(struct intel_rps *rps)
409{
410 spin_lock_irq(&mchdev_lock);
411 __gen5_ips_update(&rps->ips);
412 spin_unlock_irq(&mchdev_lock);
413}
414
e82351e7
VS		 415static unsigned int gen5_invert_freq(struct intel_rps *rps,
416 unsigned int val)
417{
418 /* Invert the frequency bin into an ips delay */
419 val = rps->max_freq - val;
420 val = rps->min_freq + val;
421
422 return val;
423}
424
4ee73792 425static int __gen5_rps_set(struct intel_rps *rps, u8 val)
3e7abf81
AS		 426{
427 struct intel_uncore *uncore = rps_to_uncore(rps);
428 u16 rgvswctl;
429
430 lockdep_assert_held(&mchdev_lock);
431
432 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
433 if (rgvswctl & MEMCTL_CMD_STS) {
a10234fd
TU		 434 drm_dbg(&rps_to_i915(rps)->drm,
435 "gpu busy, RCS change rejected\n");
4ee73792 436 return -EBUSY; /* still busy with another command */
3e7abf81
AS		 437 }
438
dd095afc 439 /* Invert the frequency bin into an ips delay */
e82351e7 440 val = gen5_invert_freq(rps, val);
3e7abf81
AS		 441
442 rgvswctl =
443 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
444 (val << MEMCTL_FREQ_SHIFT) |
445 MEMCTL_SFCAVM;
446 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
447 intel_uncore_posting_read16(uncore, MEMSWCTL);
448
449 rgvswctl |= MEMCTL_CMD_STS;
450 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
451
4ee73792
CW		 452 return 0;
453}
454
455static int gen5_rps_set(struct intel_rps *rps, u8 val)
456{
457 int err;
458
459 spin_lock_irq(&mchdev_lock);
460 err = __gen5_rps_set(rps, val);
461 spin_unlock_irq(&mchdev_lock);
462
463 return err;
3e7abf81
AS		 464}
465
466static unsigned long intel_pxfreq(u32 vidfreq)
467{
468 int div = (vidfreq & 0x3f0000) >> 16;
469 int post = (vidfreq & 0x3000) >> 12;
470 int pre = (vidfreq & 0x7);
471
472 if (!pre)
473 return 0;
474
475 return div * 133333 / (pre << post);
476}
477
478static unsigned int init_emon(struct intel_uncore *uncore)
479{
480 u8 pxw[16];
481 int i;
482
483 /* Disable to program */
484 intel_uncore_write(uncore, ECR, 0);
485 intel_uncore_posting_read(uncore, ECR);
486
487 /* Program energy weights for various events */
488 intel_uncore_write(uncore, SDEW, 0x15040d00);
489 intel_uncore_write(uncore, CSIEW0, 0x007f0000);
490 intel_uncore_write(uncore, CSIEW1, 0x1e220004);
491 intel_uncore_write(uncore, CSIEW2, 0x04000004);
492
493 for (i = 0; i < 5; i++)
494 intel_uncore_write(uncore, PEW(i), 0);
495 for (i = 0; i < 3; i++)
496 intel_uncore_write(uncore, DEW(i), 0);
497
498 /* Program P-state weights to account for frequency power adjustment */
499 for (i = 0; i < 16; i++) {
500 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
501 unsigned int freq = intel_pxfreq(pxvidfreq);
502 unsigned int vid =
503 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
504 unsigned int val;
505
506 val = vid * vid * freq / 1000 * 255;
507 val /= 127 * 127 * 900;
508
509 pxw[i] = val;
510 }
511 /* Render standby states get 0 weight */
512 pxw[14] = 0;
513 pxw[15] = 0;
514
515 for (i = 0; i < 4; i++) {
516 intel_uncore_write(uncore, PXW(i),
517 pxw[i * 4 + 0] << 24 |
518 pxw[i * 4 + 1] << 16 |
519 pxw[i * 4 + 2] << 8 |
520 pxw[i * 4 + 3] << 0);
521 }
522
523 /* Adjust magic regs to magic values (more experimental results) */
524 intel_uncore_write(uncore, OGW0, 0);
525 intel_uncore_write(uncore, OGW1, 0);
526 intel_uncore_write(uncore, EG0, 0x00007f00);
527 intel_uncore_write(uncore, EG1, 0x0000000e);
528 intel_uncore_write(uncore, EG2, 0x000e0000);
529 intel_uncore_write(uncore, EG3, 0x68000300);
530 intel_uncore_write(uncore, EG4, 0x42000000);
531 intel_uncore_write(uncore, EG5, 0x00140031);
532 intel_uncore_write(uncore, EG6, 0);
533 intel_uncore_write(uncore, EG7, 0);
534
535 for (i = 0; i < 8; i++)
536 intel_uncore_write(uncore, PXWL(i), 0);
537
538 /* Enable PMON + select events */
539 intel_uncore_write(uncore, ECR, 0x80000019);
540
541 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
542}
543
544static bool gen5_rps_enable(struct intel_rps *rps)
545{
c6073d4c 546 struct drm_i915_private *i915 = rps_to_i915(rps);
3e7abf81
AS		 547 struct intel_uncore *uncore = rps_to_uncore(rps);
548 u8 fstart, vstart;
549 u32 rgvmodectl;
550
551 spin_lock_irq(&mchdev_lock);
552
553 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
554
555 /* Enable temp reporting */
556 intel_uncore_write16(uncore, PMMISC,
557 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
558 intel_uncore_write16(uncore, TSC1,
559 intel_uncore_read16(uncore, TSC1) | TSE);
560
561 /* 100ms RC evaluation intervals */
562 intel_uncore_write(uncore, RCUPEI, 100000);
563 intel_uncore_write(uncore, RCDNEI, 100000);
564
565 /* Set max/min thresholds to 90ms and 80ms respectively */
566 intel_uncore_write(uncore, RCBMAXAVG, 90000);
567 intel_uncore_write(uncore, RCBMINAVG, 80000);
568
569 intel_uncore_write(uncore, MEMIHYST, 1);
570
571 /* Set up min, max, and cur for interrupt handling */
572 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
573 MEMMODE_FSTART_SHIFT;
574
575 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
576 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
577
578 intel_uncore_write(uncore,
579 MEMINTREN,
580 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
581
582 intel_uncore_write(uncore, VIDSTART, vstart);
583 intel_uncore_posting_read(uncore, VIDSTART);
584
585 rgvmodectl |= MEMMODE_SWMODE_EN;
586 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
587
588 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
589 MEMCTL_CMD_STS) == 0, 10))
a8fa7c07
WK		 590 drm_err(&uncore->i915->drm,
591 "stuck trying to change perf mode\n");
3e7abf81
AS		 592 mdelay(1);
593
4ee73792 594 __gen5_rps_set(rps, rps->cur_freq);
3e7abf81
AS		 595
596 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
597 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
598 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
599 rps->ips.last_time1 = jiffies_to_msecs(jiffies);
600
601 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
602 rps->ips.last_time2 = ktime_get_raw_ns();
603
c6073d4c
VS		 604 spin_lock(&i915->irq_lock);
605 ilk_enable_display_irq(i915, DE_PCU_EVENT);
606 spin_unlock(&i915->irq_lock);
607
3e7abf81
AS		 608 spin_unlock_irq(&mchdev_lock);
609
610 rps->ips.corr = init_emon(uncore);
611
612 return true;
613}
614
615static void gen5_rps_disable(struct intel_rps *rps)
616{
c6073d4c 617 struct drm_i915_private *i915 = rps_to_i915(rps);
3e7abf81
AS		 618 struct intel_uncore *uncore = rps_to_uncore(rps);
619 u16 rgvswctl;
620
621 spin_lock_irq(&mchdev_lock);
622
c6073d4c
VS		 623 spin_lock(&i915->irq_lock);
624 ilk_disable_display_irq(i915, DE_PCU_EVENT);
625 spin_unlock(&i915->irq_lock);
626
3e7abf81
AS		 627 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
628
629 /* Ack interrupts, disable EFC interrupt */
c61aa740 630 intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0);
3e7abf81 631 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
3e7abf81
AS		 632
633 /* Go back to the starting frequency */
4ee73792 634 __gen5_rps_set(rps, rps->idle_freq);
3e7abf81
AS		 635 mdelay(1);
636 rgvswctl |= MEMCTL_CMD_STS;
637 intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
638 mdelay(1);
639
640 spin_unlock_irq(&mchdev_lock);
641}
642
643static u32 rps_limits(struct intel_rps *rps, u8 val)
644{
645 u32 limits;
646
647 /*
648 * Only set the down limit when we've reached the lowest level to avoid
649 * getting more interrupts, otherwise leave this clear. This prevents a
650 * race in the hw when coming out of rc6: There's a tiny window where
651 * the hw runs at the minimal clock before selecting the desired
652 * frequency, if the down threshold expires in that window we will not
653 * receive a down interrupt.
654 */
c816723b 655 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
3e7abf81
AS		 656 limits = rps->max_freq_softlimit << 23;
657 if (val <= rps->min_freq_softlimit)
658 limits |= rps->min_freq_softlimit << 14;
659 } else {
660 limits = rps->max_freq_softlimit << 24;
661 if (val <= rps->min_freq_softlimit)
662 limits |= rps->min_freq_softlimit << 16;
663 }
664
665 return limits;
666}
667
668static void rps_set_power(struct intel_rps *rps, int new_power)
669{
9c878557
CW		 670 struct intel_gt *gt = rps_to_gt(rps);
671 struct intel_uncore *uncore = gt->uncore;
3e7abf81
AS		 672 u32 threshold_up = 0, threshold_down = 0; /* in % */
673 u32 ei_up = 0, ei_down = 0;
674
675 lockdep_assert_held(&rps->power.mutex);
676
677 if (new_power == rps->power.mode)
678 return;
679
36d516be
CW		 680 threshold_up = 95;
681 threshold_down = 85;
682
3e7abf81
AS		 683 /* Note the units here are not exactly 1us, but 1280ns. */
684 switch (new_power) {
685 case LOW_POWER:
3e7abf81 686 ei_up = 16000;
3e7abf81 687 ei_down = 32000;
3e7abf81
AS		 688 break;
689
690 case BETWEEN:
3e7abf81 691 ei_up = 13000;
3e7abf81 692 ei_down = 32000;
3e7abf81
AS		 693 break;
694
695 case HIGH_POWER:
3e7abf81 696 ei_up = 10000;
3e7abf81 697 ei_down = 32000;
3e7abf81
AS		 698 break;
699 }
700
701 /* When byt can survive without system hang with dynamic
702 * sw freq adjustments, this restriction can be lifted.
703 */
9c878557 704 if (IS_VALLEYVIEW(gt->i915))
3e7abf81
AS		 705 goto skip_hw_write;
706
9c878557 707 GT_TRACE(gt,
555a3224
CW		 708 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
709 new_power, threshold_up, ei_up, threshold_down, ei_down);
710
9c878557
CW		 711 set(uncore, GEN6_RP_UP_EI,
712 intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
35cc7f32 713 set(uncore, GEN6_RP_UP_THRESHOLD,
9c878557 714 intel_gt_ns_to_pm_interval(gt, ei_up * threshold_up * 10));
35cc7f32 715
9c878557
CW		 716 set(uncore, GEN6_RP_DOWN_EI,
717 intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
35cc7f32 718 set(uncore, GEN6_RP_DOWN_THRESHOLD,
9c878557 719 intel_gt_ns_to_pm_interval(gt, ei_down * threshold_down * 10));
35cc7f32
CW		 720
721 set(uncore, GEN6_RP_CONTROL,
c816723b 722 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
35cc7f32
CW		 723 GEN6_RP_MEDIA_HW_NORMAL_MODE |
724 GEN6_RP_MEDIA_IS_GFX |
725 GEN6_RP_ENABLE |
726 GEN6_RP_UP_BUSY_AVG |
727 GEN6_RP_DOWN_IDLE_AVG);
3e7abf81
AS		 728
729skip_hw_write:
730 rps->power.mode = new_power;
731 rps->power.up_threshold = threshold_up;
732 rps->power.down_threshold = threshold_down;
733}
734
735static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
736{
737 int new_power;
738
739 new_power = rps->power.mode;
740 switch (rps->power.mode) {
741 case LOW_POWER:
742 if (val > rps->efficient_freq + 1 &&
743 val > rps->cur_freq)
744 new_power = BETWEEN;
745 break;
746
747 case BETWEEN:
748 if (val <= rps->efficient_freq &&
749 val < rps->cur_freq)
750 new_power = LOW_POWER;
751 else if (val >= rps->rp0_freq &&
752 val > rps->cur_freq)
753 new_power = HIGH_POWER;
754 break;
755
756 case HIGH_POWER:
757 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
758 val < rps->cur_freq)
759 new_power = BETWEEN;
760 break;
761 }
762 /* Max/min bins are special */
763 if (val <= rps->min_freq_softlimit)
764 new_power = LOW_POWER;
765 if (val >= rps->max_freq_softlimit)
766 new_power = HIGH_POWER;
767
768 mutex_lock(&rps->power.mutex);
769 if (rps->power.interactive)
770 new_power = HIGH_POWER;
771 rps_set_power(rps, new_power);
772 mutex_unlock(&rps->power.mutex);
773}
774
775void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
776{
01fabda8
LDM		 777 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
778 str_yes_no(interactive));
555a3224 779
3e7abf81
AS		 780 mutex_lock(&rps->power.mutex);
781 if (interactive) {
9bad2adb 782 if (!rps->power.interactive++ && intel_rps_is_active(rps))
3e7abf81
AS		 783 rps_set_power(rps, HIGH_POWER);
784 } else {
785 GEM_BUG_ON(!rps->power.interactive);
786 rps->power.interactive--;
787 }
788 mutex_unlock(&rps->power.mutex);
789}
790
791static int gen6_rps_set(struct intel_rps *rps, u8 val)
792{
793 struct intel_uncore *uncore = rps_to_uncore(rps);
794 struct drm_i915_private *i915 = rps_to_i915(rps);
795 u32 swreq;
796
7ba79a67
VB		 797 GEM_BUG_ON(rps_uses_slpc(rps));
798
c816723b 799 if (GRAPHICS_VER(i915) >= 9)
3e7abf81
AS		 800 swreq = GEN9_FREQUENCY(val);
801 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
802 swreq = HSW_FREQUENCY(val);
803 else
804 swreq = (GEN6_FREQUENCY(val) |
805 GEN6_OFFSET(0) |
806 GEN6_AGGRESSIVE_TURBO);
35cc7f32 807 set(uncore, GEN6_RPNSWREQ, swreq);
3e7abf81 808
555a3224
CW		 809 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
810 val, intel_gpu_freq(rps, val), swreq);
811
3e7abf81
AS		 812 return 0;
813}
814
815static int vlv_rps_set(struct intel_rps *rps, u8 val)
816{
817 struct drm_i915_private *i915 = rps_to_i915(rps);
818 int err;
819
820 vlv_punit_get(i915);
821 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
822 vlv_punit_put(i915);
823
555a3224
CW		 824 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
825 val, intel_gpu_freq(rps, val));
826
3e7abf81
AS		 827 return err;
828}
829
28117632 830static int rps_set(struct intel_rps *rps, u8 val, bool update)
3e7abf81
AS		 831{
832 struct drm_i915_private *i915 = rps_to_i915(rps);
833 int err;
834
3e7abf81
AS		 835 if (val == rps->last_freq)
836 return 0;
837
838 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
839 err = vlv_rps_set(rps, val);
c816723b 840 else if (GRAPHICS_VER(i915) >= 6)
3e7abf81 841 err = gen6_rps_set(rps, val);
4ee73792
CW		 842 else
843 err = gen5_rps_set(rps, val);
3e7abf81
AS		 844 if (err)
845 return err;
846
c816723b 847 if (update && GRAPHICS_VER(i915) >= 6)
28117632 848 gen6_rps_set_thresholds(rps, val);
3e7abf81
AS		 849 rps->last_freq = val;
850
851 return 0;
852}
853
854void intel_rps_unpark(struct intel_rps *rps)
855{
9bad2adb 856 if (!intel_rps_is_enabled(rps))
3e7abf81
AS		 857 return;
858
555a3224
CW		 859 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
860
3e7abf81
AS		 861 /*
862 * Use the user's desired frequency as a guide, but for better
863 * performance, jump directly to RPe as our starting frequency.
864 */
865 mutex_lock(&rps->lock);
74e5a9ac 866
9bad2adb 867 intel_rps_set_active(rps);
043cd2d1
CW		 868 intel_rps_set(rps,
869 clamp(rps->cur_freq,
870 rps->min_freq_softlimit,
871 rps->max_freq_softlimit));
ff345271 872
3e7abf81
AS		 873 mutex_unlock(&rps->lock);
874
36d516be 875 rps->pm_iir = 0;
8e99299a 876 if (intel_rps_has_interrupts(rps))
3e7abf81 877 rps_enable_interrupts(rps);
36d516be
CW		 878 if (intel_rps_uses_timer(rps))
879 rps_start_timer(rps);
3e7abf81 880
c816723b 881 if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
3e7abf81
AS		 882 gen5_rps_update(rps);
883}
884
885void intel_rps_park(struct intel_rps *rps)
886{
3f88dde6
CW		 887 int adj;
888
7ba79a67
VB		 889 if (!intel_rps_is_enabled(rps))
890 return;
891
9bad2adb 892 if (!intel_rps_clear_active(rps))
3e7abf81
AS		 893 return;
894
36d516be
CW		 895 if (intel_rps_uses_timer(rps))
896 rps_stop_timer(rps);
8e99299a 897 if (intel_rps_has_interrupts(rps))
3e7abf81
AS		 898 rps_disable_interrupts(rps);
899
3e7abf81
AS		 900 if (rps->last_freq <= rps->idle_freq)
901 return;
902
903 /*
904 * The punit delays the write of the frequency and voltage until it
905 * determines the GPU is awake. During normal usage we don't want to
906 * waste power changing the frequency if the GPU is sleeping (rc6).
907 * However, the GPU and driver is now idle and we do not want to delay
908 * switching to minimum voltage (reducing power whilst idle) as we do
909 * not expect to be woken in the near future and so must flush the
910 * change by waking the device.
911 *
912 * We choose to take the media powerwell (either would do to trick the
913 * punit into committing the voltage change) as that takes a lot less
914 * power than the render powerwell.
915 */
916 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
28117632 917 rps_set(rps, rps->idle_freq, false);
3e7abf81 918 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
21abf0bf
CW		 919
920 /*
921 * Since we will try and restart from the previously requested
922 * frequency on unparking, treat this idle point as a downclock
923 * interrupt and reduce the frequency for resume. If we park/unpark
924 * more frequently than the rps worker can run, we will not respond
925 * to any EI and never see a change in frequency.
926 *
927 * (Note we accommodate Cherryview's limitation of only using an
928 * even bin by applying it to all.)
929 */
3f88dde6
CW		 930 adj = rps->last_adj;
931 if (adj < 0)
932 adj *= 2;
933 else /* CHV needs even encode values */
934 adj = -2;
935 rps->last_adj = adj;
936 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
f7ed83cc
CW		 937 if (rps->cur_freq < rps->efficient_freq) {
938 rps->cur_freq = rps->efficient_freq;
939 rps->last_adj = 0;
940 }
555a3224
CW		 941
942 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
3e7abf81
AS		 943}
944
1448d5c4
VB		 945u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
946{
947 struct intel_guc_slpc *slpc;
948
949 if (rps_uses_slpc(rps)) {
950 slpc = rps_to_slpc(rps);
951
952 return slpc->boost_freq;
953 } else {
954 return intel_gpu_freq(rps, rps->boost_freq);
955 }
956}
957
958static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
959{
960 bool boost = false;
961
962 /* Validate against (static) hardware limits */
963 val = intel_freq_opcode(rps, val);
964 if (val < rps->min_freq || val > rps->max_freq)
965 return -EINVAL;
966
967 mutex_lock(&rps->lock);
968 if (val != rps->boost_freq) {
969 rps->boost_freq = val;
970 boost = atomic_read(&rps->num_waiters);
971 }
972 mutex_unlock(&rps->lock);
973 if (boost)
974 schedule_work(&rps->work);
975
976 return 0;
977}
978
979int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
980{
981 struct intel_guc_slpc *slpc;
982
983 if (rps_uses_slpc(rps)) {
984 slpc = rps_to_slpc(rps);
985
986 return intel_guc_slpc_set_boost_freq(slpc, freq);
987 } else {
988 return rps_set_boost_freq(rps, freq);
989 }
990}
991
493043fe
VB		 992void intel_rps_dec_waiters(struct intel_rps *rps)
993{
994 struct intel_guc_slpc *slpc;
995
996 if (rps_uses_slpc(rps)) {
997 slpc = rps_to_slpc(rps);
998
999 intel_guc_slpc_dec_waiters(slpc);
1000 } else {
1001 atomic_dec(&rps->num_waiters);
1002 }
1003}
1004
3e7abf81
AS		 1005void intel_rps_boost(struct i915_request *rq)
1006{
493043fe
VB		 1007 struct intel_guc_slpc *slpc;
1008
4e5c8a99 1009 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
3e7abf81
AS		 1010 return;
1011
1012 /* Serializes with i915_request_retire() */
4e5c8a99
CW		 1013 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1014 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1015
493043fe
VB		 1016 if (rps_uses_slpc(rps)) {
1017 slpc = rps_to_slpc(rps);
1018
f864a29a
VB		 1019 if (slpc->min_freq_softlimit >= slpc->boost_freq)
1020 return;
1021
493043fe 1022 /* Return if old value is non zero */
f864a29a
VB		 1023 if (!atomic_fetch_inc(&slpc->num_waiters)) {
1024 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1025 rq->fence.context, rq->fence.seqno);
493043fe 1026 schedule_work(&slpc->boost_work);
f864a29a 1027 }
493043fe
VB		 1028
1029 return;
1030 }
1031
4e5c8a99
CW		 1032 if (atomic_fetch_inc(&rps->num_waiters))
1033 return;
1034
1035 if (!intel_rps_is_active(rps))
1036 return;
3e7abf81 1037
555a3224
CW		 1038 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1039 rq->fence.context, rq->fence.seqno);
1040
4e5c8a99 1041 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
3e7abf81
AS		 1042 schedule_work(&rps->work);
1043
4e5c8a99 1044 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
3e7abf81 1045 }
3e7abf81
AS		 1046}
1047
1048int intel_rps_set(struct intel_rps *rps, u8 val)
1049{
35cc7f32 1050 int err;
3e7abf81
AS		 1051
1052 lockdep_assert_held(&rps->lock);
1053 GEM_BUG_ON(val > rps->max_freq);
1054 GEM_BUG_ON(val < rps->min_freq);
1055
9bad2adb 1056 if (intel_rps_is_active(rps)) {
28117632 1057 err = rps_set(rps, val, true);
35cc7f32
CW		 1058 if (err)
1059 return err;
3e7abf81
AS		 1060
1061 /*
1062 * Make sure we continue to get interrupts
1063 * until we hit the minimum or maximum frequencies.
1064 */
8e99299a 1065 if (intel_rps_has_interrupts(rps)) {
3e7abf81
AS		 1066 struct intel_uncore *uncore = rps_to_uncore(rps);
1067
35cc7f32
CW		 1068 set(uncore,
1069 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
3e7abf81 1070
35cc7f32 1071 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
3e7abf81
AS		 1072 }
1073 }
1074
35cc7f32
CW		 1075 rps->cur_freq = val;
1076 return 0;
3e7abf81
AS		 1077}
1078
56758cc4
AD		 1079static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1080{
1081 struct drm_i915_private *i915 = rps_to_i915(rps);
1082 struct intel_uncore *uncore = rps_to_uncore(rps);
1083
4de23dca
MR		 1084 if (IS_PONTEVECCHIO(i915))
1085 return intel_uncore_read(uncore, PVC_RP_STATE_CAP);
1086 else if (IS_XEHPSDV(i915))
56758cc4
AD		 1087 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP);
1088 else if (IS_GEN9_LP(i915))
1089 return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1090 else
1091 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1092}
1093
835a4d18
AD		 1094static void
1095mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1096{
1097 struct intel_uncore *uncore = rps_to_uncore(rps);
1098 u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ?
1099 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) :
1100 intel_uncore_read(uncore, MTL_RP_STATE_CAP);
1101 u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ?
1102 intel_uncore_read(uncore, MTL_MPE_FREQUENCY) :
1103 intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY);
1104
1105 /* MTL values are in units of 16.67 MHz */
1106 caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap);
1107 caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap);
1108 caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe);
1109}
1110
1111static void
1112__gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
3e7abf81
AS		 1113{
1114 struct drm_i915_private *i915 = rps_to_i915(rps);
56758cc4 1115 u32 rp_state_cap;
3e7abf81 1116
56758cc4 1117 rp_state_cap = intel_rps_read_state_cap(rps);
3e7abf81
AS		 1118
1119 /* static values from HW: RP0 > RP1 > RPn (min_freq) */
1120 if (IS_GEN9_LP(i915)) {
56758cc4
AD		 1121 caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1122 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
1123 caps->min_freq = (rp_state_cap >> 0) & 0xff;
3e7abf81 1124 } else {
56758cc4 1125 caps->rp0_freq = (rp_state_cap >> 0) & 0xff;
95ccf312
VB		 1126 if (GRAPHICS_VER(i915) >= 10)
1127 caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1128 intel_uncore_read(to_gt(i915)->uncore,
1129 GEN10_FREQ_INFO_REC));
1130 else
1131 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
56758cc4 1132 caps->min_freq = (rp_state_cap >> 16) & 0xff;
3e7abf81
AS		 1133 }
1134
56758cc4
AD		 1135 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1136 /*
1137 * In this case rp_state_cap register reports frequencies in
1138 * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1139 * units of 16.67 MHz
1140 */
1141 caps->rp0_freq *= GEN9_FREQ_SCALER;
1142 caps->rp1_freq *= GEN9_FREQ_SCALER;
1143 caps->min_freq *= GEN9_FREQ_SCALER;
1144 }
1145}
1146
835a4d18
AD		 1147/**
1148 * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1149 * @rps: the intel_rps structure
1150 * @caps: returned freq caps
1151 *
1152 * Returned "caps" frequencies should be converted to MHz using
1153 * intel_gpu_freq()
1154 */
1155void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1156{
1157 struct drm_i915_private *i915 = rps_to_i915(rps);
1158
1159 if (IS_METEORLAKE(i915))
1160 return mtl_get_freq_caps(rps, caps);
1161 else
1162 return __gen6_rps_get_freq_caps(rps, caps);
1163}
1164
56758cc4
AD		 1165static void gen6_rps_init(struct intel_rps *rps)
1166{
1167 struct drm_i915_private *i915 = rps_to_i915(rps);
1168 struct intel_rps_freq_caps caps;
1169
1170 gen6_rps_get_freq_caps(rps, &caps);
1171 rps->rp0_freq = caps.rp0_freq;
1172 rps->rp1_freq = caps.rp1_freq;
1173 rps->min_freq = caps.min_freq;
1174
3e7abf81
AS		 1175 /* hw_max = RP0 until we check for overclocking */
1176 rps->max_freq = rps->rp0_freq;
1177
1178 rps->efficient_freq = rps->rp1_freq;
1179 if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
6266992c 1180 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
3e7abf81 1181 u32 ddcc_status = 0;
56758cc4 1182 u32 mult = 1;
3e7abf81 1183
56758cc4
AD		 1184 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1185 mult = GEN9_FREQ_SCALER;
ee421bb4
AD		 1186 if (snb_pcode_read(rps_to_gt(rps)->uncore,
1187 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
6650ebcb 1188 &ddcc_status, NULL) == 0)
3e7abf81 1189 rps->efficient_freq =
56758cc4
AD		 1190 clamp_t(u32,
1191 ((ddcc_status >> 8) & 0xff) * mult,
3e7abf81
AS		 1192 rps->min_freq,
1193 rps->max_freq);
1194 }
3e7abf81
AS		 1195}
1196
1197static bool rps_reset(struct intel_rps *rps)
1198{
a8fa7c07 1199 struct drm_i915_private *i915 = rps_to_i915(rps);
555a3224 1200
3e7abf81
AS		 1201 /* force a reset */
1202 rps->power.mode = -1;
1203 rps->last_freq = -1;
1204
28117632 1205 if (rps_set(rps, rps->min_freq, true)) {
a8fa7c07 1206 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
3e7abf81
AS		 1207 return false;
1208 }
1209
1210 rps->cur_freq = rps->min_freq;
1211 return true;
1212}
1213
1214/* See the Gen9_GT_PM_Programming_Guide doc for the below */
1215static bool gen9_rps_enable(struct intel_rps *rps)
1216{
9c878557
CW		 1217 struct intel_gt *gt = rps_to_gt(rps);
1218 struct intel_uncore *uncore = gt->uncore;
3e7abf81
AS		 1219
1220 /* Program defaults and thresholds for RPS */
c816723b 1221 if (GRAPHICS_VER(gt->i915) == 9)
3e7abf81
AS		 1222 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1223 GEN9_FREQUENCY(rps->rp1_freq));
1224
3e7abf81
AS		 1225 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1226
1ebf7aaf
CW		 1227 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1228
3e7abf81
AS		 1229 return rps_reset(rps);
1230}
1231
1232static bool gen8_rps_enable(struct intel_rps *rps)
1233{
1234 struct intel_uncore *uncore = rps_to_uncore(rps);
1235
1236 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1237 HSW_FREQUENCY(rps->rp1_freq));
1238
3e7abf81
AS		 1239 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1240
1ebf7aaf
CW		 1241 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1242
3e7abf81
AS		 1243 return rps_reset(rps);
1244}
1245
1246static bool gen6_rps_enable(struct intel_rps *rps)
1247{
1248 struct intel_uncore *uncore = rps_to_uncore(rps);
1249
1250 /* Power down if completely idle for over 50ms */
1251 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1252 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1253
1ebf7aaf
CW		 1254 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1255 GEN6_PM_RP_DOWN_THRESHOLD |
1256 GEN6_PM_RP_DOWN_TIMEOUT);
1257
3e7abf81
AS		 1258 return rps_reset(rps);
1259}
1260
1261static int chv_rps_max_freq(struct intel_rps *rps)
1262{
1263 struct drm_i915_private *i915 = rps_to_i915(rps);
0b6613c6 1264 struct intel_gt *gt = rps_to_gt(rps);
3e7abf81
AS		 1265 u32 val;
1266
1267 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1268
0b6613c6 1269 switch (gt->info.sseu.eu_total) {
3e7abf81
AS		 1270 case 8:
1271 /* (2 * 4) config */
1272 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1273 break;
1274 case 12:
1275 /* (2 * 6) config */
1276 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1277 break;
1278 case 16:
1279 /* (2 * 8) config */
1280 default:
1281 /* Setting (2 * 8) Min RP0 for any other combination */
1282 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1283 break;
1284 }
1285
1286 return val & FB_GFX_FREQ_FUSE_MASK;
1287}
1288
1289static int chv_rps_rpe_freq(struct intel_rps *rps)
1290{
1291 struct drm_i915_private *i915 = rps_to_i915(rps);
1292 u32 val;
1293
1294 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1295 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1296
1297 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1298}
1299
1300static int chv_rps_guar_freq(struct intel_rps *rps)
1301{
1302 struct drm_i915_private *i915 = rps_to_i915(rps);
1303 u32 val;
1304
1305 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1306
1307 return val & FB_GFX_FREQ_FUSE_MASK;
1308}
1309
1310static u32 chv_rps_min_freq(struct intel_rps *rps)
1311{
1312 struct drm_i915_private *i915 = rps_to_i915(rps);
1313 u32 val;
1314
1315 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1316 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1317
1318 return val & FB_GFX_FREQ_FUSE_MASK;
1319}
1320
1321static bool chv_rps_enable(struct intel_rps *rps)
1322{
1323 struct intel_uncore *uncore = rps_to_uncore(rps);
1324 struct drm_i915_private *i915 = rps_to_i915(rps);
1325 u32 val;
1326
1327 /* 1: Program defaults and thresholds for RPS*/
1328 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1329 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1330 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1331 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1332 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1333
1334 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1335
1336 /* 2: Enable RPS */
1337 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1338 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1339 GEN6_RP_MEDIA_IS_GFX |
1340 GEN6_RP_ENABLE |
1341 GEN6_RP_UP_BUSY_AVG |
1342 GEN6_RP_DOWN_IDLE_AVG);
1343
1ebf7aaf
CW		 1344 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1345 GEN6_PM_RP_DOWN_THRESHOLD |
1346 GEN6_PM_RP_DOWN_TIMEOUT);
1347
3e7abf81
AS		 1348 /* Setting Fixed Bias */
1349 vlv_punit_get(i915);
1350
1351 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1352 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1353
1354 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1355
1356 vlv_punit_put(i915);
1357
1358 /* RPS code assumes GPLL is used */
0d4c351a
PB		 1359 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1360 "GPLL not enabled\n");
3e7abf81 1361
01fabda8
LDM		 1362 drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1363 str_yes_no(val & GPLLENABLE));
a8fa7c07 1364 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
3e7abf81
AS		 1365
1366 return rps_reset(rps);
1367}
1368
1369static int vlv_rps_guar_freq(struct intel_rps *rps)
1370{
1371 struct drm_i915_private *i915 = rps_to_i915(rps);
1372 u32 val, rp1;
1373
1374 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1375
1376 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1377 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1378
1379 return rp1;
1380}
1381
1382static int vlv_rps_max_freq(struct intel_rps *rps)
1383{
1384 struct drm_i915_private *i915 = rps_to_i915(rps);
1385 u32 val, rp0;
1386
1387 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1388
1389 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1390 /* Clamp to max */
1391 rp0 = min_t(u32, rp0, 0xea);
1392
1393 return rp0;
1394}
1395
1396static int vlv_rps_rpe_freq(struct intel_rps *rps)
1397{
1398 struct drm_i915_private *i915 = rps_to_i915(rps);
1399 u32 val, rpe;
1400
1401 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1402 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1403 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1404 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1405
1406 return rpe;
1407}
1408
1409static int vlv_rps_min_freq(struct intel_rps *rps)
1410{
1411 struct drm_i915_private *i915 = rps_to_i915(rps);
1412 u32 val;
1413
1414 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1415 /*
1416 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1417 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1418 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1419 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1420 * to make sure it matches what Punit accepts.
1421 */
1422 return max_t(u32, val, 0xc0);
1423}
1424
1425static bool vlv_rps_enable(struct intel_rps *rps)
1426{
1427 struct intel_uncore *uncore = rps_to_uncore(rps);
1428 struct drm_i915_private *i915 = rps_to_i915(rps);
1429 u32 val;
1430
1431 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1432 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1433 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1434 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1435 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1436
1437 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1438
1439 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1440 GEN6_RP_MEDIA_TURBO |
1441 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1442 GEN6_RP_MEDIA_IS_GFX |
1443 GEN6_RP_ENABLE |
1444 GEN6_RP_UP_BUSY_AVG |
1445 GEN6_RP_DOWN_IDLE_CONT);
1446
1ebf7aaf
CW		 1447 /* WaGsvRC0ResidencyMethod:vlv */
1448 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1449
3e7abf81
AS		 1450 vlv_punit_get(i915);
1451
1452 /* Setting Fixed Bias */
1453 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1454 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1455
1456 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1457
1458 vlv_punit_put(i915);
1459
1460 /* RPS code assumes GPLL is used */
0d4c351a
PB		 1461 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1462 "GPLL not enabled\n");
3e7abf81 1463
01fabda8
LDM		 1464 drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1465 str_yes_no(val & GPLLENABLE));
a8fa7c07 1466 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
3e7abf81
AS		 1467
1468 return rps_reset(rps);
1469}
1470
1471static unsigned long __ips_gfx_val(struct intel_ips *ips)
1472{
1473 struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1474 struct intel_uncore *uncore = rps_to_uncore(rps);
d08c4e23 1475 unsigned int t, state1, state2;
3e7abf81 1476 u32 pxvid, ext_v;
d08c4e23 1477 u64 corr, corr2;
3e7abf81
AS		 1478
1479 lockdep_assert_held(&mchdev_lock);
1480
1481 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1482 pxvid = (pxvid >> 24) & 0x7f;
1483 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1484
1485 state1 = ext_v;
1486
1487 /* Revel in the empirically derived constants */
1488
1489 /* Correction factor in 1/100000 units */
1490 t = ips_mch_val(uncore);
1491 if (t > 80)
1492 corr = t * 2349 + 135940;
1493 else if (t >= 50)
1494 corr = t * 964 + 29317;
1495 else /* < 50 */
1496 corr = t * 301 + 1004;
1497
d08c4e23
VS		 1498 corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1499 corr2 = div_u64(corr, 100000) * ips->corr;
3e7abf81 1500
d08c4e23 1501 state2 = div_u64(corr2 * state1, 10000);
3e7abf81
AS		 1502 state2 /= 100; /* convert to mW */
1503
1504 __gen5_ips_update(ips);
1505
1506 return ips->gfx_power + state2;
1507}
1508
36d516be
CW		 1509static bool has_busy_stats(struct intel_rps *rps)
1510{
1511 struct intel_engine_cs *engine;
1512 enum intel_engine_id id;
1513
1514 for_each_engine(engine, rps_to_gt(rps), id) {
1515 if (!intel_engine_supports_stats(engine))
1516 return false;
1517 }
1518
1519 return true;
1520}
1521
3e7abf81
AS		 1522void intel_rps_enable(struct intel_rps *rps)
1523{
1524 struct drm_i915_private *i915 = rps_to_i915(rps);
1525 struct intel_uncore *uncore = rps_to_uncore(rps);
9bad2adb 1526 bool enabled = false;
3e7abf81 1527
9c878557
CW		 1528 if (!HAS_RPS(i915))
1529 return;
1530
7ba79a67
VB		 1531 if (rps_uses_slpc(rps))
1532 return;
1533
9c878557
CW		 1534 intel_gt_check_clock_frequency(rps_to_gt(rps));
1535
3e7abf81 1536 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
9bad2adb
CW		 1537 if (rps->max_freq <= rps->min_freq)
1538 /* leave disabled, no room for dynamic reclocking */;
1539 else if (IS_CHERRYVIEW(i915))
1540 enabled = chv_rps_enable(rps);
3e7abf81 1541 else if (IS_VALLEYVIEW(i915))
9bad2adb 1542 enabled = vlv_rps_enable(rps);
c816723b 1543 else if (GRAPHICS_VER(i915) >= 9)
9bad2adb 1544 enabled = gen9_rps_enable(rps);
c816723b 1545 else if (GRAPHICS_VER(i915) >= 8)
9bad2adb 1546 enabled = gen8_rps_enable(rps);
c816723b 1547 else if (GRAPHICS_VER(i915) >= 6)
9bad2adb 1548 enabled = gen6_rps_enable(rps);
3e7abf81 1549 else if (IS_IRONLAKE_M(i915))
9bad2adb
CW		 1550 enabled = gen5_rps_enable(rps);
1551 else
c816723b 1552 MISSING_CASE(GRAPHICS_VER(i915));
3e7abf81 1553 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
9bad2adb 1554 if (!enabled)
3e7abf81
AS		 1555 return;
1556
555a3224
CW		 1557 GT_TRACE(rps_to_gt(rps),
1558 "min:%x, max:%x, freq:[%d, %d]\n",
1559 rps->min_freq, rps->max_freq,
1560 intel_gpu_freq(rps, rps->min_freq),
1561 intel_gpu_freq(rps, rps->max_freq));
3e7abf81 1562
555a3224
CW		 1563 GEM_BUG_ON(rps->max_freq < rps->min_freq);
1564 GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1565
1566 GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1567 GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
9bad2adb 1568
36d516be
CW		 1569 if (has_busy_stats(rps))
1570 intel_rps_set_timer(rps);
bbd57d16 1571 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
8e99299a
CW		 1572 intel_rps_set_interrupts(rps);
1573 else
1574 /* Ironlake currently uses intel_ips.ko */ {}
1575
9bad2adb 1576 intel_rps_set_enabled(rps);
3e7abf81
AS		 1577}
1578
1579static void gen6_rps_disable(struct intel_rps *rps)
1580{
35cc7f32 1581 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
3e7abf81
AS		 1582}
1583
1584void intel_rps_disable(struct intel_rps *rps)
1585{
1586 struct drm_i915_private *i915 = rps_to_i915(rps);
1587
68eb42b3
RV		 1588 if (!intel_rps_is_enabled(rps))
1589 return;
1590
9bad2adb 1591 intel_rps_clear_enabled(rps);
8e99299a 1592 intel_rps_clear_interrupts(rps);
36d516be 1593 intel_rps_clear_timer(rps);
3e7abf81 1594
c816723b 1595 if (GRAPHICS_VER(i915) >= 6)
3e7abf81
AS		 1596 gen6_rps_disable(rps);
1597 else if (IS_IRONLAKE_M(i915))
1598 gen5_rps_disable(rps);
1599}
1600
1601static int byt_gpu_freq(struct intel_rps *rps, int val)
1602{
1603 /*
1604 * N = val - 0xb7
1605 * Slow = Fast = GPLL ref * N
1606 */
1607 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1608}
1609
1610static int byt_freq_opcode(struct intel_rps *rps, int val)
1611{
1612 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1613}
1614
1615static int chv_gpu_freq(struct intel_rps *rps, int val)
1616{
1617 /*
1618 * N = val / 2
1619 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1620 */
1621 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1622}
1623
1624static int chv_freq_opcode(struct intel_rps *rps, int val)
1625{
1626 /* CHV needs even values */
1627 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1628}
1629
1630int intel_gpu_freq(struct intel_rps *rps, int val)
1631{
1632 struct drm_i915_private *i915 = rps_to_i915(rps);
1633
c816723b 1634 if (GRAPHICS_VER(i915) >= 9)
3e7abf81
AS		 1635 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1636 GEN9_FREQ_SCALER);
1637 else if (IS_CHERRYVIEW(i915))
1638 return chv_gpu_freq(rps, val);
1639 else if (IS_VALLEYVIEW(i915))
1640 return byt_gpu_freq(rps, val);
c816723b 1641 else if (GRAPHICS_VER(i915) >= 6)
3e7abf81 1642 return val * GT_FREQUENCY_MULTIPLIER;
e82351e7
VS		 1643 else
1644 return val;
3e7abf81
AS		 1645}
1646
1647int intel_freq_opcode(struct intel_rps *rps, int val)
1648{
1649 struct drm_i915_private *i915 = rps_to_i915(rps);
1650
c816723b 1651 if (GRAPHICS_VER(i915) >= 9)
3e7abf81
AS		 1652 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1653 GT_FREQUENCY_MULTIPLIER);
1654 else if (IS_CHERRYVIEW(i915))
1655 return chv_freq_opcode(rps, val);
1656 else if (IS_VALLEYVIEW(i915))
1657 return byt_freq_opcode(rps, val);
c816723b 1658 else if (GRAPHICS_VER(i915) >= 6)
3e7abf81 1659 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
e82351e7
VS		 1660 else
1661 return val;
3e7abf81
AS		 1662}
1663
1664static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1665{
1666 struct drm_i915_private *i915 = rps_to_i915(rps);
1667
1668 rps->gpll_ref_freq =
1669 vlv_get_cck_clock(i915, "GPLL ref",
1670 CCK_GPLL_CLOCK_CONTROL,
1671 i915->czclk_freq);
1672
a8fa7c07
WK		 1673 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1674 rps->gpll_ref_freq);
3e7abf81
AS		 1675}
1676
1677static void vlv_rps_init(struct intel_rps *rps)
1678{
1679 struct drm_i915_private *i915 = rps_to_i915(rps);
1680 u32 val;
1681
1682 vlv_iosf_sb_get(i915,
1683 BIT(VLV_IOSF_SB_PUNIT) |
1684 BIT(VLV_IOSF_SB_NC) |
1685 BIT(VLV_IOSF_SB_CCK));
1686
1687 vlv_init_gpll_ref_freq(rps);
1688
1689 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1690 switch ((val >> 6) & 3) {
1691 case 0:
1692 case 1:
1693 i915->mem_freq = 800;
1694 break;
1695 case 2:
1696 i915->mem_freq = 1066;
1697 break;
1698 case 3:
1699 i915->mem_freq = 1333;
1700 break;
1701 }
a8fa7c07 1702 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
3e7abf81
AS		 1703
1704 rps->max_freq = vlv_rps_max_freq(rps);
1705 rps->rp0_freq = rps->max_freq;
a8fa7c07
WK		 1706 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1707 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
3e7abf81
AS		 1708
1709 rps->efficient_freq = vlv_rps_rpe_freq(rps);
a8fa7c07
WK		 1710 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1711 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
3e7abf81
AS		 1712
1713 rps->rp1_freq = vlv_rps_guar_freq(rps);
a8fa7c07
WK		 1714 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1715 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
3e7abf81
AS		 1716
1717 rps->min_freq = vlv_rps_min_freq(rps);
a8fa7c07
WK		 1718 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1719 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
3e7abf81
AS		 1720
1721 vlv_iosf_sb_put(i915,
1722 BIT(VLV_IOSF_SB_PUNIT) |
1723 BIT(VLV_IOSF_SB_NC) |
1724 BIT(VLV_IOSF_SB_CCK));
1725}
1726
1727static void chv_rps_init(struct intel_rps *rps)
1728{
1729 struct drm_i915_private *i915 = rps_to_i915(rps);
1730 u32 val;
1731
1732 vlv_iosf_sb_get(i915,
1733 BIT(VLV_IOSF_SB_PUNIT) |
1734 BIT(VLV_IOSF_SB_NC) |
1735 BIT(VLV_IOSF_SB_CCK));
1736
1737 vlv_init_gpll_ref_freq(rps);
1738
1739 val = vlv_cck_read(i915, CCK_FUSE_REG);
1740
1741 switch ((val >> 2) & 0x7) {
1742 case 3:
1743 i915->mem_freq = 2000;
1744 break;
1745 default:
1746 i915->mem_freq = 1600;
1747 break;
1748 }
a8fa7c07 1749 drm_dbg(&i915->drm, "DDR speed: %d MHz\n", i915->mem_freq);
3e7abf81
AS		 1750
1751 rps->max_freq = chv_rps_max_freq(rps);
1752 rps->rp0_freq = rps->max_freq;
a8fa7c07
WK		 1753 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1754 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
3e7abf81
AS		 1755
1756 rps->efficient_freq = chv_rps_rpe_freq(rps);
a8fa7c07
WK		 1757 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1758 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
3e7abf81
AS		 1759
1760 rps->rp1_freq = chv_rps_guar_freq(rps);
a8fa7c07
WK		 1761 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1762 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
3e7abf81
AS		 1763
1764 rps->min_freq = chv_rps_min_freq(rps);
a8fa7c07
WK		 1765 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1766 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
3e7abf81
AS		 1767
1768 vlv_iosf_sb_put(i915,
1769 BIT(VLV_IOSF_SB_PUNIT) |
1770 BIT(VLV_IOSF_SB_NC) |
1771 BIT(VLV_IOSF_SB_CCK));
1772
0d4c351a
PB		 1773 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1774 rps->rp1_freq | rps->min_freq) & 1,
1775 "Odd GPU freq values\n");
3e7abf81
AS		 1776}
1777
1778static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1779{
1780 ei->ktime = ktime_get_raw();
1781 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1782 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1783}
1784
1785static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1786{
1787 struct intel_uncore *uncore = rps_to_uncore(rps);
1788 const struct intel_rps_ei *prev = &rps->ei;
1789 struct intel_rps_ei now;
1790 u32 events = 0;
1791
1792 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1793 return 0;
1794
1795 vlv_c0_read(uncore, &now);
1796
1797 if (prev->ktime) {
1798 u64 time, c0;
1799 u32 render, media;
1800
1801 time = ktime_us_delta(now.ktime, prev->ktime);
1802
1803 time *= rps_to_i915(rps)->czclk_freq;
1804
1805 /* Workload can be split between render + media,
1806 * e.g. SwapBuffers being blitted in X after being rendered in
1807 * mesa. To account for this we need to combine both engines
1808 * into our activity counter.
1809 */
1810 render = now.render_c0 - prev->render_c0;
1811 media = now.media_c0 - prev->media_c0;
1812 c0 = max(render, media);
1813 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1814
1815 if (c0 > time * rps->power.up_threshold)
1816 events = GEN6_PM_RP_UP_THRESHOLD;
1817 else if (c0 < time * rps->power.down_threshold)
1818 events = GEN6_PM_RP_DOWN_THRESHOLD;
1819 }
1820
1821 rps->ei = now;
1822 return events;
1823}
1824
1825static void rps_work(struct work_struct *work)
1826{
1827 struct intel_rps *rps = container_of(work, typeof(*rps), work);
1828 struct intel_gt *gt = rps_to_gt(rps);
a8fa7c07 1829 struct drm_i915_private *i915 = rps_to_i915(rps);
3e7abf81
AS		 1830 bool client_boost = false;
1831 int new_freq, adj, min, max;
1832 u32 pm_iir = 0;
1833
03d2c54d 1834 spin_lock_irq(gt->irq_lock);
1ebf7aaf 1835 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
3e7abf81 1836 client_boost = atomic_read(&rps->num_waiters);
03d2c54d 1837 spin_unlock_irq(gt->irq_lock);
3e7abf81
AS		 1838
1839 /* Make sure we didn't queue anything we're not going to process. */
408464b4 1840 if (!pm_iir && !client_boost)
3e7abf81
AS		 1841 goto out;
1842
1843 mutex_lock(&rps->lock);
9bad2adb
CW		 1844 if (!intel_rps_is_active(rps)) {
1845 mutex_unlock(&rps->lock);
1846 return;
1847 }
3e7abf81
AS		 1848
1849 pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1850
1851 adj = rps->last_adj;
1852 new_freq = rps->cur_freq;
1853 min = rps->min_freq_softlimit;
1854 max = rps->max_freq_softlimit;
1855 if (client_boost)
1856 max = rps->max_freq;
555a3224
CW		 1857
1858 GT_TRACE(gt,
1859 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
01fabda8 1860 pm_iir, str_yes_no(client_boost),
555a3224
CW		 1861 adj, new_freq, min, max);
1862
3e7abf81
AS		 1863 if (client_boost && new_freq < rps->boost_freq) {
1864 new_freq = rps->boost_freq;
1865 adj = 0;
1866 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1867 if (adj > 0)
1868 adj *= 2;
1869 else /* CHV needs even encode values */
1870 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1871
1872 if (new_freq >= rps->max_freq_softlimit)
1873 adj = 0;
1874 } else if (client_boost) {
1875 adj = 0;
1876 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1877 if (rps->cur_freq > rps->efficient_freq)
1878 new_freq = rps->efficient_freq;
1879 else if (rps->cur_freq > rps->min_freq_softlimit)
1880 new_freq = rps->min_freq_softlimit;
1881 adj = 0;
1882 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1883 if (adj < 0)
1884 adj *= 2;
1885 else /* CHV needs even encode values */
1886 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1887
1888 if (new_freq <= rps->min_freq_softlimit)
1889 adj = 0;
1890 } else { /* unknown event */
1891 adj = 0;
1892 }
1893
3e7abf81 1894 /*
de3b4d93
CW		 1895 * sysfs frequency limits may have snuck in while
1896 * servicing the interrupt
3e7abf81
AS		 1897 */
1898 new_freq += adj;
1899 new_freq = clamp_t(int, new_freq, min, max);
1900
1901 if (intel_rps_set(rps, new_freq)) {
a8fa7c07 1902 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
de3b4d93 1903 adj = 0;
3e7abf81 1904 }
de3b4d93 1905 rps->last_adj = adj;
3e7abf81
AS		 1906
1907 mutex_unlock(&rps->lock);
1908
1909out:
03d2c54d 1910 spin_lock_irq(gt->irq_lock);
3e7abf81 1911 gen6_gt_pm_unmask_irq(gt, rps->pm_events);
03d2c54d 1912 spin_unlock_irq(gt->irq_lock);
3e7abf81
AS		 1913}
1914
1915void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1916{
1917 struct intel_gt *gt = rps_to_gt(rps);
1918 const u32 events = rps->pm_events & pm_iir;
1919
03d2c54d 1920 lockdep_assert_held(gt->irq_lock);
3e7abf81
AS		 1921
1922 if (unlikely(!events))
1923 return;
1924
555a3224
CW		 1925 GT_TRACE(gt, "irq events:%x\n", events);
1926
3e7abf81
AS		 1927 gen6_gt_pm_mask_irq(gt, events);
1928
1929 rps->pm_iir |= events;
1930 schedule_work(&rps->work);
1931}
1932
1933void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1934{
5a3e2b82 1935 struct intel_gt *gt = rps_to_gt(rps);
408464b4 1936 u32 events;
3e7abf81 1937
1ebf7aaf 1938 events = pm_iir & rps->pm_events;
408464b4 1939 if (events) {
03d2c54d 1940 spin_lock(gt->irq_lock);
408464b4 1941
555a3224
CW		 1942 GT_TRACE(gt, "irq events:%x\n", events);
1943
408464b4
CW		 1944 gen6_gt_pm_mask_irq(gt, events);
1945 rps->pm_iir |= events;
1946
3e7abf81 1947 schedule_work(&rps->work);
03d2c54d 1948 spin_unlock(gt->irq_lock);
3e7abf81
AS		 1949 }
1950
c816723b 1951 if (GRAPHICS_VER(gt->i915) >= 8)
3e7abf81
AS		 1952 return;
1953
1954 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
0669a6e1 1955 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
3e7abf81
AS		 1956
1957 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
a10234fd
TU		 1958 drm_dbg(&rps_to_i915(rps)->drm,
1959 "Command parser error, pm_iir 0x%08x\n", pm_iir);
3e7abf81
AS		 1960}
1961
1962void gen5_rps_irq_handler(struct intel_rps *rps)
1963{
1964 struct intel_uncore *uncore = rps_to_uncore(rps);
1965 u32 busy_up, busy_down, max_avg, min_avg;
1966 u8 new_freq;
1967
1968 spin_lock(&mchdev_lock);
1969
1970 intel_uncore_write16(uncore,
1971 MEMINTRSTS,
1972 intel_uncore_read(uncore, MEMINTRSTS));
1973
1974 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1975 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1976 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1977 max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1978 min_avg = intel_uncore_read(uncore, RCBMINAVG);
1979
1980 /* Handle RCS change request from hw */
1981 new_freq = rps->cur_freq;
1982 if (busy_up > max_avg)
1983 new_freq++;
1984 else if (busy_down < min_avg)
1985 new_freq--;
1986 new_freq = clamp(new_freq,
1987 rps->min_freq_softlimit,
1988 rps->max_freq_softlimit);
1989
4ee73792 1990 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
3e7abf81
AS		 1991 rps->cur_freq = new_freq;
1992
1993 spin_unlock(&mchdev_lock);
1994}
1995
a06375a9 1996void intel_rps_init_early(struct intel_rps *rps)
3e7abf81 1997{
3e7abf81
AS		 1998 mutex_init(&rps->lock);
1999 mutex_init(&rps->power.mutex);
2000
2001 INIT_WORK(&rps->work, rps_work);
36d516be 2002 timer_setup(&rps->timer, rps_timer, 0);
3e7abf81
AS		 2003
2004 atomic_set(&rps->num_waiters, 0);
a06375a9
CW		 2005}
2006
2007void intel_rps_init(struct intel_rps *rps)
2008{
2009 struct drm_i915_private *i915 = rps_to_i915(rps);
3e7abf81 2010
7ba79a67
VB		 2011 if (rps_uses_slpc(rps))
2012 return;
2013
3e7abf81
AS		 2014 if (IS_CHERRYVIEW(i915))
2015 chv_rps_init(rps);
2016 else if (IS_VALLEYVIEW(i915))
2017 vlv_rps_init(rps);
c816723b 2018 else if (GRAPHICS_VER(i915) >= 6)
3e7abf81
AS		 2019 gen6_rps_init(rps);
2020 else if (IS_IRONLAKE_M(i915))
2021 gen5_rps_init(rps);
2022
2023 /* Derive initial user preferences/limits from the hardware limits */
2024 rps->max_freq_softlimit = rps->max_freq;
fdff0a85 2025 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
3e7abf81 2026 rps->min_freq_softlimit = rps->min_freq;
fdff0a85 2027 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
3e7abf81
AS		 2028
2029 /* After setting max-softlimit, find the overclock max freq */
c816723b 2030 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
3e7abf81
AS		 2031 u32 params = 0;
2032
ee421bb4 2033 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, &params, NULL);
3e7abf81 2034 if (params & BIT(31)) { /* OC supported */
a8fa7c07
WK		 2035 drm_dbg(&i915->drm,
2036 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2037 (rps->max_freq & 0xff) * 50,
2038 (params & 0xff) * 50);
3e7abf81
AS		 2039 rps->max_freq = params & 0xff;
2040 }
2041 }
2042
2043 /* Finally allow us to boost to max by default */
2044 rps->boost_freq = rps->max_freq;
2045 rps->idle_freq = rps->min_freq;
043cd2d1
CW		 2046
2047 /* Start in the middle, from here we will autotune based on workload */
2048 rps->cur_freq = rps->efficient_freq;
3e7abf81
AS		 2049
2050 rps->pm_intrmsk_mbz = 0;
2051
2052 /*
2053 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2054 * if GEN6_PM_UP_EI_EXPIRED is masked.
2055 *
2056 * TODO: verify if this can be reproduced on VLV,CHV.
2057 */
c816723b 2058 if (GRAPHICS_VER(i915) <= 7)
3e7abf81
AS		 2059 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2060
c816723b 2061 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
3e7abf81 2062 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
933864af
MB		 2063
2064 /* GuC needs ARAT expired interrupt unmasked */
2065 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2066 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
389b7f00 2067}
8e99299a 2068
389b7f00
CW		 2069void intel_rps_sanitize(struct intel_rps *rps)
2070{
7ba79a67
VB		 2071 if (rps_uses_slpc(rps))
2072 return;
2073
c816723b 2074 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
8e99299a 2075 rps_disable_interrupts(rps);
3e7abf81
AS		 2076}
2077
01b8c2e6
DH		 2078u32 intel_rps_read_rpstat(struct intel_rps *rps)
2079{
2080 struct drm_i915_private *i915 = rps_to_i915(rps);
2081 i915_reg_t rpstat;
2082
2083 rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1;
2084
2085 return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat);
2086}
2087
12d4eb20 2088static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
3e7abf81
AS		 2089{
2090 struct drm_i915_private *i915 = rps_to_i915(rps);
2091 u32 cagf;
2092
22009b6d
BN		 2093 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2094 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat);
2095 else if (GRAPHICS_VER(i915) >= 12)
01b8c2e6
DH		 2096 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat);
2097 else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2c0a284c 2098 cagf = REG_FIELD_GET(RPE_MASK, rpstat);
c816723b 2099 else if (GRAPHICS_VER(i915) >= 9)
2c0a284c 2100 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat);
3e7abf81 2101 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2c0a284c 2102 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat);
c816723b 2103 else if (GRAPHICS_VER(i915) >= 6)
2c0a284c 2104 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat);
e82351e7 2105 else
2c0a284c 2106 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat));
3e7abf81 2107
e03512ed
AS		 2108 return cagf;
2109}
2110
12d4eb20 2111static u32 __read_cagf(struct intel_rps *rps, bool take_fw)
e03512ed
AS		 2112{
2113 struct drm_i915_private *i915 = rps_to_i915(rps);
e82351e7 2114 struct intel_uncore *uncore = rps_to_uncore(rps);
12d4eb20 2115 i915_reg_t r = INVALID_MMIO_REG;
e03512ed
AS		 2116 u32 freq;
2117
22009b6d
BN		 2118 /*
2119 * For Gen12+ reading freq from HW does not need a forcewake and
2120 * registers will return 0 freq when GT is in RC6
2121 */
2122 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
12d4eb20 2123 r = MTL_MIRROR_TARGET_WP1;
22009b6d 2124 } else if (GRAPHICS_VER(i915) >= 12) {
12d4eb20 2125 r = GEN12_RPSTAT1;
01b8c2e6 2126 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
e03512ed
AS		 2127 vlv_punit_get(i915);
2128 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2129 vlv_punit_put(i915);
c816723b 2130 } else if (GRAPHICS_VER(i915) >= 6) {
12d4eb20 2131 r = GEN6_RPSTAT1;
e03512ed 2132 } else {
12d4eb20 2133 r = MEMSTAT_ILK;
e03512ed
AS		 2134 }
2135
12d4eb20
AD		 2136 if (i915_mmio_reg_valid(r))
2137 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r);
2138
e03512ed
AS		 2139 return intel_rps_get_cagf(rps, freq);
2140}
2141
12d4eb20
AD		 2142static u32 read_cagf(struct intel_rps *rps)
2143{
2144 return __read_cagf(rps, true);
2145}
2146
e03512ed
AS		 2147u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2148{
9c878557 2149 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
e03512ed
AS		 2150 intel_wakeref_t wakeref;
2151 u32 freq = 0;
2152
2153 with_intel_runtime_pm_if_in_use(rpm, wakeref)
2154 freq = intel_gpu_freq(rps, read_cagf(rps));
2155
2156 return freq;
3e7abf81
AS		 2157}
2158
12d4eb20
AD		 2159u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)
2160{
2161 return intel_gpu_freq(rps, __read_cagf(rps, false));
2162}
2163
2164static u32 intel_rps_read_punit_req(struct intel_rps *rps)
41e5c17e
VB		 2165{
2166 struct intel_uncore *uncore = rps_to_uncore(rps);
f25e3908
VB		 2167 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2168 intel_wakeref_t wakeref;
2169 u32 freq = 0;
41e5c17e 2170
f25e3908
VB		 2171 with_intel_runtime_pm_if_in_use(rpm, wakeref)
2172 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2173
2174 return freq;
41e5c17e
VB		 2175}
2176
2177static u32 intel_rps_get_req(u32 pureq)
2178{
2179 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2180
2181 return req;
2182}
2183
2184u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2185{
2186 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2187
2188 return intel_gpu_freq(rps, freq);
2189}
2190
2191u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2192{
2193 if (rps_uses_slpc(rps))
2194 return intel_rps_read_punit_req_frequency(rps);
2195 else
2196 return intel_gpu_freq(rps, rps->cur_freq);
2197}
2198
2199u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2200{
2201 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2202
2203 if (rps_uses_slpc(rps))
2204 return slpc->max_freq_softlimit;
2205 else
2206 return intel_gpu_freq(rps, rps->max_freq_softlimit);
2207}
2208
018a7bdb
RV		 2209/**
2210 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2211 * @rps: the intel_rps structure
2212 *
2213 * Returns the max frequency in a raw format. In newer platforms raw is in
2214 * units of 50 MHz.
2215 */
2216u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2217{
2218 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2219 u32 freq;
2220
2221 if (rps_uses_slpc(rps)) {
2222 return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2223 GT_FREQUENCY_MULTIPLIER);
2224 } else {
2225 freq = rps->max_freq;
2226 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2227 /* Convert GT frequency to 50 MHz units */
2228 freq /= GEN9_FREQ_SCALER;
2229 }
2230 return freq;
2231 }
2232}
2233
41e5c17e
VB		 2234u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2235{
2236 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2237
2238 if (rps_uses_slpc(rps))
2239 return slpc->rp0_freq;
2240 else
2241 return intel_gpu_freq(rps, rps->rp0_freq);
2242}
2243
2244u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2245{
2246 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2247
2248 if (rps_uses_slpc(rps))
2249 return slpc->rp1_freq;
2250 else
2251 return intel_gpu_freq(rps, rps->rp1_freq);
2252}
2253
2254u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2255{
2256 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2257
2258 if (rps_uses_slpc(rps))
2259 return slpc->min_freq;
2260 else
2261 return intel_gpu_freq(rps, rps->min_freq);
2262}
2263
83d495a5 2264static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
cf51cc7b
VB		 2265{
2266 struct intel_gt *gt = rps_to_gt(rps);
2267 struct drm_i915_private *i915 = gt->i915;
2268 struct intel_uncore *uncore = gt->uncore;
2269 struct intel_rps_freq_caps caps;
2270 u32 rp_state_limits;
2271 u32 gt_perf_status;
2272 u32 rpmodectl, rpinclimit, rpdeclimit;
2273 u32 rpstat, cagf, reqf;
2274 u32 rpcurupei, rpcurup, rpprevup;
2275 u32 rpcurdownei, rpcurdown, rpprevdown;
2276 u32 rpupei, rpupt, rpdownei, rpdownt;
2277 u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
2278
2279 rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS);
2280 gen6_rps_get_freq_caps(rps, &caps);
2281 if (IS_GEN9_LP(i915))
2282 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS);
2283 else
2284 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS);
2285
2286 /* RPSTAT1 is in the GT power well */
2287 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
2288
2289 reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2290 if (GRAPHICS_VER(i915) >= 9) {
2291 reqf >>= 23;
2292 } else {
2293 reqf &= ~GEN6_TURBO_DISABLE;
2294 if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2295 reqf >>= 24;
2296 else
2297 reqf >>= 25;
2298 }
2299 reqf = intel_gpu_freq(rps, reqf);
2300
2301 rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL);
2302 rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2303 rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2304
01b8c2e6 2305 rpstat = intel_rps_read_rpstat(rps);
cf51cc7b
VB		 2306 rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK;
2307 rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK;
2308 rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK;
2309 rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
2310 rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
2311 rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
2312
2313 rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI);
2314 rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2315
2316 rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI);
2317 rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2318
2319 cagf = intel_rps_read_actual_frequency(rps);
2320
2321 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
2322
2323 if (GRAPHICS_VER(i915) >= 11) {
2324 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE);
2325 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK);
2326 /*
2327 * The equivalent to the PM ISR & IIR cannot be read
2328 * without affecting the current state of the system
2329 */
2330 pm_isr = 0;
2331 pm_iir = 0;
2332 } else if (GRAPHICS_VER(i915) >= 8) {
2333 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2));
2334 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2));
2335 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2));
2336 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2));
2337 } else {
2338 pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
2339 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
2340 pm_isr = intel_uncore_read(uncore, GEN6_PMISR);
2341 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR);
2342 }
2343 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2344
2345 drm_printf(p, "Video Turbo Mode: %s\n",
2346 str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO));
2347 drm_printf(p, "HW control enabled: %s\n",
2348 str_yes_no(rpmodectl & GEN6_RP_ENABLE));
2349 drm_printf(p, "SW control enabled: %s\n",
2350 str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE));
2351
2352 drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n",
2353 pm_ier, pm_imr, pm_mask);
2354 if (GRAPHICS_VER(i915) <= 10)
2355 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n",
2356 pm_isr, pm_iir);
2357 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2358 rps->pm_intrmsk_mbz);
2359 drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
2360 drm_printf(p, "Render p-state ratio: %d\n",
2361 (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8);
2362 drm_printf(p, "Render p-state VID: %d\n",
2363 gt_perf_status & 0xff);
2364 drm_printf(p, "Render p-state limit: %d\n",
2365 rp_state_limits & 0xff);
2366 drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat);
2367 drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl);
2368 drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit);
2369 drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit);
2370 drm_printf(p, "RPNSWREQ: %dMHz\n", reqf);
2371 drm_printf(p, "CAGF: %dMHz\n", cagf);
2372 drm_printf(p, "RP CUR UP EI: %d (%lldns)\n",
2373 rpcurupei,
2374 intel_gt_pm_interval_to_ns(gt, rpcurupei));
2375 drm_printf(p, "RP CUR UP: %d (%lldns)\n",
2376 rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup));
2377 drm_printf(p, "RP PREV UP: %d (%lldns)\n",
2378 rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup));
2379 drm_printf(p, "Up threshold: %d%%\n",
2380 rps->power.up_threshold);
2381 drm_printf(p, "RP UP EI: %d (%lldns)\n",
2382 rpupei, intel_gt_pm_interval_to_ns(gt, rpupei));
2383 drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n",
2384 rpupt, intel_gt_pm_interval_to_ns(gt, rpupt));
2385
2386 drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n",
2387 rpcurdownei,
2388 intel_gt_pm_interval_to_ns(gt, rpcurdownei));
2389 drm_printf(p, "RP CUR DOWN: %d (%lldns)\n",
2390 rpcurdown,
2391 intel_gt_pm_interval_to_ns(gt, rpcurdown));
2392 drm_printf(p, "RP PREV DOWN: %d (%lldns)\n",
2393 rpprevdown,
2394 intel_gt_pm_interval_to_ns(gt, rpprevdown));
2395 drm_printf(p, "Down threshold: %d%%\n",
2396 rps->power.down_threshold);
2397 drm_printf(p, "RP DOWN EI: %d (%lldns)\n",
2398 rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei));
2399 drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n",
2400 rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt));
2401
2402 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2403 intel_gpu_freq(rps, caps.min_freq));
2404 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2405 intel_gpu_freq(rps, caps.rp1_freq));
2406 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2407 intel_gpu_freq(rps, caps.rp0_freq));
2408 drm_printf(p, "Max overclocked frequency: %dMHz\n",
2409 intel_gpu_freq(rps, rps->max_freq));
2410
2411 drm_printf(p, "Current freq: %d MHz\n",
2412 intel_gpu_freq(rps, rps->cur_freq));
2413 drm_printf(p, "Actual freq: %d MHz\n", cagf);
2414 drm_printf(p, "Idle freq: %d MHz\n",
2415 intel_gpu_freq(rps, rps->idle_freq));
2416 drm_printf(p, "Min freq: %d MHz\n",
2417 intel_gpu_freq(rps, rps->min_freq));
2418 drm_printf(p, "Boost freq: %d MHz\n",
2419 intel_gpu_freq(rps, rps->boost_freq));
2420 drm_printf(p, "Max freq: %d MHz\n",
2421 intel_gpu_freq(rps, rps->max_freq));
2422 drm_printf(p,
2423 "efficient (RPe) frequency: %d MHz\n",
2424 intel_gpu_freq(rps, rps->efficient_freq));
2425}
2426
83d495a5
VB		 2427static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2428{
2429 struct intel_gt *gt = rps_to_gt(rps);
2430 struct intel_uncore *uncore = gt->uncore;
2431 struct intel_rps_freq_caps caps;
2432 u32 pm_mask;
2433
2434 gen6_rps_get_freq_caps(rps, &caps);
2435 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2436
2437 drm_printf(p, "PM MASK=0x%08x\n", pm_mask);
2438 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2439 rps->pm_intrmsk_mbz);
01b8c2e6 2440 drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps));
83d495a5
VB		 2441 drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps));
2442 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2443 intel_gpu_freq(rps, caps.min_freq));
2444 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2445 intel_gpu_freq(rps, caps.rp1_freq));
2446 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2447 intel_gpu_freq(rps, caps.rp0_freq));
2448 drm_printf(p, "Current freq: %d MHz\n",
2449 intel_rps_get_requested_frequency(rps));
2450 drm_printf(p, "Actual freq: %d MHz\n",
2451 intel_rps_read_actual_frequency(rps));
2452 drm_printf(p, "Min freq: %d MHz\n",
2453 intel_rps_get_min_frequency(rps));
2454 drm_printf(p, "Boost freq: %d MHz\n",
2455 intel_rps_get_boost_frequency(rps));
2456 drm_printf(p, "Max freq: %d MHz\n",
2457 intel_rps_get_max_frequency(rps));
2458 drm_printf(p,
2459 "efficient (RPe) frequency: %d MHz\n",
2460 intel_gpu_freq(rps, caps.rp1_freq));
2461}
2462
2463void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2464{
2465 if (rps_uses_slpc(rps))
2466 return slpc_frequency_dump(rps, p);
2467 else
2468 return rps_frequency_dump(rps, p);
2469}
2470
41e5c17e
VB		 2471static int set_max_freq(struct intel_rps *rps, u32 val)
2472{
2473 struct drm_i915_private *i915 = rps_to_i915(rps);
2474 int ret = 0;
2475
2476 mutex_lock(&rps->lock);
2477
2478 val = intel_freq_opcode(rps, val);
2479 if (val < rps->min_freq ||
2480 val > rps->max_freq ||
2481 val < rps->min_freq_softlimit) {
2482 ret = -EINVAL;
2483 goto unlock;
2484 }
2485
2486 if (val > rps->rp0_freq)
2487 drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2488 intel_gpu_freq(rps, val));
2489
2490 rps->max_freq_softlimit = val;
2491
2492 val = clamp_t(int, rps->cur_freq,
2493 rps->min_freq_softlimit,
2494 rps->max_freq_softlimit);
2495
2496 /*
2497 * We still need *_set_rps to process the new max_delay and
2498 * update the interrupt limits and PMINTRMSK even though
2499 * frequency request may be unchanged.
2500 */
2501 intel_rps_set(rps, val);
2502
2503unlock:
2504 mutex_unlock(&rps->lock);
2505
2506 return ret;
2507}
2508
2509int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2510{
2511 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2512
2513 if (rps_uses_slpc(rps))
2514 return intel_guc_slpc_set_max_freq(slpc, val);
2515 else
2516 return set_max_freq(rps, val);
2517}
2518
2519u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2520{
2521 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2522
2523 if (rps_uses_slpc(rps))
2524 return slpc->min_freq_softlimit;
2525 else
2526 return intel_gpu_freq(rps, rps->min_freq_softlimit);
2527}
2528
018a7bdb
RV		 2529/**
2530 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2531 * @rps: the intel_rps structure
2532 *
2533 * Returns the min frequency in a raw format. In newer platforms raw is in
2534 * units of 50 MHz.
2535 */
2536u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2537{
2538 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2539 u32 freq;
2540
2541 if (rps_uses_slpc(rps)) {
2542 return DIV_ROUND_CLOSEST(slpc->min_freq,
2543 GT_FREQUENCY_MULTIPLIER);
2544 } else {
2545 freq = rps->min_freq;
2546 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2547 /* Convert GT frequency to 50 MHz units */
2548 freq /= GEN9_FREQ_SCALER;
2549 }
2550 return freq;
2551 }
2552}
2553
41e5c17e
VB		 2554static int set_min_freq(struct intel_rps *rps, u32 val)
2555{
2556 int ret = 0;
2557
2558 mutex_lock(&rps->lock);
2559
2560 val = intel_freq_opcode(rps, val);
2561 if (val < rps->min_freq ||
2562 val > rps->max_freq ||
2563 val > rps->max_freq_softlimit) {
2564 ret = -EINVAL;
2565 goto unlock;
2566 }
2567
2568 rps->min_freq_softlimit = val;
2569
2570 val = clamp_t(int, rps->cur_freq,
2571 rps->min_freq_softlimit,
2572 rps->max_freq_softlimit);
2573
2574 /*
2575 * We still need *_set_rps to process the new min_delay and
2576 * update the interrupt limits and PMINTRMSK even though
2577 * frequency request may be unchanged.
2578 */
2579 intel_rps_set(rps, val);
2580
2581unlock:
2582 mutex_unlock(&rps->lock);
2583
2584 return ret;
2585}
2586
2587int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2588{
2589 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2590
2591 if (rps_uses_slpc(rps))
2592 return intel_guc_slpc_set_min_freq(slpc, val);
2593 else
2594 return set_min_freq(rps, val);
2595}
2596
1c40d40f
VB		 2597static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2598{
2599 struct intel_uncore *uncore = rps_to_uncore(rps);
2600 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2601
2602 /* Allow punit to process software requests */
2603 intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2604}
2605
2606void intel_rps_raise_unslice(struct intel_rps *rps)
2607{
2608 struct intel_uncore *uncore = rps_to_uncore(rps);
1c40d40f
VB		 2609
2610 mutex_lock(&rps->lock);
2611
2612 if (rps_uses_slpc(rps)) {
2613 /* RP limits have not been initialized yet for SLPC path */
56758cc4
AD		 2614 struct intel_rps_freq_caps caps;
2615
2616 gen6_rps_get_freq_caps(rps, &caps);
1c40d40f
VB		 2617
2618 intel_rps_set_manual(rps, true);
2619 intel_uncore_write(uncore, GEN6_RPNSWREQ,
56758cc4 2620 ((caps.rp0_freq <<
1c40d40f
VB		 2621 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2622 GEN9_IGNORE_SLICE_RATIO));
2623 intel_rps_set_manual(rps, false);
2624 } else {
2625 intel_rps_set(rps, rps->rp0_freq);
2626 }
2627
2628 mutex_unlock(&rps->lock);
2629}
2630
2631void intel_rps_lower_unslice(struct intel_rps *rps)
2632{
2633 struct intel_uncore *uncore = rps_to_uncore(rps);
1c40d40f
VB		 2634
2635 mutex_lock(&rps->lock);
2636
2637 if (rps_uses_slpc(rps)) {
2638 /* RP limits have not been initialized yet for SLPC path */
56758cc4
AD		 2639 struct intel_rps_freq_caps caps;
2640
2641 gen6_rps_get_freq_caps(rps, &caps);
1c40d40f
VB		 2642
2643 intel_rps_set_manual(rps, true);
2644 intel_uncore_write(uncore, GEN6_RPNSWREQ,
56758cc4 2645 ((caps.min_freq <<
1c40d40f
VB		 2646 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2647 GEN9_IGNORE_SLICE_RATIO));
2648 intel_rps_set_manual(rps, false);
2649 } else {
2650 intel_rps_set(rps, rps->min_freq);
2651 }
2652
2653 mutex_unlock(&rps->lock);
2654}
2655
fa68bff7
SS		 2656static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2657{
2658 struct intel_gt *gt = rps_to_gt(rps);
2659 intel_wakeref_t wakeref;
2660 u32 val;
2661
2662 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2663 val = intel_uncore_read(gt->uncore, reg32);
2664
2665 return val;
2666}
2667
2668bool rps_read_mask_mmio(struct intel_rps *rps,
2669 i915_reg_t reg32, u32 mask)
2670{
2671 return rps_read_mmio(rps, reg32) & mask;
2672}
2673
3e7abf81
AS		 2674/* External interface for intel_ips.ko */
2675
2676static struct drm_i915_private __rcu *ips_mchdev;
2677
2678/**
2679 * Tells the intel_ips driver that the i915 driver is now loaded, if
2680 * IPS got loaded first.
2681 *
2682 * This awkward dance is so that neither module has to depend on the
2683 * other in order for IPS to do the appropriate communication of
2684 * GPU turbo limits to i915.
2685 */
2686static void
2687ips_ping_for_i915_load(void)
2688{
2689 void (*link)(void);
2690
2691 link = symbol_get(ips_link_to_i915_driver);
2692 if (link) {
2693 link();
2694 symbol_put(ips_link_to_i915_driver);
2695 }
2696}
2697
2698void intel_rps_driver_register(struct intel_rps *rps)
2699{
2700 struct intel_gt *gt = rps_to_gt(rps);
2701
2702 /*
2703 * We only register the i915 ips part with intel-ips once everything is
2704 * set up, to avoid intel-ips sneaking in and reading bogus values.
2705 */
c816723b 2706 if (GRAPHICS_VER(gt->i915) == 5) {
c0168a3e 2707 GEM_BUG_ON(ips_mchdev);
3e7abf81
AS		 2708 rcu_assign_pointer(ips_mchdev, gt->i915);
2709 ips_ping_for_i915_load();
2710 }
2711}
2712
2713void intel_rps_driver_unregister(struct intel_rps *rps)
2714{
ad3662e2 2715 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
c0168a3e 2716 rcu_assign_pointer(ips_mchdev, NULL);
3e7abf81
AS		 2717}
2718
2719static struct drm_i915_private *mchdev_get(void)
2720{
2721 struct drm_i915_private *i915;
2722
2723 rcu_read_lock();
2724 i915 = rcu_dereference(ips_mchdev);
6cb304b3 2725 if (i915 && !kref_get_unless_zero(&i915->drm.ref))
3e7abf81
AS		 2726 i915 = NULL;
2727 rcu_read_unlock();
2728
2729 return i915;
2730}
2731
2732/**
2733 * i915_read_mch_val - return value for IPS use
2734 *
2735 * Calculate and return a value for the IPS driver to use when deciding whether
2736 * we have thermal and power headroom to increase CPU or GPU power budget.
2737 */
2738unsigned long i915_read_mch_val(void)
2739{
2740 struct drm_i915_private *i915;
2741 unsigned long chipset_val = 0;
2742 unsigned long graphics_val = 0;
2743 intel_wakeref_t wakeref;
2744
2745 i915 = mchdev_get();
2746 if (!i915)
2747 return 0;
2748
2749 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
c14adcbd 2750 struct intel_ips *ips = &to_gt(i915)->rps.ips;
3e7abf81
AS		 2751
2752 spin_lock_irq(&mchdev_lock);
2753 chipset_val = __ips_chipset_val(ips);
2754 graphics_val = __ips_gfx_val(ips);
2755 spin_unlock_irq(&mchdev_lock);
2756 }
2757
2758 drm_dev_put(&i915->drm);
2759 return chipset_val + graphics_val;
2760}
2761EXPORT_SYMBOL_GPL(i915_read_mch_val);
2762
2763/**
2764 * i915_gpu_raise - raise GPU frequency limit
2765 *
2766 * Raise the limit; IPS indicates we have thermal headroom.
2767 */
2768bool i915_gpu_raise(void)
2769{
2770 struct drm_i915_private *i915;
2771 struct intel_rps *rps;
2772
2773 i915 = mchdev_get();
2774 if (!i915)
2775 return false;
2776
c14adcbd 2777 rps = &to_gt(i915)->rps;
3e7abf81
AS		 2778
2779 spin_lock_irq(&mchdev_lock);
2780 if (rps->max_freq_softlimit < rps->max_freq)
2781 rps->max_freq_softlimit++;
2782 spin_unlock_irq(&mchdev_lock);
2783
2784 drm_dev_put(&i915->drm);
2785 return true;
2786}
2787EXPORT_SYMBOL_GPL(i915_gpu_raise);
2788
2789/**
2790 * i915_gpu_lower - lower GPU frequency limit
2791 *
2792 * IPS indicates we're close to a thermal limit, so throttle back the GPU
2793 * frequency maximum.
2794 */
2795bool i915_gpu_lower(void)
2796{
2797 struct drm_i915_private *i915;
2798 struct intel_rps *rps;
2799
2800 i915 = mchdev_get();
2801 if (!i915)
2802 return false;
2803
c14adcbd 2804 rps = &to_gt(i915)->rps;
3e7abf81
AS		 2805
2806 spin_lock_irq(&mchdev_lock);
2807 if (rps->max_freq_softlimit > rps->min_freq)
2808 rps->max_freq_softlimit--;
2809 spin_unlock_irq(&mchdev_lock);
2810
2811 drm_dev_put(&i915->drm);
2812 return true;
2813}
2814EXPORT_SYMBOL_GPL(i915_gpu_lower);
2815
2816/**
2817 * i915_gpu_busy - indicate GPU business to IPS
2818 *
2819 * Tell the IPS driver whether or not the GPU is busy.
2820 */
2821bool i915_gpu_busy(void)
2822{
2823 struct drm_i915_private *i915;
2824 bool ret;
2825
2826 i915 = mchdev_get();
2827 if (!i915)
2828 return false;
2829
c14adcbd 2830 ret = to_gt(i915)->awake;
3e7abf81
AS		 2831
2832 drm_dev_put(&i915->drm);
2833 return ret;
2834}
2835EXPORT_SYMBOL_GPL(i915_gpu_busy);
2836
2837/**
2838 * i915_gpu_turbo_disable - disable graphics turbo
2839 *
2840 * Disable graphics turbo by resetting the max frequency and setting the
2841 * current frequency to the default.
2842 */
2843bool i915_gpu_turbo_disable(void)
2844{
2845 struct drm_i915_private *i915;
2846 struct intel_rps *rps;
2847 bool ret;
2848
2849 i915 = mchdev_get();
2850 if (!i915)
2851 return false;
2852
c14adcbd 2853 rps = &to_gt(i915)->rps;
3e7abf81
AS		 2854
2855 spin_lock_irq(&mchdev_lock);
2856 rps->max_freq_softlimit = rps->min_freq;
c14adcbd 2857 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
3e7abf81
AS		 2858 spin_unlock_irq(&mchdev_lock);
2859
2860 drm_dev_put(&i915->drm);
2861 return ret;
2862}
2863EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
46495adc
CW		 2864
2865#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2866#include "selftest_rps.c"
8ee2c227 2867#include "selftest_slpc.c"
46495adc 2868#endif
Linux 6.x block layer and io_uring tree(s)