Commit | Line | Data |
---|---|---|
c1132367 AS |
1 | /* |
2 | * SPDX-License-Identifier: MIT | |
3 | * | |
4 | * Copyright © 2019 Intel Corporation | |
5 | */ | |
6 | ||
7 | #include <linux/pm_runtime.h> | |
8 | ||
9 | #include "i915_drv.h" | |
9e859eb9 | 10 | #include "i915_vgpu.h" |
c1132367 AS |
11 | #include "intel_gt.h" |
12 | #include "intel_gt_pm.h" | |
13 | #include "intel_rc6.h" | |
14 | #include "intel_sideband.h" | |
15 | ||
16 | /** | |
17 | * DOC: RC6 | |
18 | * | |
19 | * RC6 is a special power stage which allows the GPU to enter an very | |
20 | * low-voltage mode when idle, using down to 0V while at this stage. This | |
21 | * stage is entered automatically when the GPU is idle when RC6 support is | |
22 | * enabled, and as soon as new workload arises GPU wakes up automatically as | |
23 | * well. | |
24 | * | |
25 | * There are different RC6 modes available in Intel GPU, which differentiate | |
26 | * among each other with the latency required to enter and leave RC6 and | |
27 | * voltage consumed by the GPU in different states. | |
28 | * | |
29 | * The combination of the following flags define which states GPU is allowed | |
30 | * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and | |
31 | * RC6pp is deepest RC6. Their support by hardware varies according to the | |
32 | * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one | |
33 | * which brings the most power savings; deeper states save more power, but | |
34 | * require higher latency to switch to and wake up. | |
35 | */ | |
36 | ||
37 | static struct intel_gt *rc6_to_gt(struct intel_rc6 *rc6) | |
38 | { | |
39 | return container_of(rc6, struct intel_gt, rc6); | |
40 | } | |
41 | ||
42 | static struct intel_uncore *rc6_to_uncore(struct intel_rc6 *rc) | |
43 | { | |
44 | return rc6_to_gt(rc)->uncore; | |
45 | } | |
46 | ||
47 | static struct drm_i915_private *rc6_to_i915(struct intel_rc6 *rc) | |
48 | { | |
49 | return rc6_to_gt(rc)->i915; | |
50 | } | |
51 | ||
9834dfef | 52 | static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) |
c1132367 AS |
53 | { |
54 | intel_uncore_write_fw(uncore, reg, val); | |
55 | } | |
56 | ||
57 | static void gen11_rc6_enable(struct intel_rc6 *rc6) | |
58 | { | |
695dc55b RV |
59 | struct intel_gt *gt = rc6_to_gt(rc6); |
60 | struct intel_uncore *uncore = gt->uncore; | |
c1132367 AS |
61 | struct intel_engine_cs *engine; |
62 | enum intel_engine_id id; | |
695dc55b RV |
63 | u32 pg_enable; |
64 | int i; | |
c1132367 AS |
65 | |
66 | /* 2b: Program RC6 thresholds.*/ | |
67 | set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); | |
68 | set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); | |
69 | ||
70 | set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ | |
71 | set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ | |
c6e07ada | 72 | for_each_engine(engine, rc6_to_gt(rc6), id) |
c1132367 AS |
73 | set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); |
74 | ||
75 | set(uncore, GUC_MAX_IDLE_COUNT, 0xA); | |
76 | ||
77 | set(uncore, GEN6_RC_SLEEP, 0); | |
78 | ||
79 | set(uncore, GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */ | |
80 | ||
81 | /* | |
82 | * 2c: Program Coarse Power Gating Policies. | |
83 | * | |
84 | * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we | |
85 | * use instead is a more conservative estimate for the maximum time | |
86 | * it takes us to service a CS interrupt and submit a new ELSP - that | |
87 | * is the time which the GPU is idle waiting for the CPU to select the | |
88 | * next request to execute. If the idle hysteresis is less than that | |
89 | * interrupt service latency, the hardware will automatically gate | |
90 | * the power well and we will then incur the wake up cost on top of | |
91 | * the service latency. A similar guide from plane_state is that we | |
92 | * do not want the enable hysteresis to less than the wakeup latency. | |
93 | * | |
94 | * igt/gem_exec_nop/sequential provides a rough estimate for the | |
028a7a98 CW |
95 | * service latency, and puts it under 10us for Icelake, similar to |
96 | * Broadwell+, To be conservative, we want to factor in a context | |
97 | * switch on top (due to ksoftirqd). | |
c1132367 | 98 | */ |
028a7a98 CW |
99 | set(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 60); |
100 | set(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 60); | |
c1132367 AS |
101 | |
102 | /* 3a: Enable RC6 */ | |
730eaeb5 CW |
103 | rc6->ctl_enable = |
104 | GEN6_RC_CTL_HW_ENABLE | | |
105 | GEN6_RC_CTL_RC6_ENABLE | | |
106 | GEN6_RC_CTL_EI_MODE(1); | |
c1132367 | 107 | |
695dc55b RV |
108 | pg_enable = |
109 | GEN9_RENDER_PG_ENABLE | | |
110 | GEN9_MEDIA_PG_ENABLE | | |
111 | GEN11_MEDIA_SAMPLER_PG_ENABLE; | |
112 | ||
113 | if (INTEL_GEN(gt->i915) >= 12) { | |
114 | for (i = 0; i < I915_MAX_VCS; i++) | |
115 | if (HAS_ENGINE(gt, _VCS(i))) | |
116 | pg_enable |= (VDN_HCP_POWERGATE_ENABLE(i) | | |
117 | VDN_MFX_POWERGATE_ENABLE(i)); | |
118 | } | |
119 | ||
120 | set(uncore, GEN9_PG_ENABLE, pg_enable); | |
c1132367 AS |
121 | } |
122 | ||
123 | static void gen9_rc6_enable(struct intel_rc6 *rc6) | |
124 | { | |
125 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
126 | struct intel_engine_cs *engine; | |
127 | enum intel_engine_id id; | |
c1132367 AS |
128 | |
129 | /* 2b: Program RC6 thresholds.*/ | |
130 | if (INTEL_GEN(rc6_to_i915(rc6)) >= 10) { | |
131 | set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85); | |
132 | set(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150); | |
133 | } else if (IS_SKYLAKE(rc6_to_i915(rc6))) { | |
134 | /* | |
135 | * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only | |
136 | * when CPG is enabled | |
137 | */ | |
138 | set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16); | |
139 | } else { | |
140 | set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16); | |
141 | } | |
142 | ||
143 | set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ | |
144 | set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ | |
c6e07ada | 145 | for_each_engine(engine, rc6_to_gt(rc6), id) |
c1132367 AS |
146 | set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); |
147 | ||
148 | set(uncore, GUC_MAX_IDLE_COUNT, 0xA); | |
149 | ||
150 | set(uncore, GEN6_RC_SLEEP, 0); | |
151 | ||
152 | /* | |
153 | * 2c: Program Coarse Power Gating Policies. | |
154 | * | |
155 | * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we | |
156 | * use instead is a more conservative estimate for the maximum time | |
157 | * it takes us to service a CS interrupt and submit a new ELSP - that | |
158 | * is the time which the GPU is idle waiting for the CPU to select the | |
159 | * next request to execute. If the idle hysteresis is less than that | |
160 | * interrupt service latency, the hardware will automatically gate | |
161 | * the power well and we will then incur the wake up cost on top of | |
162 | * the service latency. A similar guide from plane_state is that we | |
163 | * do not want the enable hysteresis to less than the wakeup latency. | |
164 | * | |
165 | * igt/gem_exec_nop/sequential provides a rough estimate for the | |
166 | * service latency, and puts it around 10us for Broadwell (and other | |
167 | * big core) and around 40us for Broxton (and other low power cores). | |
168 | * [Note that for legacy ringbuffer submission, this is less than 1us!] | |
169 | * However, the wakeup latency on Broxton is closer to 100us. To be | |
170 | * conservative, we have to factor in a context switch on top (due | |
171 | * to ksoftirqd). | |
172 | */ | |
173 | set(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250); | |
174 | set(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 250); | |
175 | ||
176 | /* 3a: Enable RC6 */ | |
177 | set(uncore, GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */ | |
178 | ||
c1132367 | 179 | |
730eaeb5 CW |
180 | rc6->ctl_enable = |
181 | GEN6_RC_CTL_HW_ENABLE | | |
182 | GEN6_RC_CTL_RC6_ENABLE | | |
dab3aff7 | 183 | GEN6_RC_CTL_EI_MODE(1); |
c1132367 | 184 | |
980f87a2 ID |
185 | /* |
186 | * WaRsDisableCoarsePowerGating:skl,cnl | |
187 | * - Render/Media PG need to be disabled with RC6. | |
188 | */ | |
189 | if (!NEEDS_WaRsDisableCoarsePowerGating(rc6_to_i915(rc6))) | |
190 | set(uncore, GEN9_PG_ENABLE, | |
191 | GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE); | |
c1132367 AS |
192 | } |
193 | ||
194 | static void gen8_rc6_enable(struct intel_rc6 *rc6) | |
195 | { | |
196 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
197 | struct intel_engine_cs *engine; | |
198 | enum intel_engine_id id; | |
199 | ||
200 | /* 2b: Program RC6 thresholds.*/ | |
201 | set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); | |
202 | set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ | |
203 | set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ | |
c6e07ada | 204 | for_each_engine(engine, rc6_to_gt(rc6), id) |
c1132367 AS |
205 | set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); |
206 | set(uncore, GEN6_RC_SLEEP, 0); | |
207 | set(uncore, GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */ | |
208 | ||
209 | /* 3: Enable RC6 */ | |
730eaeb5 | 210 | rc6->ctl_enable = |
c1132367 AS |
211 | GEN6_RC_CTL_HW_ENABLE | |
212 | GEN7_RC_CTL_TO_MODE | | |
730eaeb5 | 213 | GEN6_RC_CTL_RC6_ENABLE; |
c1132367 AS |
214 | } |
215 | ||
216 | static void gen6_rc6_enable(struct intel_rc6 *rc6) | |
217 | { | |
218 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
219 | struct drm_i915_private *i915 = rc6_to_i915(rc6); | |
220 | struct intel_engine_cs *engine; | |
221 | enum intel_engine_id id; | |
222 | u32 rc6vids, rc6_mask; | |
223 | int ret; | |
224 | ||
225 | set(uncore, GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16); | |
226 | set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30); | |
227 | set(uncore, GEN6_RC6pp_WAKE_RATE_LIMIT, 30); | |
228 | set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); | |
229 | set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); | |
230 | ||
c6e07ada | 231 | for_each_engine(engine, rc6_to_gt(rc6), id) |
c1132367 AS |
232 | set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); |
233 | ||
234 | set(uncore, GEN6_RC_SLEEP, 0); | |
235 | set(uncore, GEN6_RC1e_THRESHOLD, 1000); | |
816cc4d6 | 236 | set(uncore, GEN6_RC6_THRESHOLD, 50000); |
c1132367 AS |
237 | set(uncore, GEN6_RC6p_THRESHOLD, 150000); |
238 | set(uncore, GEN6_RC6pp_THRESHOLD, 64000); /* unused */ | |
239 | ||
240 | /* We don't use those on Haswell */ | |
241 | rc6_mask = GEN6_RC_CTL_RC6_ENABLE; | |
242 | if (HAS_RC6p(i915)) | |
243 | rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE; | |
244 | if (HAS_RC6pp(i915)) | |
245 | rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE; | |
730eaeb5 | 246 | rc6->ctl_enable = |
c1132367 AS |
247 | rc6_mask | |
248 | GEN6_RC_CTL_EI_MODE(1) | | |
730eaeb5 | 249 | GEN6_RC_CTL_HW_ENABLE; |
c1132367 AS |
250 | |
251 | rc6vids = 0; | |
252 | ret = sandybridge_pcode_read(i915, GEN6_PCODE_READ_RC6VIDS, | |
253 | &rc6vids, NULL); | |
254 | if (IS_GEN(i915, 6) && ret) { | |
1ca6ce93 | 255 | drm_dbg(&i915->drm, "Couldn't check for BIOS workaround\n"); |
c1132367 AS |
256 | } else if (IS_GEN(i915, 6) && |
257 | (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) { | |
1ca6ce93 WK |
258 | drm_dbg(&i915->drm, |
259 | "You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n", | |
260 | GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450); | |
c1132367 AS |
261 | rc6vids &= 0xffff00; |
262 | rc6vids |= GEN6_ENCODE_RC6_VID(450); | |
263 | ret = sandybridge_pcode_write(i915, GEN6_PCODE_WRITE_RC6VIDS, rc6vids); | |
264 | if (ret) | |
1ca6ce93 WK |
265 | drm_err(&i915->drm, |
266 | "Couldn't fix incorrect rc6 voltage\n"); | |
c1132367 AS |
267 | } |
268 | } | |
269 | ||
270 | /* Check that the pcbr address is not empty. */ | |
271 | static int chv_rc6_init(struct intel_rc6 *rc6) | |
272 | { | |
273 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
1ca6ce93 | 274 | struct drm_i915_private *i915 = rc6_to_i915(rc6); |
c1132367 AS |
275 | resource_size_t pctx_paddr, paddr; |
276 | resource_size_t pctx_size = 32 * SZ_1K; | |
277 | u32 pcbr; | |
278 | ||
279 | pcbr = intel_uncore_read(uncore, VLV_PCBR); | |
280 | if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) { | |
1ca6ce93 WK |
281 | drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n"); |
282 | paddr = i915->dsm.end + 1 - pctx_size; | |
c1132367 AS |
283 | GEM_BUG_ON(paddr > U32_MAX); |
284 | ||
285 | pctx_paddr = (paddr & ~4095); | |
286 | intel_uncore_write(uncore, VLV_PCBR, pctx_paddr); | |
287 | } | |
288 | ||
289 | return 0; | |
290 | } | |
291 | ||
292 | static int vlv_rc6_init(struct intel_rc6 *rc6) | |
293 | { | |
294 | struct drm_i915_private *i915 = rc6_to_i915(rc6); | |
295 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
296 | struct drm_i915_gem_object *pctx; | |
297 | resource_size_t pctx_paddr; | |
298 | resource_size_t pctx_size = 24 * SZ_1K; | |
299 | u32 pcbr; | |
300 | ||
301 | pcbr = intel_uncore_read(uncore, VLV_PCBR); | |
302 | if (pcbr) { | |
303 | /* BIOS set it up already, grab the pre-alloc'd space */ | |
304 | resource_size_t pcbr_offset; | |
305 | ||
306 | pcbr_offset = (pcbr & ~4095) - i915->dsm.start; | |
307 | pctx = i915_gem_object_create_stolen_for_preallocated(i915, | |
308 | pcbr_offset, | |
c1132367 | 309 | pctx_size); |
0e5493ca CT |
310 | if (IS_ERR(pctx)) |
311 | return PTR_ERR(pctx); | |
c1132367 AS |
312 | |
313 | goto out; | |
314 | } | |
315 | ||
1ca6ce93 | 316 | drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n"); |
c1132367 AS |
317 | |
318 | /* | |
319 | * From the Gunit register HAS: | |
320 | * The Gfx driver is expected to program this register and ensure | |
321 | * proper allocation within Gfx stolen memory. For example, this | |
322 | * register should be programmed such than the PCBR range does not | |
323 | * overlap with other ranges, such as the frame buffer, protected | |
324 | * memory, or any other relevant ranges. | |
325 | */ | |
326 | pctx = i915_gem_object_create_stolen(i915, pctx_size); | |
0e5493ca | 327 | if (IS_ERR(pctx)) { |
1ca6ce93 WK |
328 | drm_dbg(&i915->drm, |
329 | "not enough stolen space for PCTX, disabling\n"); | |
0e5493ca | 330 | return PTR_ERR(pctx); |
c1132367 AS |
331 | } |
332 | ||
520f8350 MA |
333 | GEM_BUG_ON(range_overflows_end_t(u64, |
334 | i915->dsm.start, | |
335 | pctx->stolen->start, | |
336 | U32_MAX)); | |
c1132367 AS |
337 | pctx_paddr = i915->dsm.start + pctx->stolen->start; |
338 | intel_uncore_write(uncore, VLV_PCBR, pctx_paddr); | |
339 | ||
340 | out: | |
341 | rc6->pctx = pctx; | |
342 | return 0; | |
343 | } | |
344 | ||
345 | static void chv_rc6_enable(struct intel_rc6 *rc6) | |
346 | { | |
347 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
348 | struct intel_engine_cs *engine; | |
349 | enum intel_engine_id id; | |
350 | ||
351 | /* 2a: Program RC6 thresholds.*/ | |
352 | set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16); | |
353 | set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */ | |
354 | set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */ | |
355 | ||
c6e07ada | 356 | for_each_engine(engine, rc6_to_gt(rc6), id) |
c1132367 AS |
357 | set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); |
358 | set(uncore, GEN6_RC_SLEEP, 0); | |
359 | ||
360 | /* TO threshold set to 500 us (0x186 * 1.28 us) */ | |
361 | set(uncore, GEN6_RC6_THRESHOLD, 0x186); | |
362 | ||
363 | /* Allows RC6 residency counter to work */ | |
364 | set(uncore, VLV_COUNTER_CONTROL, | |
365 | _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | | |
366 | VLV_MEDIA_RC6_COUNT_EN | | |
367 | VLV_RENDER_RC6_COUNT_EN)); | |
368 | ||
369 | /* 3: Enable RC6 */ | |
730eaeb5 | 370 | rc6->ctl_enable = GEN7_RC_CTL_TO_MODE; |
c1132367 AS |
371 | } |
372 | ||
373 | static void vlv_rc6_enable(struct intel_rc6 *rc6) | |
374 | { | |
375 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
376 | struct intel_engine_cs *engine; | |
377 | enum intel_engine_id id; | |
378 | ||
379 | set(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000); | |
380 | set(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); | |
381 | set(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); | |
382 | ||
c6e07ada | 383 | for_each_engine(engine, rc6_to_gt(rc6), id) |
c1132367 AS |
384 | set(uncore, RING_MAX_IDLE(engine->mmio_base), 10); |
385 | ||
386 | set(uncore, GEN6_RC6_THRESHOLD, 0x557); | |
387 | ||
388 | /* Allows RC6 residency counter to work */ | |
389 | set(uncore, VLV_COUNTER_CONTROL, | |
390 | _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH | | |
391 | VLV_MEDIA_RC0_COUNT_EN | | |
392 | VLV_RENDER_RC0_COUNT_EN | | |
393 | VLV_MEDIA_RC6_COUNT_EN | | |
394 | VLV_RENDER_RC6_COUNT_EN)); | |
395 | ||
730eaeb5 CW |
396 | rc6->ctl_enable = |
397 | GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL; | |
c1132367 AS |
398 | } |
399 | ||
400 | static bool bxt_check_bios_rc6_setup(struct intel_rc6 *rc6) | |
401 | { | |
402 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
403 | struct drm_i915_private *i915 = rc6_to_i915(rc6); | |
404 | u32 rc6_ctx_base, rc_ctl, rc_sw_target; | |
405 | bool enable_rc6 = true; | |
406 | ||
407 | rc_ctl = intel_uncore_read(uncore, GEN6_RC_CONTROL); | |
408 | rc_sw_target = intel_uncore_read(uncore, GEN6_RC_STATE); | |
409 | rc_sw_target &= RC_SW_TARGET_STATE_MASK; | |
410 | rc_sw_target >>= RC_SW_TARGET_STATE_SHIFT; | |
1ca6ce93 | 411 | drm_dbg(&i915->drm, "BIOS enabled RC states: " |
c1132367 AS |
412 | "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n", |
413 | onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE), | |
414 | onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE), | |
415 | rc_sw_target); | |
416 | ||
417 | if (!(intel_uncore_read(uncore, RC6_LOCATION) & RC6_CTX_IN_DRAM)) { | |
1ca6ce93 | 418 | drm_dbg(&i915->drm, "RC6 Base location not set properly.\n"); |
c1132367 AS |
419 | enable_rc6 = false; |
420 | } | |
421 | ||
422 | /* | |
423 | * The exact context size is not known for BXT, so assume a page size | |
424 | * for this check. | |
425 | */ | |
426 | rc6_ctx_base = | |
427 | intel_uncore_read(uncore, RC6_CTX_BASE) & RC6_CTX_BASE_MASK; | |
428 | if (!(rc6_ctx_base >= i915->dsm_reserved.start && | |
429 | rc6_ctx_base + PAGE_SIZE < i915->dsm_reserved.end)) { | |
1ca6ce93 | 430 | drm_dbg(&i915->drm, "RC6 Base address not as expected.\n"); |
c1132367 AS |
431 | enable_rc6 = false; |
432 | } | |
433 | ||
434 | if (!((intel_uncore_read(uncore, PWRCTX_MAXCNT_RCSUNIT) & IDLE_TIME_MASK) > 1 && | |
435 | (intel_uncore_read(uncore, PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1 && | |
436 | (intel_uncore_read(uncore, PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1 && | |
437 | (intel_uncore_read(uncore, PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1)) { | |
1ca6ce93 WK |
438 | drm_dbg(&i915->drm, |
439 | "Engine Idle wait time not set properly.\n"); | |
c1132367 AS |
440 | enable_rc6 = false; |
441 | } | |
442 | ||
443 | if (!intel_uncore_read(uncore, GEN8_PUSHBUS_CONTROL) || | |
444 | !intel_uncore_read(uncore, GEN8_PUSHBUS_ENABLE) || | |
445 | !intel_uncore_read(uncore, GEN8_PUSHBUS_SHIFT)) { | |
1ca6ce93 | 446 | drm_dbg(&i915->drm, "Pushbus not setup properly.\n"); |
c1132367 AS |
447 | enable_rc6 = false; |
448 | } | |
449 | ||
450 | if (!intel_uncore_read(uncore, GEN6_GFXPAUSE)) { | |
1ca6ce93 | 451 | drm_dbg(&i915->drm, "GFX pause not setup properly.\n"); |
c1132367 AS |
452 | enable_rc6 = false; |
453 | } | |
454 | ||
455 | if (!intel_uncore_read(uncore, GEN8_MISC_CTRL0)) { | |
1ca6ce93 | 456 | drm_dbg(&i915->drm, "GPM control not setup properly.\n"); |
c1132367 AS |
457 | enable_rc6 = false; |
458 | } | |
459 | ||
460 | return enable_rc6; | |
461 | } | |
462 | ||
463 | static bool rc6_supported(struct intel_rc6 *rc6) | |
464 | { | |
465 | struct drm_i915_private *i915 = rc6_to_i915(rc6); | |
466 | ||
467 | if (!HAS_RC6(i915)) | |
468 | return false; | |
469 | ||
470 | if (intel_vgpu_active(i915)) | |
471 | return false; | |
472 | ||
473 | if (is_mock_gt(rc6_to_gt(rc6))) | |
474 | return false; | |
475 | ||
476 | if (IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(rc6)) { | |
dc483ba5 | 477 | drm_notice(&i915->drm, |
c1132367 AS |
478 | "RC6 and powersaving disabled by BIOS\n"); |
479 | return false; | |
480 | } | |
481 | ||
482 | return true; | |
483 | } | |
484 | ||
485 | static void rpm_get(struct intel_rc6 *rc6) | |
486 | { | |
487 | GEM_BUG_ON(rc6->wakeref); | |
488 | pm_runtime_get_sync(&rc6_to_i915(rc6)->drm.pdev->dev); | |
489 | rc6->wakeref = true; | |
490 | } | |
491 | ||
492 | static void rpm_put(struct intel_rc6 *rc6) | |
493 | { | |
494 | GEM_BUG_ON(!rc6->wakeref); | |
495 | pm_runtime_put(&rc6_to_i915(rc6)->drm.pdev->dev); | |
496 | rc6->wakeref = false; | |
497 | } | |
498 | ||
65f6d12c | 499 | static bool pctx_corrupted(struct intel_rc6 *rc6) |
2248a283 ID |
500 | { |
501 | struct drm_i915_private *i915 = rc6_to_i915(rc6); | |
502 | ||
503 | if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915)) | |
61e258ee | 504 | return false; |
2248a283 | 505 | |
65f6d12c | 506 | if (intel_uncore_read(rc6_to_uncore(rc6), GEN8_RC6_CTX_INFO)) |
61e258ee | 507 | return false; |
2248a283 | 508 | |
dc483ba5 | 509 | drm_notice(&i915->drm, |
3c1fe1eb | 510 | "RC6 context corruption, disabling runtime power management\n"); |
61e258ee | 511 | return true; |
2248a283 ID |
512 | } |
513 | ||
c1132367 AS |
514 | static void __intel_rc6_disable(struct intel_rc6 *rc6) |
515 | { | |
516 | struct drm_i915_private *i915 = rc6_to_i915(rc6); | |
517 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
518 | ||
519 | intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); | |
520 | if (INTEL_GEN(i915) >= 9) | |
521 | set(uncore, GEN9_PG_ENABLE, 0); | |
522 | set(uncore, GEN6_RC_CONTROL, 0); | |
523 | set(uncore, GEN6_RC_STATE, 0); | |
524 | intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); | |
525 | } | |
526 | ||
527 | void intel_rc6_init(struct intel_rc6 *rc6) | |
528 | { | |
529 | struct drm_i915_private *i915 = rc6_to_i915(rc6); | |
530 | int err; | |
531 | ||
532 | /* Disable runtime-pm until we can save the GPU state with rc6 pctx */ | |
533 | rpm_get(rc6); | |
534 | ||
535 | if (!rc6_supported(rc6)) | |
536 | return; | |
537 | ||
538 | if (IS_CHERRYVIEW(i915)) | |
539 | err = chv_rc6_init(rc6); | |
540 | else if (IS_VALLEYVIEW(i915)) | |
541 | err = vlv_rc6_init(rc6); | |
542 | else | |
543 | err = 0; | |
544 | ||
545 | /* Sanitize rc6, ensure it is disabled before we are ready. */ | |
546 | __intel_rc6_disable(rc6); | |
547 | ||
548 | rc6->supported = err == 0; | |
549 | } | |
550 | ||
551 | void intel_rc6_sanitize(struct intel_rc6 *rc6) | |
552 | { | |
e88e9f8e CW |
553 | memset(rc6->prev_hw_residency, 0, sizeof(rc6->prev_hw_residency)); |
554 | ||
c601cb21 CW |
555 | if (rc6->enabled) { /* unbalanced suspend/resume */ |
556 | rpm_get(rc6); | |
557 | rc6->enabled = false; | |
558 | } | |
559 | ||
c1132367 AS |
560 | if (rc6->supported) |
561 | __intel_rc6_disable(rc6); | |
562 | } | |
563 | ||
564 | void intel_rc6_enable(struct intel_rc6 *rc6) | |
565 | { | |
566 | struct drm_i915_private *i915 = rc6_to_i915(rc6); | |
567 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
568 | ||
569 | if (!rc6->supported) | |
570 | return; | |
571 | ||
572 | GEM_BUG_ON(rc6->enabled); | |
573 | ||
574 | intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); | |
575 | ||
576 | if (IS_CHERRYVIEW(i915)) | |
577 | chv_rc6_enable(rc6); | |
578 | else if (IS_VALLEYVIEW(i915)) | |
579 | vlv_rc6_enable(rc6); | |
580 | else if (INTEL_GEN(i915) >= 11) | |
581 | gen11_rc6_enable(rc6); | |
582 | else if (INTEL_GEN(i915) >= 9) | |
583 | gen9_rc6_enable(rc6); | |
584 | else if (IS_BROADWELL(i915)) | |
585 | gen8_rc6_enable(rc6); | |
586 | else if (INTEL_GEN(i915) >= 6) | |
587 | gen6_rc6_enable(rc6); | |
588 | ||
7d929989 | 589 | rc6->manual = rc6->ctl_enable & GEN6_RC_CTL_RC6_ENABLE; |
61e258ee CW |
590 | if (NEEDS_RC6_CTX_CORRUPTION_WA(i915)) |
591 | rc6->ctl_enable = 0; | |
592 | ||
c1132367 AS |
593 | intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); |
594 | ||
65f6d12c CW |
595 | if (unlikely(pctx_corrupted(rc6))) |
596 | return; | |
597 | ||
c1132367 AS |
598 | /* rc6 is ready, runtime-pm is go! */ |
599 | rpm_put(rc6); | |
600 | rc6->enabled = true; | |
601 | } | |
602 | ||
730eaeb5 CW |
603 | void intel_rc6_unpark(struct intel_rc6 *rc6) |
604 | { | |
605 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
606 | ||
607 | if (!rc6->enabled) | |
608 | return; | |
609 | ||
610 | /* Restore HW timers for automatic RC6 entry while busy */ | |
611 | set(uncore, GEN6_RC_CONTROL, rc6->ctl_enable); | |
612 | } | |
613 | ||
614 | void intel_rc6_park(struct intel_rc6 *rc6) | |
615 | { | |
616 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
13c5a577 | 617 | unsigned int target; |
730eaeb5 CW |
618 | |
619 | if (!rc6->enabled) | |
620 | return; | |
621 | ||
65f6d12c | 622 | if (unlikely(pctx_corrupted(rc6))) { |
61e258ee CW |
623 | intel_rc6_disable(rc6); |
624 | return; | |
625 | } | |
626 | ||
7d929989 | 627 | if (!rc6->manual) |
972745fd CW |
628 | return; |
629 | ||
730eaeb5 CW |
630 | /* Turn off the HW timers and go directly to rc6 */ |
631 | set(uncore, GEN6_RC_CONTROL, GEN6_RC_CTL_RC6_ENABLE); | |
13c5a577 CW |
632 | |
633 | if (HAS_RC6pp(rc6_to_i915(rc6))) | |
634 | target = 0x6; /* deepest rc6 */ | |
635 | else if (HAS_RC6p(rc6_to_i915(rc6))) | |
636 | target = 0x5; /* deep rc6 */ | |
637 | else | |
638 | target = 0x4; /* normal rc6 */ | |
639 | set(uncore, GEN6_RC_STATE, target << RC_SW_TARGET_STATE_SHIFT); | |
730eaeb5 CW |
640 | } |
641 | ||
c1132367 AS |
642 | void intel_rc6_disable(struct intel_rc6 *rc6) |
643 | { | |
644 | if (!rc6->enabled) | |
645 | return; | |
646 | ||
647 | rpm_get(rc6); | |
648 | rc6->enabled = false; | |
649 | ||
650 | __intel_rc6_disable(rc6); | |
651 | } | |
652 | ||
653 | void intel_rc6_fini(struct intel_rc6 *rc6) | |
654 | { | |
655 | struct drm_i915_gem_object *pctx; | |
656 | ||
657 | intel_rc6_disable(rc6); | |
658 | ||
659 | pctx = fetch_and_zero(&rc6->pctx); | |
660 | if (pctx) | |
661 | i915_gem_object_put(pctx); | |
662 | ||
663 | if (rc6->wakeref) | |
664 | rpm_put(rc6); | |
665 | } | |
666 | ||
667 | static u64 vlv_residency_raw(struct intel_uncore *uncore, const i915_reg_t reg) | |
668 | { | |
669 | u32 lower, upper, tmp; | |
670 | int loop = 2; | |
671 | ||
672 | /* | |
673 | * The register accessed do not need forcewake. We borrow | |
674 | * uncore lock to prevent concurrent access to range reg. | |
675 | */ | |
676 | lockdep_assert_held(&uncore->lock); | |
677 | ||
678 | /* | |
679 | * vlv and chv residency counters are 40 bits in width. | |
680 | * With a control bit, we can choose between upper or lower | |
681 | * 32bit window into this counter. | |
682 | * | |
683 | * Although we always use the counter in high-range mode elsewhere, | |
684 | * userspace may attempt to read the value before rc6 is initialised, | |
685 | * before we have set the default VLV_COUNTER_CONTROL value. So always | |
686 | * set the high bit to be safe. | |
687 | */ | |
688 | set(uncore, VLV_COUNTER_CONTROL, | |
689 | _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH)); | |
690 | upper = intel_uncore_read_fw(uncore, reg); | |
691 | do { | |
692 | tmp = upper; | |
693 | ||
694 | set(uncore, VLV_COUNTER_CONTROL, | |
695 | _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH)); | |
696 | lower = intel_uncore_read_fw(uncore, reg); | |
697 | ||
698 | set(uncore, VLV_COUNTER_CONTROL, | |
699 | _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH)); | |
700 | upper = intel_uncore_read_fw(uncore, reg); | |
701 | } while (upper != tmp && --loop); | |
702 | ||
703 | /* | |
704 | * Everywhere else we always use VLV_COUNTER_CONTROL with the | |
705 | * VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set | |
706 | * now. | |
707 | */ | |
708 | ||
709 | return lower | (u64)upper << 8; | |
710 | } | |
711 | ||
712 | u64 intel_rc6_residency_ns(struct intel_rc6 *rc6, const i915_reg_t reg) | |
713 | { | |
714 | struct drm_i915_private *i915 = rc6_to_i915(rc6); | |
715 | struct intel_uncore *uncore = rc6_to_uncore(rc6); | |
716 | u64 time_hw, prev_hw, overflow_hw; | |
717 | unsigned int fw_domains; | |
718 | unsigned long flags; | |
719 | unsigned int i; | |
720 | u32 mul, div; | |
721 | ||
722 | if (!rc6->supported) | |
723 | return 0; | |
724 | ||
725 | /* | |
726 | * Store previous hw counter values for counter wrap-around handling. | |
727 | * | |
728 | * There are only four interesting registers and they live next to each | |
729 | * other so we can use the relative address, compared to the smallest | |
730 | * one as the index into driver storage. | |
731 | */ | |
732 | i = (i915_mmio_reg_offset(reg) - | |
733 | i915_mmio_reg_offset(GEN6_GT_GFX_RC6_LOCKED)) / sizeof(u32); | |
0d4c351a | 734 | if (drm_WARN_ON_ONCE(&i915->drm, i >= ARRAY_SIZE(rc6->cur_residency))) |
c1132367 AS |
735 | return 0; |
736 | ||
737 | fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ); | |
738 | ||
739 | spin_lock_irqsave(&uncore->lock, flags); | |
740 | intel_uncore_forcewake_get__locked(uncore, fw_domains); | |
741 | ||
742 | /* On VLV and CHV, residency time is in CZ units rather than 1.28us */ | |
743 | if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { | |
744 | mul = 1000000; | |
745 | div = i915->czclk_freq; | |
746 | overflow_hw = BIT_ULL(40); | |
747 | time_hw = vlv_residency_raw(uncore, reg); | |
748 | } else { | |
749 | /* 833.33ns units on Gen9LP, 1.28us elsewhere. */ | |
750 | if (IS_GEN9_LP(i915)) { | |
751 | mul = 10000; | |
752 | div = 12; | |
753 | } else { | |
754 | mul = 1280; | |
755 | div = 1; | |
756 | } | |
757 | ||
758 | overflow_hw = BIT_ULL(32); | |
759 | time_hw = intel_uncore_read_fw(uncore, reg); | |
760 | } | |
761 | ||
762 | /* | |
763 | * Counter wrap handling. | |
764 | * | |
765 | * But relying on a sufficient frequency of queries otherwise counters | |
766 | * can still wrap. | |
767 | */ | |
768 | prev_hw = rc6->prev_hw_residency[i]; | |
769 | rc6->prev_hw_residency[i] = time_hw; | |
770 | ||
771 | /* RC6 delta from last sample. */ | |
772 | if (time_hw >= prev_hw) | |
773 | time_hw -= prev_hw; | |
774 | else | |
775 | time_hw += overflow_hw - prev_hw; | |
776 | ||
777 | /* Add delta to RC6 extended raw driver copy. */ | |
778 | time_hw += rc6->cur_residency[i]; | |
779 | rc6->cur_residency[i] = time_hw; | |
780 | ||
781 | intel_uncore_forcewake_put__locked(uncore, fw_domains); | |
782 | spin_unlock_irqrestore(&uncore->lock, flags); | |
783 | ||
784 | return mul_u64_u32_div(time_hw, mul, div); | |
785 | } | |
786 | ||
787 | u64 intel_rc6_residency_us(struct intel_rc6 *rc6, i915_reg_t reg) | |
788 | { | |
789 | return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(rc6, reg), 1000); | |
790 | } | |
ba446f74 CW |
791 | |
792 | #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) | |
793 | #include "selftest_rc6.c" | |
794 | #endif |