2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
28 #include "display/intel_de.h"
29 #include "display/intel_display_trace.h"
30 #include "display/skl_watermark.h"
32 #include "gt/intel_engine_regs.h"
33 #include "gt/intel_gt.h"
34 #include "gt/intel_gt_mcr.h"
35 #include "gt/intel_gt_regs.h"
38 #include "intel_mchbar_regs.h"
40 #include "vlv_sideband.h"
42 struct drm_i915_clock_gating_funcs {
43 void (*init_clock_gating)(struct drm_i915_private *i915);
46 /* used in computing the new watermarks state */
47 struct intel_wm_config {
48 unsigned int num_pipes_active;
53 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
55 if (HAS_LLC(dev_priv)) {
57 * WaCompressedResourceDisplayNewHashMode:skl,kbl
58 * Display WA #0390: skl,kbl
60 * Must match Sampler, Pixel Back End, and Media. See
61 * WaCompressedResourceSamplerPbeMediaNewHashMode.
63 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
66 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
67 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
69 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
70 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
73 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
74 * Display WA #0859: skl,bxt,kbl,glk,cfl
76 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
79 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
81 gen9_init_clock_gating(dev_priv);
83 /* WaDisableSDEUnitClockGating:bxt */
84 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
88 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
90 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
93 * Wa: Backlight PWM may stop in the asserted state, causing backlight
96 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
97 PWM1_GATING_DIS | PWM2_GATING_DIS);
100 * Lower the display internal timeout.
101 * This is needed to avoid any hard hangs when DSI port PLL
102 * is off and a MMIO access is attempted by any privilege
103 * application, using batch buffers or any other means.
105 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
108 * WaFbcTurnOffFbcWatermark:bxt
109 * Display WA #0562: bxt
111 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
114 * WaFbcHighMemBwCorruptionAvoidance:bxt
115 * Display WA #0883: bxt
117 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
120 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
122 gen9_init_clock_gating(dev_priv);
125 * WaDisablePWMClockGating:glk
126 * Backlight PWM may stop in the asserted state, causing backlight
129 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
130 PWM1_GATING_DIS | PWM2_GATING_DIS);
133 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
137 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
139 switch (tmp & CLKCFG_FSB_MASK) {
141 dev_priv->fsb_freq = 533; /* 133*4 */
144 dev_priv->fsb_freq = 800; /* 200*4 */
147 dev_priv->fsb_freq = 667; /* 167*4 */
150 dev_priv->fsb_freq = 400; /* 100*4 */
154 switch (tmp & CLKCFG_MEM_MASK) {
156 dev_priv->mem_freq = 533;
159 dev_priv->mem_freq = 667;
162 dev_priv->mem_freq = 800;
166 /* detect pineview DDR3 setting */
167 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
168 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
171 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
175 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
176 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
178 switch (ddrpll & 0xff) {
180 dev_priv->mem_freq = 800;
183 dev_priv->mem_freq = 1066;
186 dev_priv->mem_freq = 1333;
189 dev_priv->mem_freq = 1600;
192 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
194 dev_priv->mem_freq = 0;
198 switch (csipll & 0x3ff) {
200 dev_priv->fsb_freq = 3200;
203 dev_priv->fsb_freq = 3733;
206 dev_priv->fsb_freq = 4266;
209 dev_priv->fsb_freq = 4800;
212 dev_priv->fsb_freq = 5333;
215 dev_priv->fsb_freq = 5866;
218 dev_priv->fsb_freq = 6400;
221 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
223 dev_priv->fsb_freq = 0;
228 static const struct cxsr_latency cxsr_latency_table[] = {
229 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
230 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
231 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
232 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
233 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
235 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
236 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
237 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
238 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
239 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
241 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
242 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
243 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
244 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
245 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
247 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
248 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
249 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
250 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
251 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
253 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
254 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
255 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
256 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
257 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
259 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
260 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
261 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
262 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
263 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
266 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
271 const struct cxsr_latency *latency;
274 if (fsb == 0 || mem == 0)
277 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
278 latency = &cxsr_latency_table[i];
279 if (is_desktop == latency->is_desktop &&
280 is_ddr3 == latency->is_ddr3 &&
281 fsb == latency->fsb_freq && mem == latency->mem_freq)
285 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
290 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
294 vlv_punit_get(dev_priv);
296 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
298 val &= ~FORCE_DDR_HIGH_FREQ;
300 val |= FORCE_DDR_HIGH_FREQ;
301 val &= ~FORCE_DDR_LOW_FREQ;
302 val |= FORCE_DDR_FREQ_REQ_ACK;
303 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
305 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
306 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
307 drm_err(&dev_priv->drm,
308 "timed out waiting for Punit DDR DVFS request\n");
310 vlv_punit_put(dev_priv);
313 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
317 vlv_punit_get(dev_priv);
319 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
321 val |= DSP_MAXFIFO_PM5_ENABLE;
323 val &= ~DSP_MAXFIFO_PM5_ENABLE;
324 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
326 vlv_punit_put(dev_priv);
329 #define FW_WM(value, plane) \
330 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
332 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
337 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
338 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
339 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
340 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
341 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
342 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
343 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
344 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
345 } else if (IS_PINEVIEW(dev_priv)) {
346 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
347 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
349 val |= PINEVIEW_SELF_REFRESH_EN;
351 val &= ~PINEVIEW_SELF_REFRESH_EN;
352 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
353 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
354 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
355 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
356 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
357 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
358 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
359 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
360 } else if (IS_I915GM(dev_priv)) {
362 * FIXME can't find a bit like this for 915G, and
363 * and yet it does have the related watermark in
364 * FW_BLC_SELF. What's going on?
366 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
367 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
368 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
369 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
370 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
375 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
377 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
378 str_enabled_disabled(enable),
379 str_enabled_disabled(was_enabled));
385 * intel_set_memory_cxsr - Configure CxSR state
386 * @dev_priv: i915 device
387 * @enable: Allow vs. disallow CxSR
389 * Allow or disallow the system to enter a special CxSR
390 * (C-state self refresh) state. What typically happens in CxSR mode
391 * is that several display FIFOs may get combined into a single larger
392 * FIFO for a particular plane (so called max FIFO mode) to allow the
393 * system to defer memory fetches longer, and the memory will enter
396 * Note that enabling CxSR does not guarantee that the system enter
397 * this special mode, nor does it guarantee that the system stays
398 * in that mode once entered. So this just allows/disallows the system
399 * to autonomously utilize the CxSR mode. Other factors such as core
400 * C-states will affect when/if the system actually enters/exits the
403 * Note that on VLV/CHV this actually only controls the max FIFO mode,
404 * and the system is free to enter/exit memory self refresh at any time
405 * even when the use of CxSR has been disallowed.
407 * While the system is actually in the CxSR/max FIFO mode, some plane
408 * control registers will not get latched on vblank. Thus in order to
409 * guarantee the system will respond to changes in the plane registers
410 * we must always disallow CxSR prior to making changes to those registers.
411 * Unfortunately the system will re-evaluate the CxSR conditions at
412 * frame start which happens after vblank start (which is when the plane
413 * registers would get latched), so we can't proceed with the plane update
414 * during the same frame where we disallowed CxSR.
416 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
417 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
418 * the hardware w.r.t. HPLL SR when writing to plane registers.
419 * Disallowing just CxSR is sufficient.
421 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
425 mutex_lock(&dev_priv->display.wm.wm_mutex);
426 ret = _intel_set_memory_cxsr(dev_priv, enable);
427 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
428 dev_priv->display.wm.vlv.cxsr = enable;
429 else if (IS_G4X(dev_priv))
430 dev_priv->display.wm.g4x.cxsr = enable;
431 mutex_unlock(&dev_priv->display.wm.wm_mutex);
437 * Latency for FIFO fetches is dependent on several factors:
438 * - memory configuration (speed, channels)
440 * - current MCH state
441 * It can be fairly high in some situations, so here we assume a fairly
442 * pessimal value. It's a tradeoff between extra memory fetches (if we
443 * set this value too high, the FIFO will fetch frequently to stay full)
444 * and power consumption (set it too low to save power and we might see
445 * FIFO underruns and display "flicker").
447 * A value of 5us seems to be a good balance; safe for very low end
448 * platforms but not overly aggressive on lower latency configs.
450 static const int pessimal_latency_ns = 5000;
452 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
453 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
455 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
457 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
458 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
459 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
460 enum pipe pipe = crtc->pipe;
461 int sprite0_start, sprite1_start;
462 u32 dsparb, dsparb2, dsparb3;
466 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
467 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
468 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
469 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
472 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
473 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
474 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
475 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
478 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
479 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
480 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
481 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
488 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
489 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
490 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
491 fifo_state->plane[PLANE_CURSOR] = 63;
494 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
495 enum i9xx_plane_id i9xx_plane)
497 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
500 size = dsparb & 0x7f;
501 if (i9xx_plane == PLANE_B)
502 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
504 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
505 dsparb, plane_name(i9xx_plane), size);
510 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
511 enum i9xx_plane_id i9xx_plane)
513 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
516 size = dsparb & 0x1ff;
517 if (i9xx_plane == PLANE_B)
518 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
519 size >>= 1; /* Convert to cachelines */
521 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
522 dsparb, plane_name(i9xx_plane), size);
527 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
528 enum i9xx_plane_id i9xx_plane)
530 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
533 size = dsparb & 0x7f;
534 size >>= 2; /* Convert to cachelines */
536 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
537 dsparb, plane_name(i9xx_plane), size);
542 /* Pineview has different values for various configs */
543 static const struct intel_watermark_params pnv_display_wm = {
544 .fifo_size = PINEVIEW_DISPLAY_FIFO,
545 .max_wm = PINEVIEW_MAX_WM,
546 .default_wm = PINEVIEW_DFT_WM,
547 .guard_size = PINEVIEW_GUARD_WM,
548 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
551 static const struct intel_watermark_params pnv_display_hplloff_wm = {
552 .fifo_size = PINEVIEW_DISPLAY_FIFO,
553 .max_wm = PINEVIEW_MAX_WM,
554 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
555 .guard_size = PINEVIEW_GUARD_WM,
556 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
559 static const struct intel_watermark_params pnv_cursor_wm = {
560 .fifo_size = PINEVIEW_CURSOR_FIFO,
561 .max_wm = PINEVIEW_CURSOR_MAX_WM,
562 .default_wm = PINEVIEW_CURSOR_DFT_WM,
563 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
564 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
567 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
568 .fifo_size = PINEVIEW_CURSOR_FIFO,
569 .max_wm = PINEVIEW_CURSOR_MAX_WM,
570 .default_wm = PINEVIEW_CURSOR_DFT_WM,
571 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
572 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
575 static const struct intel_watermark_params i965_cursor_wm_info = {
576 .fifo_size = I965_CURSOR_FIFO,
577 .max_wm = I965_CURSOR_MAX_WM,
578 .default_wm = I965_CURSOR_DFT_WM,
580 .cacheline_size = I915_FIFO_LINE_SIZE,
583 static const struct intel_watermark_params i945_wm_info = {
584 .fifo_size = I945_FIFO_SIZE,
585 .max_wm = I915_MAX_WM,
588 .cacheline_size = I915_FIFO_LINE_SIZE,
591 static const struct intel_watermark_params i915_wm_info = {
592 .fifo_size = I915_FIFO_SIZE,
593 .max_wm = I915_MAX_WM,
596 .cacheline_size = I915_FIFO_LINE_SIZE,
599 static const struct intel_watermark_params i830_a_wm_info = {
600 .fifo_size = I855GM_FIFO_SIZE,
601 .max_wm = I915_MAX_WM,
604 .cacheline_size = I830_FIFO_LINE_SIZE,
607 static const struct intel_watermark_params i830_bc_wm_info = {
608 .fifo_size = I855GM_FIFO_SIZE,
609 .max_wm = I915_MAX_WM/2,
612 .cacheline_size = I830_FIFO_LINE_SIZE,
615 static const struct intel_watermark_params i845_wm_info = {
616 .fifo_size = I830_FIFO_SIZE,
617 .max_wm = I915_MAX_WM,
620 .cacheline_size = I830_FIFO_LINE_SIZE,
624 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
625 * @pixel_rate: Pipe pixel rate in kHz
626 * @cpp: Plane bytes per pixel
627 * @latency: Memory wakeup latency in 0.1us units
629 * Compute the watermark using the method 1 or "small buffer"
630 * formula. The caller may additonally add extra cachelines
631 * to account for TLB misses and clock crossings.
633 * This method is concerned with the short term drain rate
634 * of the FIFO, ie. it does not account for blanking periods
635 * which would effectively reduce the average drain rate across
636 * a longer period. The name "small" refers to the fact the
637 * FIFO is relatively small compared to the amount of data
640 * The FIFO level vs. time graph might look something like:
644 * __---__---__ (- plane active, _ blanking)
647 * or perhaps like this:
650 * __----__----__ (- plane active, _ blanking)
654 * The watermark in bytes
656 static unsigned int intel_wm_method1(unsigned int pixel_rate,
658 unsigned int latency)
662 ret = mul_u32_u32(pixel_rate, cpp * latency);
663 ret = DIV_ROUND_UP_ULL(ret, 10000);
669 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
670 * @pixel_rate: Pipe pixel rate in kHz
671 * @htotal: Pipe horizontal total
672 * @width: Plane width in pixels
673 * @cpp: Plane bytes per pixel
674 * @latency: Memory wakeup latency in 0.1us units
676 * Compute the watermark using the method 2 or "large buffer"
677 * formula. The caller may additonally add extra cachelines
678 * to account for TLB misses and clock crossings.
680 * This method is concerned with the long term drain rate
681 * of the FIFO, ie. it does account for blanking periods
682 * which effectively reduce the average drain rate across
683 * a longer period. The name "large" refers to the fact the
684 * FIFO is relatively large compared to the amount of data
687 * The FIFO level vs. time graph might look something like:
692 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
696 * The watermark in bytes
698 static unsigned int intel_wm_method2(unsigned int pixel_rate,
702 unsigned int latency)
707 * FIXME remove once all users are computing
708 * watermarks in the correct place.
710 if (WARN_ON_ONCE(htotal == 0))
713 ret = (latency * pixel_rate) / (htotal * 10000);
714 ret = (ret + 1) * width * cpp;
720 * intel_calculate_wm - calculate watermark level
721 * @pixel_rate: pixel clock
722 * @wm: chip FIFO params
723 * @fifo_size: size of the FIFO buffer
724 * @cpp: bytes per pixel
725 * @latency_ns: memory latency for the platform
727 * Calculate the watermark level (the level at which the display plane will
728 * start fetching from memory again). Each chip has a different display
729 * FIFO size and allocation, so the caller needs to figure that out and pass
730 * in the correct intel_watermark_params structure.
732 * As the pixel clock runs, the FIFO will be drained at a rate that depends
733 * on the pixel size. When it reaches the watermark level, it'll start
734 * fetching FIFO line sized based chunks from memory until the FIFO fills
735 * past the watermark point. If the FIFO drains completely, a FIFO underrun
736 * will occur, and a display engine hang could result.
738 static unsigned int intel_calculate_wm(int pixel_rate,
739 const struct intel_watermark_params *wm,
740 int fifo_size, int cpp,
741 unsigned int latency_ns)
743 int entries, wm_size;
746 * Note: we need to make sure we don't overflow for various clock &
748 * clocks go from a few thousand to several hundred thousand.
749 * latency is usually a few thousand
751 entries = intel_wm_method1(pixel_rate, cpp,
753 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
755 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
757 wm_size = fifo_size - entries;
758 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
760 /* Don't promote wm_size to unsigned... */
761 if (wm_size > wm->max_wm)
762 wm_size = wm->max_wm;
764 wm_size = wm->default_wm;
767 * Bspec seems to indicate that the value shouldn't be lower than
768 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
769 * Lets go for 8 which is the burst size since certain platforms
770 * already use a hardcoded 8 (which is what the spec says should be
779 static bool is_disabling(int old, int new, int threshold)
781 return old >= threshold && new < threshold;
784 static bool is_enabling(int old, int new, int threshold)
786 return old < threshold && new >= threshold;
789 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
791 return dev_priv->display.wm.max_level + 1;
794 bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
795 const struct intel_plane_state *plane_state)
797 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
799 /* FIXME check the 'enable' instead */
800 if (!crtc_state->hw.active)
804 * Treat cursor with fb as always visible since cursor updates
805 * can happen faster than the vrefresh rate, and the current
806 * watermark code doesn't handle that correctly. Cursor updates
807 * which set/clear the fb or change the cursor size are going
808 * to get throttled by intel_legacy_cursor_update() to work
809 * around this problem with the watermark code.
811 if (plane->id == PLANE_CURSOR)
812 return plane_state->hw.fb != NULL;
814 return plane_state->uapi.visible;
817 static bool intel_crtc_active(struct intel_crtc *crtc)
819 /* Be paranoid as we can arrive here with only partial
820 * state retrieved from the hardware during setup.
822 * We can ditch the adjusted_mode.crtc_clock check as soon
823 * as Haswell has gained clock readout/fastboot support.
825 * We can ditch the crtc->primary->state->fb check as soon as we can
826 * properly reconstruct framebuffers.
828 * FIXME: The intel_crtc->active here should be switched to
829 * crtc->state->active once we have proper CRTC states wired up
832 return crtc && crtc->active && crtc->base.primary->state->fb &&
833 crtc->config->hw.adjusted_mode.crtc_clock;
836 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
838 struct intel_crtc *crtc, *enabled = NULL;
840 for_each_intel_crtc(&dev_priv->drm, crtc) {
841 if (intel_crtc_active(crtc)) {
851 static void pnv_update_wm(struct drm_i915_private *dev_priv)
853 struct intel_crtc *crtc;
854 const struct cxsr_latency *latency;
858 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
863 drm_dbg_kms(&dev_priv->drm,
864 "Unknown FSB/MEM found, disable CxSR\n");
865 intel_set_memory_cxsr(dev_priv, false);
869 crtc = single_enabled_crtc(dev_priv);
871 const struct drm_framebuffer *fb =
872 crtc->base.primary->state->fb;
873 int pixel_rate = crtc->config->pixel_rate;
874 int cpp = fb->format->cpp[0];
877 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
878 pnv_display_wm.fifo_size,
879 cpp, latency->display_sr);
880 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
881 reg &= ~DSPFW_SR_MASK;
882 reg |= FW_WM(wm, SR);
883 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
884 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
887 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
888 pnv_display_wm.fifo_size,
889 4, latency->cursor_sr);
890 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
891 reg &= ~DSPFW_CURSOR_SR_MASK;
892 reg |= FW_WM(wm, CURSOR_SR);
893 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
895 /* Display HPLL off SR */
896 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
897 pnv_display_hplloff_wm.fifo_size,
898 cpp, latency->display_hpll_disable);
899 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
900 reg &= ~DSPFW_HPLL_SR_MASK;
901 reg |= FW_WM(wm, HPLL_SR);
902 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
904 /* cursor HPLL off SR */
905 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
906 pnv_display_hplloff_wm.fifo_size,
907 4, latency->cursor_hpll_disable);
908 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
909 reg &= ~DSPFW_HPLL_CURSOR_MASK;
910 reg |= FW_WM(wm, HPLL_CURSOR);
911 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
912 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
914 intel_set_memory_cxsr(dev_priv, true);
916 intel_set_memory_cxsr(dev_priv, false);
921 * Documentation says:
922 * "If the line size is small, the TLB fetches can get in the way of the
923 * data fetches, causing some lag in the pixel data return which is not
924 * accounted for in the above formulas. The following adjustment only
925 * needs to be applied if eight whole lines fit in the buffer at once.
926 * The WM is adjusted upwards by the difference between the FIFO size
927 * and the size of 8 whole lines. This adjustment is always performed
928 * in the actual pixel depth regardless of whether FBC is enabled or not."
930 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
932 int tlb_miss = fifo_size * 64 - width * cpp * 8;
934 return max(0, tlb_miss);
937 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
938 const struct g4x_wm_values *wm)
942 for_each_pipe(dev_priv, pipe)
943 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
945 intel_uncore_write(&dev_priv->uncore, DSPFW1,
946 FW_WM(wm->sr.plane, SR) |
947 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
948 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
949 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
950 intel_uncore_write(&dev_priv->uncore, DSPFW2,
951 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
952 FW_WM(wm->sr.fbc, FBC_SR) |
953 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
954 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
955 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
956 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
957 intel_uncore_write(&dev_priv->uncore, DSPFW3,
958 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
959 FW_WM(wm->sr.cursor, CURSOR_SR) |
960 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
961 FW_WM(wm->hpll.plane, HPLL_SR));
963 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
966 #define FW_WM_VLV(value, plane) \
967 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
969 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
970 const struct vlv_wm_values *wm)
974 for_each_pipe(dev_priv, pipe) {
975 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
977 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
978 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
979 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
980 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
981 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
985 * Zero the (unused) WM1 watermarks, and also clear all the
986 * high order bits so that there are no out of bounds values
987 * present in the registers during the reprogramming.
989 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
990 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
991 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
992 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
993 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
995 intel_uncore_write(&dev_priv->uncore, DSPFW1,
996 FW_WM(wm->sr.plane, SR) |
997 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
998 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
999 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1000 intel_uncore_write(&dev_priv->uncore, DSPFW2,
1001 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1002 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1003 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1004 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1005 FW_WM(wm->sr.cursor, CURSOR_SR));
1007 if (IS_CHERRYVIEW(dev_priv)) {
1008 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1009 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1010 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1011 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1012 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1013 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1014 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1015 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1016 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1017 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1018 FW_WM(wm->sr.plane >> 9, SR_HI) |
1019 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1020 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1021 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1022 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1023 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1024 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1025 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1026 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1027 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1029 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1030 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1031 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1032 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1033 FW_WM(wm->sr.plane >> 9, SR_HI) |
1034 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1035 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1036 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1037 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1038 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1039 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1042 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1047 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1049 /* all latencies in usec */
1050 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1051 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1052 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1054 dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
1057 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1060 * DSPCNTR[13] supposedly controls whether the
1061 * primary plane can use the FIFO space otherwise
1062 * reserved for the sprite plane. It's not 100% clear
1063 * what the actual FIFO size is, but it looks like we
1064 * can happily set both primary and sprite watermarks
1065 * up to 127 cachelines. So that would seem to mean
1066 * that either DSPCNTR[13] doesn't do anything, or that
1067 * the total FIFO is >= 256 cachelines in size. Either
1068 * way, we don't seem to have to worry about this
1069 * repartitioning as the maximum watermark value the
1070 * register can hold for each plane is lower than the
1071 * minimum FIFO size.
1077 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1079 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1081 MISSING_CASE(plane_id);
1086 static int g4x_fbc_fifo_size(int level)
1089 case G4X_WM_LEVEL_SR:
1091 case G4X_WM_LEVEL_HPLL:
1094 MISSING_CASE(level);
1099 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1100 const struct intel_plane_state *plane_state,
1103 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1104 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1105 const struct drm_display_mode *pipe_mode =
1106 &crtc_state->hw.pipe_mode;
1107 unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
1108 unsigned int pixel_rate, htotal, cpp, width, wm;
1113 if (!intel_wm_plane_visible(crtc_state, plane_state))
1116 cpp = plane_state->hw.fb->format->cpp[0];
1119 * WaUse32BppForSRWM:ctg,elk
1121 * The spec fails to list this restriction for the
1122 * HPLL watermark, which seems a little strange.
1123 * Let's use 32bpp for the HPLL watermark as well.
1125 if (plane->id == PLANE_PRIMARY &&
1126 level != G4X_WM_LEVEL_NORMAL)
1129 pixel_rate = crtc_state->pixel_rate;
1130 htotal = pipe_mode->crtc_htotal;
1131 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1133 if (plane->id == PLANE_CURSOR) {
1134 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1135 } else if (plane->id == PLANE_PRIMARY &&
1136 level == G4X_WM_LEVEL_NORMAL) {
1137 wm = intel_wm_method1(pixel_rate, cpp, latency);
1139 unsigned int small, large;
1141 small = intel_wm_method1(pixel_rate, cpp, latency);
1142 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1144 wm = min(small, large);
1147 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1150 wm = DIV_ROUND_UP(wm, 64) + 2;
1152 return min_t(unsigned int, wm, USHRT_MAX);
1155 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1156 int level, enum plane_id plane_id, u16 value)
1158 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1161 for (; level < intel_wm_num_levels(dev_priv); level++) {
1162 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1164 dirty |= raw->plane[plane_id] != value;
1165 raw->plane[plane_id] = value;
1171 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1172 int level, u16 value)
1174 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1177 /* NORMAL level doesn't have an FBC watermark */
1178 level = max(level, G4X_WM_LEVEL_SR);
1180 for (; level < intel_wm_num_levels(dev_priv); level++) {
1181 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1183 dirty |= raw->fbc != value;
1190 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1191 const struct intel_plane_state *plane_state,
1194 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1195 const struct intel_plane_state *plane_state)
1197 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1198 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1199 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1200 enum plane_id plane_id = plane->id;
1204 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1205 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1206 if (plane_id == PLANE_PRIMARY)
1207 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1211 for (level = 0; level < num_levels; level++) {
1212 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1215 wm = g4x_compute_wm(crtc_state, plane_state, level);
1216 max_wm = g4x_plane_fifo_size(plane_id, level);
1221 dirty |= raw->plane[plane_id] != wm;
1222 raw->plane[plane_id] = wm;
1224 if (plane_id != PLANE_PRIMARY ||
1225 level == G4X_WM_LEVEL_NORMAL)
1228 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1229 raw->plane[plane_id]);
1230 max_wm = g4x_fbc_fifo_size(level);
1233 * FBC wm is not mandatory as we
1234 * can always just disable its use.
1239 dirty |= raw->fbc != wm;
1243 /* mark watermarks as invalid */
1244 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1246 if (plane_id == PLANE_PRIMARY)
1247 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1251 drm_dbg_kms(&dev_priv->drm,
1252 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1254 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1255 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1256 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1258 if (plane_id == PLANE_PRIMARY)
1259 drm_dbg_kms(&dev_priv->drm,
1260 "FBC watermarks: SR=%d, HPLL=%d\n",
1261 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1262 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1268 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1269 enum plane_id plane_id, int level)
1271 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1273 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1276 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1279 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1281 if (level > dev_priv->display.wm.max_level)
1284 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1285 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1286 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1289 /* mark all levels starting from 'level' as invalid */
1290 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1291 struct g4x_wm_state *wm_state, int level)
1293 if (level <= G4X_WM_LEVEL_NORMAL) {
1294 enum plane_id plane_id;
1296 for_each_plane_id_on_crtc(crtc, plane_id)
1297 wm_state->wm.plane[plane_id] = USHRT_MAX;
1300 if (level <= G4X_WM_LEVEL_SR) {
1301 wm_state->cxsr = false;
1302 wm_state->sr.cursor = USHRT_MAX;
1303 wm_state->sr.plane = USHRT_MAX;
1304 wm_state->sr.fbc = USHRT_MAX;
1307 if (level <= G4X_WM_LEVEL_HPLL) {
1308 wm_state->hpll_en = false;
1309 wm_state->hpll.cursor = USHRT_MAX;
1310 wm_state->hpll.plane = USHRT_MAX;
1311 wm_state->hpll.fbc = USHRT_MAX;
1315 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1318 if (level < G4X_WM_LEVEL_SR)
1321 if (level >= G4X_WM_LEVEL_SR &&
1322 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1325 if (level >= G4X_WM_LEVEL_HPLL &&
1326 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1332 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1333 struct intel_crtc *crtc)
1335 struct intel_crtc_state *crtc_state =
1336 intel_atomic_get_new_crtc_state(state, crtc);
1337 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1338 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1339 const struct g4x_pipe_wm *raw;
1340 const struct intel_plane_state *old_plane_state;
1341 const struct intel_plane_state *new_plane_state;
1342 struct intel_plane *plane;
1343 enum plane_id plane_id;
1345 unsigned int dirty = 0;
1347 for_each_oldnew_intel_plane_in_state(state, plane,
1349 new_plane_state, i) {
1350 if (new_plane_state->hw.crtc != &crtc->base &&
1351 old_plane_state->hw.crtc != &crtc->base)
1354 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1355 dirty |= BIT(plane->id);
1361 level = G4X_WM_LEVEL_NORMAL;
1362 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1365 raw = &crtc_state->wm.g4x.raw[level];
1366 for_each_plane_id_on_crtc(crtc, plane_id)
1367 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1369 level = G4X_WM_LEVEL_SR;
1370 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1373 raw = &crtc_state->wm.g4x.raw[level];
1374 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1375 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1376 wm_state->sr.fbc = raw->fbc;
1378 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1380 level = G4X_WM_LEVEL_HPLL;
1381 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1384 raw = &crtc_state->wm.g4x.raw[level];
1385 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1386 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1387 wm_state->hpll.fbc = raw->fbc;
1389 wm_state->hpll_en = wm_state->cxsr;
1394 if (level == G4X_WM_LEVEL_NORMAL)
1397 /* invalidate the higher levels */
1398 g4x_invalidate_wms(crtc, wm_state, level);
1401 * Determine if the FBC watermark(s) can be used. IF
1402 * this isn't the case we prefer to disable the FBC
1403 * watermark(s) rather than disable the SR/HPLL
1404 * level(s) entirely. 'level-1' is the highest valid
1407 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1412 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1413 struct intel_crtc *crtc)
1415 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1416 struct intel_crtc_state *new_crtc_state =
1417 intel_atomic_get_new_crtc_state(state, crtc);
1418 const struct intel_crtc_state *old_crtc_state =
1419 intel_atomic_get_old_crtc_state(state, crtc);
1420 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1421 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1422 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1423 enum plane_id plane_id;
1425 if (!new_crtc_state->hw.active ||
1426 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1427 *intermediate = *optimal;
1429 intermediate->cxsr = false;
1430 intermediate->hpll_en = false;
1434 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1435 !new_crtc_state->disable_cxsr;
1436 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1437 !new_crtc_state->disable_cxsr;
1438 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1440 for_each_plane_id_on_crtc(crtc, plane_id) {
1441 intermediate->wm.plane[plane_id] =
1442 max(optimal->wm.plane[plane_id],
1443 active->wm.plane[plane_id]);
1445 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1446 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1449 intermediate->sr.plane = max(optimal->sr.plane,
1451 intermediate->sr.cursor = max(optimal->sr.cursor,
1453 intermediate->sr.fbc = max(optimal->sr.fbc,
1456 intermediate->hpll.plane = max(optimal->hpll.plane,
1457 active->hpll.plane);
1458 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1459 active->hpll.cursor);
1460 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1463 drm_WARN_ON(&dev_priv->drm,
1464 (intermediate->sr.plane >
1465 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1466 intermediate->sr.cursor >
1467 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1468 intermediate->cxsr);
1469 drm_WARN_ON(&dev_priv->drm,
1470 (intermediate->sr.plane >
1471 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1472 intermediate->sr.cursor >
1473 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1474 intermediate->hpll_en);
1476 drm_WARN_ON(&dev_priv->drm,
1477 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1478 intermediate->fbc_en && intermediate->cxsr);
1479 drm_WARN_ON(&dev_priv->drm,
1480 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1481 intermediate->fbc_en && intermediate->hpll_en);
1485 * If our intermediate WM are identical to the final WM, then we can
1486 * omit the post-vblank programming; only update if it's different.
1488 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1489 new_crtc_state->wm.need_postvbl_update = true;
1494 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1495 struct g4x_wm_values *wm)
1497 struct intel_crtc *crtc;
1498 int num_active_pipes = 0;
1504 for_each_intel_crtc(&dev_priv->drm, crtc) {
1505 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1510 if (!wm_state->cxsr)
1512 if (!wm_state->hpll_en)
1513 wm->hpll_en = false;
1514 if (!wm_state->fbc_en)
1520 if (num_active_pipes != 1) {
1522 wm->hpll_en = false;
1526 for_each_intel_crtc(&dev_priv->drm, crtc) {
1527 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1528 enum pipe pipe = crtc->pipe;
1530 wm->pipe[pipe] = wm_state->wm;
1531 if (crtc->active && wm->cxsr)
1532 wm->sr = wm_state->sr;
1533 if (crtc->active && wm->hpll_en)
1534 wm->hpll = wm_state->hpll;
1538 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1540 struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1541 struct g4x_wm_values new_wm = {};
1543 g4x_merge_wm(dev_priv, &new_wm);
1545 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1548 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1549 _intel_set_memory_cxsr(dev_priv, false);
1551 g4x_write_wm_values(dev_priv, &new_wm);
1553 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1554 _intel_set_memory_cxsr(dev_priv, true);
1559 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1560 struct intel_crtc *crtc)
1562 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1563 const struct intel_crtc_state *crtc_state =
1564 intel_atomic_get_new_crtc_state(state, crtc);
1566 mutex_lock(&dev_priv->display.wm.wm_mutex);
1567 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1568 g4x_program_watermarks(dev_priv);
1569 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1572 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1573 struct intel_crtc *crtc)
1575 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1576 const struct intel_crtc_state *crtc_state =
1577 intel_atomic_get_new_crtc_state(state, crtc);
1579 if (!crtc_state->wm.need_postvbl_update)
1582 mutex_lock(&dev_priv->display.wm.wm_mutex);
1583 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1584 g4x_program_watermarks(dev_priv);
1585 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1588 /* latency must be in 0.1us units. */
1589 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1590 unsigned int htotal,
1593 unsigned int latency)
1597 ret = intel_wm_method2(pixel_rate, htotal,
1598 width, cpp, latency);
1599 ret = DIV_ROUND_UP(ret, 64);
1604 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1606 /* all latencies in usec */
1607 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1609 dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
1611 if (IS_CHERRYVIEW(dev_priv)) {
1612 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1613 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1615 dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1619 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1620 const struct intel_plane_state *plane_state,
1623 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1624 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1625 const struct drm_display_mode *pipe_mode =
1626 &crtc_state->hw.pipe_mode;
1627 unsigned int pixel_rate, htotal, cpp, width, wm;
1629 if (dev_priv->display.wm.pri_latency[level] == 0)
1632 if (!intel_wm_plane_visible(crtc_state, plane_state))
1635 cpp = plane_state->hw.fb->format->cpp[0];
1636 pixel_rate = crtc_state->pixel_rate;
1637 htotal = pipe_mode->crtc_htotal;
1638 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1640 if (plane->id == PLANE_CURSOR) {
1642 * FIXME the formula gives values that are
1643 * too big for the cursor FIFO, and hence we
1644 * would never be able to use cursors. For
1645 * now just hardcode the watermark.
1649 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1650 dev_priv->display.wm.pri_latency[level] * 10);
1653 return min_t(unsigned int, wm, USHRT_MAX);
1656 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1658 return (active_planes & (BIT(PLANE_SPRITE0) |
1659 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1662 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1664 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1665 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1666 const struct g4x_pipe_wm *raw =
1667 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1668 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1669 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1670 int num_active_planes = hweight8(active_planes);
1671 const int fifo_size = 511;
1672 int fifo_extra, fifo_left = fifo_size;
1673 int sprite0_fifo_extra = 0;
1674 unsigned int total_rate;
1675 enum plane_id plane_id;
1678 * When enabling sprite0 after sprite1 has already been enabled
1679 * we tend to get an underrun unless sprite0 already has some
1680 * FIFO space allcoated. Hence we always allocate at least one
1681 * cacheline for sprite0 whenever sprite1 is enabled.
1683 * All other plane enable sequences appear immune to this problem.
1685 if (vlv_need_sprite0_fifo_workaround(active_planes))
1686 sprite0_fifo_extra = 1;
1688 total_rate = raw->plane[PLANE_PRIMARY] +
1689 raw->plane[PLANE_SPRITE0] +
1690 raw->plane[PLANE_SPRITE1] +
1693 if (total_rate > fifo_size)
1696 if (total_rate == 0)
1699 for_each_plane_id_on_crtc(crtc, plane_id) {
1702 if ((active_planes & BIT(plane_id)) == 0) {
1703 fifo_state->plane[plane_id] = 0;
1707 rate = raw->plane[plane_id];
1708 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1709 fifo_left -= fifo_state->plane[plane_id];
1712 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1713 fifo_left -= sprite0_fifo_extra;
1715 fifo_state->plane[PLANE_CURSOR] = 63;
1717 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1719 /* spread the remainder evenly */
1720 for_each_plane_id_on_crtc(crtc, plane_id) {
1726 if ((active_planes & BIT(plane_id)) == 0)
1729 plane_extra = min(fifo_extra, fifo_left);
1730 fifo_state->plane[plane_id] += plane_extra;
1731 fifo_left -= plane_extra;
1734 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1736 /* give it all to the first plane if none are active */
1737 if (active_planes == 0) {
1738 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1739 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1745 /* mark all levels starting from 'level' as invalid */
1746 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1747 struct vlv_wm_state *wm_state, int level)
1749 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1751 for (; level < intel_wm_num_levels(dev_priv); level++) {
1752 enum plane_id plane_id;
1754 for_each_plane_id_on_crtc(crtc, plane_id)
1755 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1757 wm_state->sr[level].cursor = USHRT_MAX;
1758 wm_state->sr[level].plane = USHRT_MAX;
1762 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1767 return fifo_size - wm;
1771 * Starting from 'level' set all higher
1772 * levels to 'value' in the "raw" watermarks.
1774 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1775 int level, enum plane_id plane_id, u16 value)
1777 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1778 int num_levels = intel_wm_num_levels(dev_priv);
1781 for (; level < num_levels; level++) {
1782 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1784 dirty |= raw->plane[plane_id] != value;
1785 raw->plane[plane_id] = value;
1791 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1792 const struct intel_plane_state *plane_state)
1794 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1795 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1796 enum plane_id plane_id = plane->id;
1797 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1801 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1802 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1806 for (level = 0; level < num_levels; level++) {
1807 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1808 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1809 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1814 dirty |= raw->plane[plane_id] != wm;
1815 raw->plane[plane_id] = wm;
1818 /* mark all higher levels as invalid */
1819 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1823 drm_dbg_kms(&dev_priv->drm,
1824 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1826 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1827 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1828 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1833 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1834 enum plane_id plane_id, int level)
1836 const struct g4x_pipe_wm *raw =
1837 &crtc_state->wm.vlv.raw[level];
1838 const struct vlv_fifo_state *fifo_state =
1839 &crtc_state->wm.vlv.fifo_state;
1841 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1844 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1846 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1847 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1848 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1849 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1852 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1853 struct intel_crtc *crtc)
1855 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1856 struct intel_crtc_state *crtc_state =
1857 intel_atomic_get_new_crtc_state(state, crtc);
1858 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1859 const struct vlv_fifo_state *fifo_state =
1860 &crtc_state->wm.vlv.fifo_state;
1861 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1862 int num_active_planes = hweight8(active_planes);
1863 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1864 const struct intel_plane_state *old_plane_state;
1865 const struct intel_plane_state *new_plane_state;
1866 struct intel_plane *plane;
1867 enum plane_id plane_id;
1869 unsigned int dirty = 0;
1871 for_each_oldnew_intel_plane_in_state(state, plane,
1873 new_plane_state, i) {
1874 if (new_plane_state->hw.crtc != &crtc->base &&
1875 old_plane_state->hw.crtc != &crtc->base)
1878 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1879 dirty |= BIT(plane->id);
1883 * DSPARB registers may have been reset due to the
1884 * power well being turned off. Make sure we restore
1885 * them to a consistent state even if no primary/sprite
1886 * planes are initially active.
1889 crtc_state->fifo_changed = true;
1894 /* cursor changes don't warrant a FIFO recompute */
1895 if (dirty & ~BIT(PLANE_CURSOR)) {
1896 const struct intel_crtc_state *old_crtc_state =
1897 intel_atomic_get_old_crtc_state(state, crtc);
1898 const struct vlv_fifo_state *old_fifo_state =
1899 &old_crtc_state->wm.vlv.fifo_state;
1901 ret = vlv_compute_fifo(crtc_state);
1905 if (needs_modeset ||
1906 memcmp(old_fifo_state, fifo_state,
1907 sizeof(*fifo_state)) != 0)
1908 crtc_state->fifo_changed = true;
1911 /* initially allow all levels */
1912 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1914 * Note that enabling cxsr with no primary/sprite planes
1915 * enabled can wedge the pipe. Hence we only allow cxsr
1916 * with exactly one enabled primary/sprite plane.
1918 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1920 for (level = 0; level < wm_state->num_levels; level++) {
1921 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1922 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1924 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1927 for_each_plane_id_on_crtc(crtc, plane_id) {
1928 wm_state->wm[level].plane[plane_id] =
1929 vlv_invert_wm_value(raw->plane[plane_id],
1930 fifo_state->plane[plane_id]);
1933 wm_state->sr[level].plane =
1934 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1935 raw->plane[PLANE_SPRITE0],
1936 raw->plane[PLANE_SPRITE1]),
1939 wm_state->sr[level].cursor =
1940 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1947 /* limit to only levels we can actually handle */
1948 wm_state->num_levels = level;
1950 /* invalidate the higher levels */
1951 vlv_invalidate_wms(crtc, wm_state, level);
1956 #define VLV_FIFO(plane, value) \
1957 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1959 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1960 struct intel_crtc *crtc)
1962 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1963 struct intel_uncore *uncore = &dev_priv->uncore;
1964 const struct intel_crtc_state *crtc_state =
1965 intel_atomic_get_new_crtc_state(state, crtc);
1966 const struct vlv_fifo_state *fifo_state =
1967 &crtc_state->wm.vlv.fifo_state;
1968 int sprite0_start, sprite1_start, fifo_size;
1969 u32 dsparb, dsparb2, dsparb3;
1971 if (!crtc_state->fifo_changed)
1974 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1975 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1976 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1978 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1979 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1981 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1984 * uncore.lock serves a double purpose here. It allows us to
1985 * use the less expensive I915_{READ,WRITE}_FW() functions, and
1986 * it protects the DSPARB registers from getting clobbered by
1987 * parallel updates from multiple pipes.
1989 * intel_pipe_update_start() has already disabled interrupts
1990 * for us, so a plain spin_lock() is sufficient here.
1992 spin_lock(&uncore->lock);
1994 switch (crtc->pipe) {
1996 dsparb = intel_uncore_read_fw(uncore, DSPARB);
1997 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1999 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2000 VLV_FIFO(SPRITEB, 0xff));
2001 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2002 VLV_FIFO(SPRITEB, sprite1_start));
2004 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2005 VLV_FIFO(SPRITEB_HI, 0x1));
2006 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2007 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2009 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2010 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2013 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2014 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2016 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2017 VLV_FIFO(SPRITED, 0xff));
2018 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2019 VLV_FIFO(SPRITED, sprite1_start));
2021 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2022 VLV_FIFO(SPRITED_HI, 0xff));
2023 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2024 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2026 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2027 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2030 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2031 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2033 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2034 VLV_FIFO(SPRITEF, 0xff));
2035 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2036 VLV_FIFO(SPRITEF, sprite1_start));
2038 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2039 VLV_FIFO(SPRITEF_HI, 0xff));
2040 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2041 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2043 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2044 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2050 intel_uncore_posting_read_fw(uncore, DSPARB);
2052 spin_unlock(&uncore->lock);
2057 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2058 struct intel_crtc *crtc)
2060 struct intel_crtc_state *new_crtc_state =
2061 intel_atomic_get_new_crtc_state(state, crtc);
2062 const struct intel_crtc_state *old_crtc_state =
2063 intel_atomic_get_old_crtc_state(state, crtc);
2064 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2065 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2066 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2069 if (!new_crtc_state->hw.active ||
2070 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2071 *intermediate = *optimal;
2073 intermediate->cxsr = false;
2077 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2078 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2079 !new_crtc_state->disable_cxsr;
2081 for (level = 0; level < intermediate->num_levels; level++) {
2082 enum plane_id plane_id;
2084 for_each_plane_id_on_crtc(crtc, plane_id) {
2085 intermediate->wm[level].plane[plane_id] =
2086 min(optimal->wm[level].plane[plane_id],
2087 active->wm[level].plane[plane_id]);
2090 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2091 active->sr[level].plane);
2092 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2093 active->sr[level].cursor);
2096 vlv_invalidate_wms(crtc, intermediate, level);
2100 * If our intermediate WM are identical to the final WM, then we can
2101 * omit the post-vblank programming; only update if it's different.
2103 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2104 new_crtc_state->wm.need_postvbl_update = true;
2109 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2110 struct vlv_wm_values *wm)
2112 struct intel_crtc *crtc;
2113 int num_active_pipes = 0;
2115 wm->level = dev_priv->display.wm.max_level;
2118 for_each_intel_crtc(&dev_priv->drm, crtc) {
2119 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2124 if (!wm_state->cxsr)
2128 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2131 if (num_active_pipes != 1)
2134 if (num_active_pipes > 1)
2135 wm->level = VLV_WM_LEVEL_PM2;
2137 for_each_intel_crtc(&dev_priv->drm, crtc) {
2138 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2139 enum pipe pipe = crtc->pipe;
2141 wm->pipe[pipe] = wm_state->wm[wm->level];
2142 if (crtc->active && wm->cxsr)
2143 wm->sr = wm_state->sr[wm->level];
2145 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2146 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2147 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2148 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2152 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2154 struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2155 struct vlv_wm_values new_wm = {};
2157 vlv_merge_wm(dev_priv, &new_wm);
2159 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2162 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2163 chv_set_memory_dvfs(dev_priv, false);
2165 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2166 chv_set_memory_pm5(dev_priv, false);
2168 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2169 _intel_set_memory_cxsr(dev_priv, false);
2171 vlv_write_wm_values(dev_priv, &new_wm);
2173 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2174 _intel_set_memory_cxsr(dev_priv, true);
2176 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2177 chv_set_memory_pm5(dev_priv, true);
2179 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2180 chv_set_memory_dvfs(dev_priv, true);
2185 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2186 struct intel_crtc *crtc)
2188 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2189 const struct intel_crtc_state *crtc_state =
2190 intel_atomic_get_new_crtc_state(state, crtc);
2192 mutex_lock(&dev_priv->display.wm.wm_mutex);
2193 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2194 vlv_program_watermarks(dev_priv);
2195 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2198 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2199 struct intel_crtc *crtc)
2201 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2202 const struct intel_crtc_state *crtc_state =
2203 intel_atomic_get_new_crtc_state(state, crtc);
2205 if (!crtc_state->wm.need_postvbl_update)
2208 mutex_lock(&dev_priv->display.wm.wm_mutex);
2209 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2210 vlv_program_watermarks(dev_priv);
2211 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2214 static void i965_update_wm(struct drm_i915_private *dev_priv)
2216 struct intel_crtc *crtc;
2221 /* Calc sr entries for one plane configs */
2222 crtc = single_enabled_crtc(dev_priv);
2224 /* self-refresh has much higher latency */
2225 static const int sr_latency_ns = 12000;
2226 const struct drm_display_mode *pipe_mode =
2227 &crtc->config->hw.pipe_mode;
2228 const struct drm_framebuffer *fb =
2229 crtc->base.primary->state->fb;
2230 int pixel_rate = crtc->config->pixel_rate;
2231 int htotal = pipe_mode->crtc_htotal;
2232 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2233 int cpp = fb->format->cpp[0];
2236 entries = intel_wm_method2(pixel_rate, htotal,
2237 width, cpp, sr_latency_ns / 100);
2238 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2239 srwm = I965_FIFO_SIZE - entries;
2243 drm_dbg_kms(&dev_priv->drm,
2244 "self-refresh entries: %d, wm: %d\n",
2247 entries = intel_wm_method2(pixel_rate, htotal,
2248 crtc->base.cursor->state->crtc_w, 4,
2249 sr_latency_ns / 100);
2250 entries = DIV_ROUND_UP(entries,
2251 i965_cursor_wm_info.cacheline_size) +
2252 i965_cursor_wm_info.guard_size;
2254 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2255 if (cursor_sr > i965_cursor_wm_info.max_wm)
2256 cursor_sr = i965_cursor_wm_info.max_wm;
2258 drm_dbg_kms(&dev_priv->drm,
2259 "self-refresh watermark: display plane %d "
2260 "cursor %d\n", srwm, cursor_sr);
2262 cxsr_enabled = true;
2264 cxsr_enabled = false;
2265 /* Turn off self refresh if both pipes are enabled */
2266 intel_set_memory_cxsr(dev_priv, false);
2269 drm_dbg_kms(&dev_priv->drm,
2270 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2273 /* 965 has limitations... */
2274 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2278 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2279 FW_WM(8, PLANEC_OLD));
2280 /* update cursor SR watermark */
2281 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2284 intel_set_memory_cxsr(dev_priv, true);
2289 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2290 enum i9xx_plane_id i9xx_plane)
2292 struct intel_plane *plane;
2294 for_each_intel_plane(&i915->drm, plane) {
2295 if (plane->id == PLANE_PRIMARY &&
2296 plane->i9xx_plane == i9xx_plane)
2297 return intel_crtc_for_pipe(i915, plane->pipe);
2303 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2305 const struct intel_watermark_params *wm_info;
2310 int planea_wm, planeb_wm;
2311 struct intel_crtc *crtc;
2313 if (IS_I945GM(dev_priv))
2314 wm_info = &i945_wm_info;
2315 else if (DISPLAY_VER(dev_priv) != 2)
2316 wm_info = &i915_wm_info;
2318 wm_info = &i830_a_wm_info;
2320 if (DISPLAY_VER(dev_priv) == 2)
2321 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2323 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2324 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2325 if (intel_crtc_active(crtc)) {
2326 const struct drm_framebuffer *fb =
2327 crtc->base.primary->state->fb;
2330 if (DISPLAY_VER(dev_priv) == 2)
2333 cpp = fb->format->cpp[0];
2335 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2336 wm_info, fifo_size, cpp,
2337 pessimal_latency_ns);
2339 planea_wm = fifo_size - wm_info->guard_size;
2340 if (planea_wm > (long)wm_info->max_wm)
2341 planea_wm = wm_info->max_wm;
2344 if (DISPLAY_VER(dev_priv) == 2)
2345 wm_info = &i830_bc_wm_info;
2347 if (DISPLAY_VER(dev_priv) == 2)
2348 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2350 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2351 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2352 if (intel_crtc_active(crtc)) {
2353 const struct drm_framebuffer *fb =
2354 crtc->base.primary->state->fb;
2357 if (DISPLAY_VER(dev_priv) == 2)
2360 cpp = fb->format->cpp[0];
2362 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2363 wm_info, fifo_size, cpp,
2364 pessimal_latency_ns);
2366 planeb_wm = fifo_size - wm_info->guard_size;
2367 if (planeb_wm > (long)wm_info->max_wm)
2368 planeb_wm = wm_info->max_wm;
2371 drm_dbg_kms(&dev_priv->drm,
2372 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2374 crtc = single_enabled_crtc(dev_priv);
2375 if (IS_I915GM(dev_priv) && crtc) {
2376 struct drm_i915_gem_object *obj;
2378 obj = intel_fb_obj(crtc->base.primary->state->fb);
2380 /* self-refresh seems busted with untiled */
2381 if (!i915_gem_object_is_tiled(obj))
2386 * Overlay gets an aggressive default since video jitter is bad.
2390 /* Play safe and disable self-refresh before adjusting watermarks. */
2391 intel_set_memory_cxsr(dev_priv, false);
2393 /* Calc sr entries for one plane configs */
2394 if (HAS_FW_BLC(dev_priv) && crtc) {
2395 /* self-refresh has much higher latency */
2396 static const int sr_latency_ns = 6000;
2397 const struct drm_display_mode *pipe_mode =
2398 &crtc->config->hw.pipe_mode;
2399 const struct drm_framebuffer *fb =
2400 crtc->base.primary->state->fb;
2401 int pixel_rate = crtc->config->pixel_rate;
2402 int htotal = pipe_mode->crtc_htotal;
2403 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2407 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2410 cpp = fb->format->cpp[0];
2412 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2413 sr_latency_ns / 100);
2414 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2415 drm_dbg_kms(&dev_priv->drm,
2416 "self-refresh entries: %d\n", entries);
2417 srwm = wm_info->fifo_size - entries;
2421 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2422 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2423 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2425 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2428 drm_dbg_kms(&dev_priv->drm,
2429 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2430 planea_wm, planeb_wm, cwm, srwm);
2432 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2433 fwater_hi = (cwm & 0x1f);
2435 /* Set request length to 8 cachelines per fetch */
2436 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2437 fwater_hi = fwater_hi | (1 << 8);
2439 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2440 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2443 intel_set_memory_cxsr(dev_priv, true);
2446 static void i845_update_wm(struct drm_i915_private *dev_priv)
2448 struct intel_crtc *crtc;
2452 crtc = single_enabled_crtc(dev_priv);
2456 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2458 i845_get_fifo_size(dev_priv, PLANE_A),
2459 4, pessimal_latency_ns);
2460 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2461 fwater_lo |= (3<<8) | planea_wm;
2463 drm_dbg_kms(&dev_priv->drm,
2464 "Setting FIFO watermarks - A: %d\n", planea_wm);
2466 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2469 /* latency must be in 0.1us units. */
2470 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2472 unsigned int latency)
2476 ret = intel_wm_method1(pixel_rate, cpp, latency);
2477 ret = DIV_ROUND_UP(ret, 64) + 2;
2482 /* latency must be in 0.1us units. */
2483 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2484 unsigned int htotal,
2487 unsigned int latency)
2491 ret = intel_wm_method2(pixel_rate, htotal,
2492 width, cpp, latency);
2493 ret = DIV_ROUND_UP(ret, 64) + 2;
2498 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2501 * Neither of these should be possible since this function shouldn't be
2502 * called if the CRTC is off or the plane is invisible. But let's be
2503 * extra paranoid to avoid a potential divide-by-zero if we screw up
2504 * elsewhere in the driver.
2508 if (WARN_ON(!horiz_pixels))
2511 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2514 struct ilk_wm_maximums {
2522 * For both WM_PIPE and WM_LP.
2523 * mem_value must be in 0.1us units.
2525 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2526 const struct intel_plane_state *plane_state,
2527 u32 mem_value, bool is_lp)
2529 u32 method1, method2;
2535 if (!intel_wm_plane_visible(crtc_state, plane_state))
2538 cpp = plane_state->hw.fb->format->cpp[0];
2540 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2545 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2546 crtc_state->hw.pipe_mode.crtc_htotal,
2547 drm_rect_width(&plane_state->uapi.src) >> 16,
2550 return min(method1, method2);
2554 * For both WM_PIPE and WM_LP.
2555 * mem_value must be in 0.1us units.
2557 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2558 const struct intel_plane_state *plane_state,
2561 u32 method1, method2;
2567 if (!intel_wm_plane_visible(crtc_state, plane_state))
2570 cpp = plane_state->hw.fb->format->cpp[0];
2572 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2573 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2574 crtc_state->hw.pipe_mode.crtc_htotal,
2575 drm_rect_width(&plane_state->uapi.src) >> 16,
2577 return min(method1, method2);
2581 * For both WM_PIPE and WM_LP.
2582 * mem_value must be in 0.1us units.
2584 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2585 const struct intel_plane_state *plane_state,
2593 if (!intel_wm_plane_visible(crtc_state, plane_state))
2596 cpp = plane_state->hw.fb->format->cpp[0];
2598 return ilk_wm_method2(crtc_state->pixel_rate,
2599 crtc_state->hw.pipe_mode.crtc_htotal,
2600 drm_rect_width(&plane_state->uapi.src) >> 16,
2604 /* Only for WM_LP. */
2605 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2606 const struct intel_plane_state *plane_state,
2611 if (!intel_wm_plane_visible(crtc_state, plane_state))
2614 cpp = plane_state->hw.fb->format->cpp[0];
2616 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2621 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2623 if (DISPLAY_VER(dev_priv) >= 8)
2625 else if (DISPLAY_VER(dev_priv) >= 7)
2632 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2633 int level, bool is_sprite)
2635 if (DISPLAY_VER(dev_priv) >= 8)
2636 /* BDW primary/sprite plane watermarks */
2637 return level == 0 ? 255 : 2047;
2638 else if (DISPLAY_VER(dev_priv) >= 7)
2639 /* IVB/HSW primary/sprite plane watermarks */
2640 return level == 0 ? 127 : 1023;
2641 else if (!is_sprite)
2642 /* ILK/SNB primary plane watermarks */
2643 return level == 0 ? 127 : 511;
2645 /* ILK/SNB sprite plane watermarks */
2646 return level == 0 ? 63 : 255;
2650 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2652 if (DISPLAY_VER(dev_priv) >= 7)
2653 return level == 0 ? 63 : 255;
2655 return level == 0 ? 31 : 63;
2658 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2660 if (DISPLAY_VER(dev_priv) >= 8)
2666 /* Calculate the maximum primary/sprite plane watermark */
2667 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2669 const struct intel_wm_config *config,
2670 enum intel_ddb_partitioning ddb_partitioning,
2673 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2675 /* if sprites aren't enabled, sprites get nothing */
2676 if (is_sprite && !config->sprites_enabled)
2679 /* HSW allows LP1+ watermarks even with multiple pipes */
2680 if (level == 0 || config->num_pipes_active > 1) {
2681 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2684 * For some reason the non self refresh
2685 * FIFO size is only half of the self
2686 * refresh FIFO size on ILK/SNB.
2688 if (DISPLAY_VER(dev_priv) <= 6)
2692 if (config->sprites_enabled) {
2693 /* level 0 is always calculated with 1:1 split */
2694 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2703 /* clamp to max that the registers can hold */
2704 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2707 /* Calculate the maximum cursor plane watermark */
2708 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2710 const struct intel_wm_config *config)
2712 /* HSW LP1+ watermarks w/ multiple pipes */
2713 if (level > 0 && config->num_pipes_active > 1)
2716 /* otherwise just report max that registers can hold */
2717 return ilk_cursor_wm_reg_max(dev_priv, level);
2720 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2722 const struct intel_wm_config *config,
2723 enum intel_ddb_partitioning ddb_partitioning,
2724 struct ilk_wm_maximums *max)
2726 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2727 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2728 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2729 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2732 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2734 struct ilk_wm_maximums *max)
2736 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2737 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2738 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2739 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2742 static bool ilk_validate_wm_level(int level,
2743 const struct ilk_wm_maximums *max,
2744 struct intel_wm_level *result)
2748 /* already determined to be invalid? */
2749 if (!result->enable)
2752 result->enable = result->pri_val <= max->pri &&
2753 result->spr_val <= max->spr &&
2754 result->cur_val <= max->cur;
2756 ret = result->enable;
2759 * HACK until we can pre-compute everything,
2760 * and thus fail gracefully if LP0 watermarks
2763 if (level == 0 && !result->enable) {
2764 if (result->pri_val > max->pri)
2765 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2766 level, result->pri_val, max->pri);
2767 if (result->spr_val > max->spr)
2768 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2769 level, result->spr_val, max->spr);
2770 if (result->cur_val > max->cur)
2771 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2772 level, result->cur_val, max->cur);
2774 result->pri_val = min_t(u32, result->pri_val, max->pri);
2775 result->spr_val = min_t(u32, result->spr_val, max->spr);
2776 result->cur_val = min_t(u32, result->cur_val, max->cur);
2777 result->enable = true;
2783 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2784 const struct intel_crtc *crtc,
2786 struct intel_crtc_state *crtc_state,
2787 const struct intel_plane_state *pristate,
2788 const struct intel_plane_state *sprstate,
2789 const struct intel_plane_state *curstate,
2790 struct intel_wm_level *result)
2792 u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2793 u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2794 u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2796 /* WM1+ latency values stored in 0.5us units */
2804 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2805 pri_latency, level);
2806 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2810 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2813 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2815 result->enable = true;
2818 static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2822 sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2824 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2826 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2827 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2828 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2829 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2830 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2833 static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2837 sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2839 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2840 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2841 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2842 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2845 static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2849 mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2851 /* ILK primary LP0 latency is 700 ns */
2853 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2854 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2857 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2860 /* ILK sprite LP0 latency is 1300 ns */
2861 if (DISPLAY_VER(dev_priv) == 5)
2865 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2868 /* ILK cursor LP0 latency is 1300 ns */
2869 if (DISPLAY_VER(dev_priv) == 5)
2873 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2875 /* how many WM levels are we expecting */
2876 if (HAS_HW_SAGV_WM(dev_priv))
2878 else if (DISPLAY_VER(dev_priv) >= 9)
2880 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2882 else if (DISPLAY_VER(dev_priv) >= 6)
2888 void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2889 const char *name, const u16 wm[])
2891 int level, max_level = ilk_wm_max_level(dev_priv);
2893 for (level = 0; level <= max_level; level++) {
2894 unsigned int latency = wm[level];
2897 drm_dbg_kms(&dev_priv->drm,
2898 "%s WM%d latency not provided\n",
2904 * - latencies are in us on gen9.
2905 * - before then, WM1+ latency values are in 0.5us units
2907 if (DISPLAY_VER(dev_priv) >= 9)
2912 drm_dbg_kms(&dev_priv->drm,
2913 "%s WM%d latency %u (%u.%u usec)\n", name, level,
2914 wm[level], latency / 10, latency % 10);
2918 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2921 int level, max_level = ilk_wm_max_level(dev_priv);
2926 wm[0] = max(wm[0], min);
2927 for (level = 1; level <= max_level; level++)
2928 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2933 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2938 * The BIOS provided WM memory latency values are often
2939 * inadequate for high resolution displays. Adjust them.
2941 changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2942 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2943 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2948 drm_dbg_kms(&dev_priv->drm,
2949 "WM latency values increased to avoid potential underruns\n");
2950 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2951 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2952 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2955 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2958 * On some SNB machines (Thinkpad X220 Tablet at least)
2959 * LP3 usage can cause vblank interrupts to be lost.
2960 * The DEIIR bit will go high but it looks like the CPU
2961 * never gets interrupted.
2963 * It's not clear whether other interrupt source could
2964 * be affected or if this is somehow limited to vblank
2965 * interrupts only. To play it safe we disable LP3
2966 * watermarks entirely.
2968 if (dev_priv->display.wm.pri_latency[3] == 0 &&
2969 dev_priv->display.wm.spr_latency[3] == 0 &&
2970 dev_priv->display.wm.cur_latency[3] == 0)
2973 dev_priv->display.wm.pri_latency[3] = 0;
2974 dev_priv->display.wm.spr_latency[3] = 0;
2975 dev_priv->display.wm.cur_latency[3] = 0;
2977 drm_dbg_kms(&dev_priv->drm,
2978 "LP3 watermarks disabled due to potential for lost interrupts\n");
2979 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2980 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2981 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2984 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
2986 if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2987 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2988 else if (DISPLAY_VER(dev_priv) >= 6)
2989 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2991 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2993 memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
2994 sizeof(dev_priv->display.wm.pri_latency));
2995 memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
2996 sizeof(dev_priv->display.wm.pri_latency));
2998 intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
2999 intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
3001 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3002 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3003 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3005 if (DISPLAY_VER(dev_priv) == 6) {
3006 snb_wm_latency_quirk(dev_priv);
3007 snb_wm_lp3_irq_quirk(dev_priv);
3011 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3012 struct intel_pipe_wm *pipe_wm)
3014 /* LP0 watermark maximums depend on this pipe alone */
3015 const struct intel_wm_config config = {
3016 .num_pipes_active = 1,
3017 .sprites_enabled = pipe_wm->sprites_enabled,
3018 .sprites_scaled = pipe_wm->sprites_scaled,
3020 struct ilk_wm_maximums max;
3022 /* LP0 watermarks always use 1/2 DDB partitioning */
3023 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3025 /* At least LP0 must be valid */
3026 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3027 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3034 /* Compute new watermarks for the pipe */
3035 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3036 struct intel_crtc *crtc)
3038 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3039 struct intel_crtc_state *crtc_state =
3040 intel_atomic_get_new_crtc_state(state, crtc);
3041 struct intel_pipe_wm *pipe_wm;
3042 struct intel_plane *plane;
3043 const struct intel_plane_state *plane_state;
3044 const struct intel_plane_state *pristate = NULL;
3045 const struct intel_plane_state *sprstate = NULL;
3046 const struct intel_plane_state *curstate = NULL;
3047 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3048 struct ilk_wm_maximums max;
3050 pipe_wm = &crtc_state->wm.ilk.optimal;
3052 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3053 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3054 pristate = plane_state;
3055 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3056 sprstate = plane_state;
3057 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3058 curstate = plane_state;
3061 pipe_wm->pipe_enabled = crtc_state->hw.active;
3062 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3063 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3065 usable_level = max_level;
3067 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3068 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3071 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3072 if (pipe_wm->sprites_scaled)
3075 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3076 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3077 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3079 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3082 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3084 for (level = 1; level <= usable_level; level++) {
3085 struct intel_wm_level *wm = &pipe_wm->wm[level];
3087 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3088 pristate, sprstate, curstate, wm);
3091 * Disable any watermark level that exceeds the
3092 * register maximums since such watermarks are
3095 if (!ilk_validate_wm_level(level, &max, wm)) {
3096 memset(wm, 0, sizeof(*wm));
3105 * Build a set of 'intermediate' watermark values that satisfy both the old
3106 * state and the new state. These can be programmed to the hardware
3109 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3110 struct intel_crtc *crtc)
3112 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3113 struct intel_crtc_state *new_crtc_state =
3114 intel_atomic_get_new_crtc_state(state, crtc);
3115 const struct intel_crtc_state *old_crtc_state =
3116 intel_atomic_get_old_crtc_state(state, crtc);
3117 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3118 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3119 int level, max_level = ilk_wm_max_level(dev_priv);
3122 * Start with the final, target watermarks, then combine with the
3123 * currently active watermarks to get values that are safe both before
3124 * and after the vblank.
3126 *a = new_crtc_state->wm.ilk.optimal;
3127 if (!new_crtc_state->hw.active ||
3128 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) ||
3129 state->skip_intermediate_wm)
3132 a->pipe_enabled |= b->pipe_enabled;
3133 a->sprites_enabled |= b->sprites_enabled;
3134 a->sprites_scaled |= b->sprites_scaled;
3136 for (level = 0; level <= max_level; level++) {
3137 struct intel_wm_level *a_wm = &a->wm[level];
3138 const struct intel_wm_level *b_wm = &b->wm[level];
3140 a_wm->enable &= b_wm->enable;
3141 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3142 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3143 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3144 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3148 * We need to make sure that these merged watermark values are
3149 * actually a valid configuration themselves. If they're not,
3150 * there's no safe way to transition from the old state to
3151 * the new state, so we need to fail the atomic transaction.
3153 if (!ilk_validate_pipe_wm(dev_priv, a))
3157 * If our intermediate WM are identical to the final WM, then we can
3158 * omit the post-vblank programming; only update if it's different.
3160 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3161 new_crtc_state->wm.need_postvbl_update = true;
3167 * Merge the watermarks from all active pipes for a specific level.
3169 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3171 struct intel_wm_level *ret_wm)
3173 const struct intel_crtc *crtc;
3175 ret_wm->enable = true;
3177 for_each_intel_crtc(&dev_priv->drm, crtc) {
3178 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3179 const struct intel_wm_level *wm = &active->wm[level];
3181 if (!active->pipe_enabled)
3185 * The watermark values may have been used in the past,
3186 * so we must maintain them in the registers for some
3187 * time even if the level is now disabled.
3190 ret_wm->enable = false;
3192 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3193 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3194 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3195 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3200 * Merge all low power watermarks for all active pipes.
3202 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3203 const struct intel_wm_config *config,
3204 const struct ilk_wm_maximums *max,
3205 struct intel_pipe_wm *merged)
3207 int level, max_level = ilk_wm_max_level(dev_priv);
3208 int last_enabled_level = max_level;
3210 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3211 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3212 config->num_pipes_active > 1)
3213 last_enabled_level = 0;
3215 /* ILK: FBC WM must be disabled always */
3216 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3218 /* merge each WM1+ level */
3219 for (level = 1; level <= max_level; level++) {
3220 struct intel_wm_level *wm = &merged->wm[level];
3222 ilk_merge_wm_level(dev_priv, level, wm);
3224 if (level > last_enabled_level)
3226 else if (!ilk_validate_wm_level(level, max, wm))
3227 /* make sure all following levels get disabled */
3228 last_enabled_level = level - 1;
3231 * The spec says it is preferred to disable
3232 * FBC WMs instead of disabling a WM level.
3234 if (wm->fbc_val > max->fbc) {
3236 merged->fbc_wm_enabled = false;
3241 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3242 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3243 dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3244 for (level = 2; level <= max_level; level++) {
3245 struct intel_wm_level *wm = &merged->wm[level];
3252 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3254 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3255 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3258 /* The value we need to program into the WM_LPx latency field */
3259 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3262 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3265 return dev_priv->display.wm.pri_latency[level];
3268 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3269 const struct intel_pipe_wm *merged,
3270 enum intel_ddb_partitioning partitioning,
3271 struct ilk_wm_values *results)
3273 struct intel_crtc *crtc;
3276 results->enable_fbc_wm = merged->fbc_wm_enabled;
3277 results->partitioning = partitioning;
3279 /* LP1+ register values */
3280 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3281 const struct intel_wm_level *r;
3283 level = ilk_wm_lp_to_level(wm_lp, merged);
3285 r = &merged->wm[level];
3288 * Maintain the watermark values even if the level is
3289 * disabled. Doing otherwise could cause underruns.
3291 results->wm_lp[wm_lp - 1] =
3292 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3293 WM_LP_PRIMARY(r->pri_val) |
3294 WM_LP_CURSOR(r->cur_val);
3297 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3299 if (DISPLAY_VER(dev_priv) >= 8)
3300 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3302 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3304 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3307 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3308 * level is disabled. Doing otherwise could cause underruns.
3310 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3311 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3312 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3316 /* LP0 register values */
3317 for_each_intel_crtc(&dev_priv->drm, crtc) {
3318 enum pipe pipe = crtc->pipe;
3319 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3320 const struct intel_wm_level *r = &pipe_wm->wm[0];
3322 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3325 results->wm_pipe[pipe] =
3326 WM0_PIPE_PRIMARY(r->pri_val) |
3327 WM0_PIPE_SPRITE(r->spr_val) |
3328 WM0_PIPE_CURSOR(r->cur_val);
3332 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3333 * case both are at the same level. Prefer r1 in case they're the same. */
3334 static struct intel_pipe_wm *
3335 ilk_find_best_result(struct drm_i915_private *dev_priv,
3336 struct intel_pipe_wm *r1,
3337 struct intel_pipe_wm *r2)
3339 int level, max_level = ilk_wm_max_level(dev_priv);
3340 int level1 = 0, level2 = 0;
3342 for (level = 1; level <= max_level; level++) {
3343 if (r1->wm[level].enable)
3345 if (r2->wm[level].enable)
3349 if (level1 == level2) {
3350 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3354 } else if (level1 > level2) {
3361 /* dirty bits used to track which watermarks need changes */
3362 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3363 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3364 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3365 #define WM_DIRTY_FBC (1 << 24)
3366 #define WM_DIRTY_DDB (1 << 25)
3368 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3369 const struct ilk_wm_values *old,
3370 const struct ilk_wm_values *new)
3372 unsigned int dirty = 0;
3376 for_each_pipe(dev_priv, pipe) {
3377 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3378 dirty |= WM_DIRTY_PIPE(pipe);
3379 /* Must disable LP1+ watermarks too */
3380 dirty |= WM_DIRTY_LP_ALL;
3384 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3385 dirty |= WM_DIRTY_FBC;
3386 /* Must disable LP1+ watermarks too */
3387 dirty |= WM_DIRTY_LP_ALL;
3390 if (old->partitioning != new->partitioning) {
3391 dirty |= WM_DIRTY_DDB;
3392 /* Must disable LP1+ watermarks too */
3393 dirty |= WM_DIRTY_LP_ALL;
3396 /* LP1+ watermarks already deemed dirty, no need to continue */
3397 if (dirty & WM_DIRTY_LP_ALL)
3400 /* Find the lowest numbered LP1+ watermark in need of an update... */
3401 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3402 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3403 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3407 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3408 for (; wm_lp <= 3; wm_lp++)
3409 dirty |= WM_DIRTY_LP(wm_lp);
3414 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3417 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3418 bool changed = false;
3420 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3421 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3422 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3425 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3426 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3427 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3430 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3431 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3432 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3437 * Don't touch WM_LP_SPRITE_ENABLE here.
3438 * Doing so could cause underruns.
3445 * The spec says we shouldn't write when we don't need, because every write
3446 * causes WMs to be re-evaluated, expending some power.
3448 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3449 struct ilk_wm_values *results)
3451 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3455 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3459 _ilk_disable_lp_wm(dev_priv, dirty);
3461 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3462 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3463 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3464 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3465 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3466 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3468 if (dirty & WM_DIRTY_DDB) {
3469 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3470 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3471 if (results->partitioning == INTEL_DDB_PART_1_2)
3472 val &= ~WM_MISC_DATA_PARTITION_5_6;
3474 val |= WM_MISC_DATA_PARTITION_5_6;
3475 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3477 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3478 if (results->partitioning == INTEL_DDB_PART_1_2)
3479 val &= ~DISP_DATA_PARTITION_5_6;
3481 val |= DISP_DATA_PARTITION_5_6;
3482 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3486 if (dirty & WM_DIRTY_FBC) {
3487 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3488 if (results->enable_fbc_wm)
3489 val &= ~DISP_FBC_WM_DIS;
3491 val |= DISP_FBC_WM_DIS;
3492 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3495 if (dirty & WM_DIRTY_LP(1) &&
3496 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3497 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3499 if (DISPLAY_VER(dev_priv) >= 7) {
3500 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3501 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3502 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3503 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3506 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3507 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3508 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3509 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3510 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3511 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3513 dev_priv->display.wm.hw = *results;
3516 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3518 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3521 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3522 struct intel_wm_config *config)
3524 struct intel_crtc *crtc;
3526 /* Compute the currently _active_ config */
3527 for_each_intel_crtc(&dev_priv->drm, crtc) {
3528 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3530 if (!wm->pipe_enabled)
3533 config->sprites_enabled |= wm->sprites_enabled;
3534 config->sprites_scaled |= wm->sprites_scaled;
3535 config->num_pipes_active++;
3539 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3541 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3542 struct ilk_wm_maximums max;
3543 struct intel_wm_config config = {};
3544 struct ilk_wm_values results = {};
3545 enum intel_ddb_partitioning partitioning;
3547 ilk_compute_wm_config(dev_priv, &config);
3549 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3550 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3552 /* 5/6 split only in single pipe config on IVB+ */
3553 if (DISPLAY_VER(dev_priv) >= 7 &&
3554 config.num_pipes_active == 1 && config.sprites_enabled) {
3555 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3556 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3558 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3560 best_lp_wm = &lp_wm_1_2;
3563 partitioning = (best_lp_wm == &lp_wm_1_2) ?
3564 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3566 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3568 ilk_write_wm_values(dev_priv, &results);
3571 static void ilk_initial_watermarks(struct intel_atomic_state *state,
3572 struct intel_crtc *crtc)
3574 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3575 const struct intel_crtc_state *crtc_state =
3576 intel_atomic_get_new_crtc_state(state, crtc);
3578 mutex_lock(&dev_priv->display.wm.wm_mutex);
3579 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3580 ilk_program_watermarks(dev_priv);
3581 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3584 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3585 struct intel_crtc *crtc)
3587 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3588 const struct intel_crtc_state *crtc_state =
3589 intel_atomic_get_new_crtc_state(state, crtc);
3591 if (!crtc_state->wm.need_postvbl_update)
3594 mutex_lock(&dev_priv->display.wm.wm_mutex);
3595 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3596 ilk_program_watermarks(dev_priv);
3597 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3600 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3602 struct drm_device *dev = crtc->base.dev;
3603 struct drm_i915_private *dev_priv = to_i915(dev);
3604 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3605 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3606 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3607 enum pipe pipe = crtc->pipe;
3609 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3611 memset(active, 0, sizeof(*active));
3613 active->pipe_enabled = crtc->active;
3615 if (active->pipe_enabled) {
3616 u32 tmp = hw->wm_pipe[pipe];
3619 * For active pipes LP0 watermark is marked as
3620 * enabled, and LP1+ watermaks as disabled since
3621 * we can't really reverse compute them in case
3622 * multiple pipes are active.
3624 active->wm[0].enable = true;
3625 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3626 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3627 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3629 int level, max_level = ilk_wm_max_level(dev_priv);
3632 * For inactive pipes, all watermark levels
3633 * should be marked as enabled but zeroed,
3634 * which is what we'd compute them to.
3636 for (level = 0; level <= max_level; level++)
3637 active->wm[level].enable = true;
3640 crtc->wm.active.ilk = *active;
3643 #define _FW_WM(value, plane) \
3644 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3645 #define _FW_WM_VLV(value, plane) \
3646 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3648 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3649 struct g4x_wm_values *wm)
3653 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3654 wm->sr.plane = _FW_WM(tmp, SR);
3655 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3656 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3657 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3659 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3660 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3661 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3662 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3663 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3664 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3665 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3667 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3668 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3669 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3670 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3671 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3674 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3675 struct vlv_wm_values *wm)
3680 for_each_pipe(dev_priv, pipe) {
3681 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3683 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3684 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3685 wm->ddl[pipe].plane[PLANE_CURSOR] =
3686 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3687 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3688 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3689 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3690 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3693 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3694 wm->sr.plane = _FW_WM(tmp, SR);
3695 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3696 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3697 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3699 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3700 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3701 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3702 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3704 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3705 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3707 if (IS_CHERRYVIEW(dev_priv)) {
3708 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3709 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3710 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3712 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3713 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3714 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3716 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3717 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3718 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3720 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3721 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3722 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3723 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3724 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3725 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3726 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3727 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3728 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3729 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3730 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3732 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3733 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3734 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3736 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3737 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3738 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3739 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3740 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3741 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3742 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3743 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3750 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3752 struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3753 struct intel_crtc *crtc;
3755 g4x_read_wm_values(dev_priv, wm);
3757 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3759 for_each_intel_crtc(&dev_priv->drm, crtc) {
3760 struct intel_crtc_state *crtc_state =
3761 to_intel_crtc_state(crtc->base.state);
3762 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3763 struct g4x_pipe_wm *raw;
3764 enum pipe pipe = crtc->pipe;
3765 enum plane_id plane_id;
3766 int level, max_level;
3768 active->cxsr = wm->cxsr;
3769 active->hpll_en = wm->hpll_en;
3770 active->fbc_en = wm->fbc_en;
3772 active->sr = wm->sr;
3773 active->hpll = wm->hpll;
3775 for_each_plane_id_on_crtc(crtc, plane_id) {
3776 active->wm.plane[plane_id] =
3777 wm->pipe[pipe].plane[plane_id];
3780 if (wm->cxsr && wm->hpll_en)
3781 max_level = G4X_WM_LEVEL_HPLL;
3783 max_level = G4X_WM_LEVEL_SR;
3785 max_level = G4X_WM_LEVEL_NORMAL;
3787 level = G4X_WM_LEVEL_NORMAL;
3788 raw = &crtc_state->wm.g4x.raw[level];
3789 for_each_plane_id_on_crtc(crtc, plane_id)
3790 raw->plane[plane_id] = active->wm.plane[plane_id];
3792 level = G4X_WM_LEVEL_SR;
3793 if (level > max_level)
3796 raw = &crtc_state->wm.g4x.raw[level];
3797 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3798 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3799 raw->plane[PLANE_SPRITE0] = 0;
3800 raw->fbc = active->sr.fbc;
3802 level = G4X_WM_LEVEL_HPLL;
3803 if (level > max_level)
3806 raw = &crtc_state->wm.g4x.raw[level];
3807 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3808 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3809 raw->plane[PLANE_SPRITE0] = 0;
3810 raw->fbc = active->hpll.fbc;
3814 for_each_plane_id_on_crtc(crtc, plane_id)
3815 g4x_raw_plane_wm_set(crtc_state, level,
3816 plane_id, USHRT_MAX);
3817 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3819 crtc_state->wm.g4x.optimal = *active;
3820 crtc_state->wm.g4x.intermediate = *active;
3822 drm_dbg_kms(&dev_priv->drm,
3823 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3825 wm->pipe[pipe].plane[PLANE_PRIMARY],
3826 wm->pipe[pipe].plane[PLANE_CURSOR],
3827 wm->pipe[pipe].plane[PLANE_SPRITE0]);
3830 drm_dbg_kms(&dev_priv->drm,
3831 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3832 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3833 drm_dbg_kms(&dev_priv->drm,
3834 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3835 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3836 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3837 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3838 str_yes_no(wm->fbc_en));
3841 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3843 struct intel_plane *plane;
3844 struct intel_crtc *crtc;
3846 mutex_lock(&dev_priv->display.wm.wm_mutex);
3848 for_each_intel_plane(&dev_priv->drm, plane) {
3849 struct intel_crtc *crtc =
3850 intel_crtc_for_pipe(dev_priv, plane->pipe);
3851 struct intel_crtc_state *crtc_state =
3852 to_intel_crtc_state(crtc->base.state);
3853 struct intel_plane_state *plane_state =
3854 to_intel_plane_state(plane->base.state);
3855 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
3856 enum plane_id plane_id = plane->id;
3859 if (plane_state->uapi.visible)
3862 for (level = 0; level < 3; level++) {
3863 struct g4x_pipe_wm *raw =
3864 &crtc_state->wm.g4x.raw[level];
3866 raw->plane[plane_id] = 0;
3867 wm_state->wm.plane[plane_id] = 0;
3870 if (plane_id == PLANE_PRIMARY) {
3871 for (level = 0; level < 3; level++) {
3872 struct g4x_pipe_wm *raw =
3873 &crtc_state->wm.g4x.raw[level];
3877 wm_state->sr.fbc = 0;
3878 wm_state->hpll.fbc = 0;
3879 wm_state->fbc_en = false;
3883 for_each_intel_crtc(&dev_priv->drm, crtc) {
3884 struct intel_crtc_state *crtc_state =
3885 to_intel_crtc_state(crtc->base.state);
3887 crtc_state->wm.g4x.intermediate =
3888 crtc_state->wm.g4x.optimal;
3889 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3892 g4x_program_watermarks(dev_priv);
3894 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3897 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3899 struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3900 struct intel_crtc *crtc;
3903 vlv_read_wm_values(dev_priv, wm);
3905 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3906 wm->level = VLV_WM_LEVEL_PM2;
3908 if (IS_CHERRYVIEW(dev_priv)) {
3909 vlv_punit_get(dev_priv);
3911 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3912 if (val & DSP_MAXFIFO_PM5_ENABLE)
3913 wm->level = VLV_WM_LEVEL_PM5;
3916 * If DDR DVFS is disabled in the BIOS, Punit
3917 * will never ack the request. So if that happens
3918 * assume we don't have to enable/disable DDR DVFS
3919 * dynamically. To test that just set the REQ_ACK
3920 * bit to poke the Punit, but don't change the
3921 * HIGH/LOW bits so that we don't actually change
3922 * the current state.
3924 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3925 val |= FORCE_DDR_FREQ_REQ_ACK;
3926 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3928 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3929 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3930 drm_dbg_kms(&dev_priv->drm,
3931 "Punit not acking DDR DVFS request, "
3932 "assuming DDR DVFS is disabled\n");
3933 dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
3935 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3936 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3937 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3940 vlv_punit_put(dev_priv);
3943 for_each_intel_crtc(&dev_priv->drm, crtc) {
3944 struct intel_crtc_state *crtc_state =
3945 to_intel_crtc_state(crtc->base.state);
3946 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3947 const struct vlv_fifo_state *fifo_state =
3948 &crtc_state->wm.vlv.fifo_state;
3949 enum pipe pipe = crtc->pipe;
3950 enum plane_id plane_id;
3953 vlv_get_fifo_size(crtc_state);
3955 active->num_levels = wm->level + 1;
3956 active->cxsr = wm->cxsr;
3958 for (level = 0; level < active->num_levels; level++) {
3959 struct g4x_pipe_wm *raw =
3960 &crtc_state->wm.vlv.raw[level];
3962 active->sr[level].plane = wm->sr.plane;
3963 active->sr[level].cursor = wm->sr.cursor;
3965 for_each_plane_id_on_crtc(crtc, plane_id) {
3966 active->wm[level].plane[plane_id] =
3967 wm->pipe[pipe].plane[plane_id];
3969 raw->plane[plane_id] =
3970 vlv_invert_wm_value(active->wm[level].plane[plane_id],
3971 fifo_state->plane[plane_id]);
3975 for_each_plane_id_on_crtc(crtc, plane_id)
3976 vlv_raw_plane_wm_set(crtc_state, level,
3977 plane_id, USHRT_MAX);
3978 vlv_invalidate_wms(crtc, active, level);
3980 crtc_state->wm.vlv.optimal = *active;
3981 crtc_state->wm.vlv.intermediate = *active;
3983 drm_dbg_kms(&dev_priv->drm,
3984 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3986 wm->pipe[pipe].plane[PLANE_PRIMARY],
3987 wm->pipe[pipe].plane[PLANE_CURSOR],
3988 wm->pipe[pipe].plane[PLANE_SPRITE0],
3989 wm->pipe[pipe].plane[PLANE_SPRITE1]);
3992 drm_dbg_kms(&dev_priv->drm,
3993 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3994 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3997 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3999 struct intel_plane *plane;
4000 struct intel_crtc *crtc;
4002 mutex_lock(&dev_priv->display.wm.wm_mutex);
4004 for_each_intel_plane(&dev_priv->drm, plane) {
4005 struct intel_crtc *crtc =
4006 intel_crtc_for_pipe(dev_priv, plane->pipe);
4007 struct intel_crtc_state *crtc_state =
4008 to_intel_crtc_state(crtc->base.state);
4009 struct intel_plane_state *plane_state =
4010 to_intel_plane_state(plane->base.state);
4011 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
4012 const struct vlv_fifo_state *fifo_state =
4013 &crtc_state->wm.vlv.fifo_state;
4014 enum plane_id plane_id = plane->id;
4017 if (plane_state->uapi.visible)
4020 for (level = 0; level < wm_state->num_levels; level++) {
4021 struct g4x_pipe_wm *raw =
4022 &crtc_state->wm.vlv.raw[level];
4024 raw->plane[plane_id] = 0;
4026 wm_state->wm[level].plane[plane_id] =
4027 vlv_invert_wm_value(raw->plane[plane_id],
4028 fifo_state->plane[plane_id]);
4032 for_each_intel_crtc(&dev_priv->drm, crtc) {
4033 struct intel_crtc_state *crtc_state =
4034 to_intel_crtc_state(crtc->base.state);
4036 crtc_state->wm.vlv.intermediate =
4037 crtc_state->wm.vlv.optimal;
4038 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4041 vlv_program_watermarks(dev_priv);
4043 mutex_unlock(&dev_priv->display.wm.wm_mutex);
4047 * FIXME should probably kill this and improve
4048 * the real watermark readout/sanitation instead
4050 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
4052 intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
4053 intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
4054 intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
4057 * Don't touch WM_LP_SPRITE_ENABLE here.
4058 * Doing so could cause underruns.
4062 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
4064 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
4065 struct intel_crtc *crtc;
4067 ilk_init_lp_watermarks(dev_priv);
4069 for_each_intel_crtc(&dev_priv->drm, crtc)
4070 ilk_pipe_wm_get_hw_state(crtc);
4072 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
4073 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
4074 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
4076 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
4077 if (DISPLAY_VER(dev_priv) >= 7) {
4078 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
4079 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
4082 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4083 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4084 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4085 else if (IS_IVYBRIDGE(dev_priv))
4086 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4087 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4090 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4093 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
4096 * On Ibex Peak and Cougar Point, we need to disable clock
4097 * gating for the panel power sequencer or it will fail to
4098 * start up when no ports are active.
4100 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
4103 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
4107 for_each_pipe(dev_priv, pipe) {
4108 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
4110 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
4111 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
4115 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
4117 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4121 * WaFbcDisableDpfcClockGating:ilk
4123 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
4124 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
4125 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
4127 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
4128 MARIUNIT_CLOCK_GATE_DISABLE |
4129 SVSMUNIT_CLOCK_GATE_DISABLE);
4130 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
4131 VFMUNIT_CLOCK_GATE_DISABLE);
4134 * According to the spec the following bits should be set in
4135 * order to enable memory self-refresh
4136 * The bit 22/21 of 0x42004
4137 * The bit 5 of 0x42020
4138 * The bit 15 of 0x45000
4140 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4141 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4142 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4143 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
4144 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
4145 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4149 * Based on the document from hardware guys the following bits
4150 * should be set unconditionally in order to enable FBC.
4151 * The bit 22 of 0x42000
4152 * The bit 22 of 0x42004
4153 * The bit 7,8,9 of 0x42020.
4155 if (IS_IRONLAKE_M(dev_priv)) {
4156 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
4157 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4158 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
4161 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4163 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4165 g4x_disable_trickle_feed(dev_priv);
4167 ibx_init_clock_gating(dev_priv);
4170 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
4176 * On Ibex Peak and Cougar Point, we need to disable clock
4177 * gating for the panel power sequencer or it will fail to
4178 * start up when no ports are active.
4180 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
4181 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
4182 PCH_CPUNIT_CLOCK_GATE_DISABLE);
4183 intel_uncore_rmw(&dev_priv->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
4184 /* The below fixes the weird display corruption, a few pixels shifted
4185 * downward, on (only) LVDS of some HP laptops with IVY.
4187 for_each_pipe(dev_priv, pipe) {
4188 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
4189 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
4190 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4191 if (dev_priv->display.vbt.fdi_rx_polarity_inverted)
4192 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4193 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
4194 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
4195 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
4197 /* WADP0ClockGatingDisable */
4198 for_each_pipe(dev_priv, pipe) {
4199 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
4200 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4204 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
4208 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
4209 if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
4210 drm_dbg_kms(&dev_priv->drm,
4211 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
4215 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
4217 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4219 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4221 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
4223 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
4224 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
4225 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
4226 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4228 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4229 * gating disable must be set. Failure to set it results in
4230 * flickering pixels due to Z write ordering failures after
4231 * some amount of runtime in the Mesa "fire" demo, and Unigine
4232 * Sanctuary and Tropics, and apparently anything else with
4233 * alpha test or pixel discard.
4235 * According to the spec, bit 11 (RCCUNIT) must also be set,
4236 * but we didn't debug actual testcases to find it out.
4238 * WaDisableRCCUnitClockGating:snb
4239 * WaDisableRCPBUnitClockGating:snb
4241 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4242 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4243 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4246 * According to the spec the following bits should be
4247 * set in order to enable memory self-refresh and fbc:
4248 * The bit21 and bit22 of 0x42000
4249 * The bit21 and bit22 of 0x42004
4250 * The bit5 and bit7 of 0x42020
4251 * The bit14 of 0x70180
4252 * The bit14 of 0x71180
4254 * WaFbcAsynchFlipDisableFbcQueue:snb
4256 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
4257 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
4258 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
4259 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4260 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4261 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
4262 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
4263 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
4264 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
4265 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
4267 g4x_disable_trickle_feed(dev_priv);
4269 cpt_init_clock_gating(dev_priv);
4271 gen6_check_mch_setup(dev_priv);
4274 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
4277 * TODO: this bit should only be enabled when really needed, then
4278 * disabled when not needed anymore in order to save power.
4280 if (HAS_PCH_LPT_LP(dev_priv))
4281 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
4282 0, PCH_LP_PARTITION_LEVEL_DISABLE);
4284 /* WADPOClockGatingDisable:hsw */
4285 intel_uncore_rmw(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
4286 0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4289 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
4291 if (HAS_PCH_LPT_LP(dev_priv)) {
4292 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
4294 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4295 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
4299 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
4300 int general_prio_credits,
4301 int high_prio_credits)
4306 /* WaTempDisableDOPClkGating:bdw */
4307 misccpctl = intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4308 GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4310 val = intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4311 val &= ~L3_PRIO_CREDITS_MASK;
4312 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
4313 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
4314 intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_L3SQCREG1, val);
4317 * Wait at least 100 clocks before re-enabling clock gating.
4318 * See the definition of L3SQCREG1 in BSpec.
4320 intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
4322 intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_MISCCPCTL, misccpctl);
4325 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
4327 /* Wa_1409120013:icl,ehl */
4328 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4329 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4331 /*Wa_14010594013:icl, ehl */
4332 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
4333 0, ICL_DELAY_PMRSP);
4336 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
4339 if (DISPLAY_VER(dev_priv) == 12)
4340 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4341 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4343 /* Wa_1409825376:tgl (pre-prod)*/
4344 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
4345 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, TGL_VRH_GATING_DIS);
4347 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
4348 if (DISPLAY_VER(dev_priv) == 12)
4349 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
4350 CLKREQ_POLICY_MEM_UP_OVRD, 0);
4353 static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
4355 gen12lp_init_clock_gating(dev_priv);
4357 /* Wa_22011091694:adlp */
4358 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
4360 /* Bspec/49189 Initialize Sequence */
4361 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
4364 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
4366 gen12lp_init_clock_gating(dev_priv);
4368 /* Wa_1409836686:dg1[a0] */
4369 if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4370 intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, DPT_GATING_DIS);
4373 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
4375 /* Wa_22010146351:xehpsdv */
4376 if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4377 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4380 static void dg2_init_clock_gating(struct drm_i915_private *i915)
4382 /* Wa_22010954014:dg2 */
4383 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4387 * Wa_14010733611:dg2_g10
4388 * Wa_22010146351:dg2_g10
4390 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
4391 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4392 SGR_DIS | SGGI_DIS);
4395 static void pvc_init_clock_gating(struct drm_i915_private *dev_priv)
4397 /* Wa_14012385139:pvc */
4398 if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4399 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4401 /* Wa_22010954014:pvc */
4402 if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4403 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
4406 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
4408 if (!HAS_PCH_CNP(dev_priv))
4411 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
4412 intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
4415 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
4417 cnp_init_clock_gating(dev_priv);
4418 gen9_init_clock_gating(dev_priv);
4420 /* WAC6entrylatency:cfl */
4421 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4424 * WaFbcTurnOffFbcWatermark:cfl
4425 * Display WA #0562: cfl
4427 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4430 * WaFbcNukeOnHostModify:cfl
4431 * Display WA #0873: cfl
4433 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4434 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4437 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
4439 gen9_init_clock_gating(dev_priv);
4441 /* WAC6entrylatency:kbl */
4442 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4444 /* WaDisableSDEUnitClockGating:kbl */
4445 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4446 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6,
4447 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4449 /* WaDisableGamClockGating:kbl */
4450 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4451 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1,
4452 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
4455 * WaFbcTurnOffFbcWatermark:kbl
4456 * Display WA #0562: kbl
4458 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4461 * WaFbcNukeOnHostModify:kbl
4462 * Display WA #0873: kbl
4464 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4465 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4468 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
4470 gen9_init_clock_gating(dev_priv);
4472 /* WaDisableDopClockGating:skl */
4473 intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
4474 GEN8_DOP_CLOCK_GATE_ENABLE, 0);
4476 /* WAC6entrylatency:skl */
4477 intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
4480 * WaFbcTurnOffFbcWatermark:skl
4481 * Display WA #0562: skl
4483 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
4486 * WaFbcNukeOnHostModify:skl
4487 * Display WA #0873: skl
4489 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4490 0, DPFC_NUKE_ON_ANY_MODIFICATION);
4493 * WaFbcHighMemBwCorruptionAvoidance:skl
4494 * Display WA #0883: skl
4496 intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
4499 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
4503 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4504 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4506 /* WaSwitchSolVfFArbitrationPriority:bdw */
4507 intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4509 /* WaPsrDPAMaskVBlankInSRD:bdw */
4510 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
4512 for_each_pipe(dev_priv, pipe) {
4513 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
4514 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
4515 0, BDW_DPRS_MASK_VBLANK_SRD);
4518 /* WaVSRefCountFullforceMissDisable:bdw */
4519 /* WaDSRefCountFullforceMissDisable:bdw */
4520 intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4521 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4523 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4524 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4526 /* WaDisableSDEUnitClockGating:bdw */
4527 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4529 /* WaProgramL3SqcReg1Default:bdw */
4530 gen8_set_l3sqc_credits(dev_priv, 30, 2);
4532 /* WaKVMNotificationOnConfigChange:bdw */
4533 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR2_1,
4534 0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
4536 lpt_init_clock_gating(dev_priv);
4538 /* WaDisableDopClockGating:bdw
4540 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
4543 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
4546 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
4548 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4549 intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
4551 /* This is required by WaCatErrorRejectionIssue:hsw */
4552 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4553 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4555 /* WaSwitchSolVfFArbitrationPriority:hsw */
4556 intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
4558 lpt_init_clock_gating(dev_priv);
4561 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
4565 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
4567 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
4568 intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
4570 /* WaDisableBackToBackFlipFix:ivb */
4571 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4572 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4573 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4575 if (IS_IVB_GT1(dev_priv))
4576 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4577 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4579 /* must write both registers */
4580 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4581 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4582 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
4583 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4587 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4588 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
4590 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4591 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4593 /* This is required by WaCatErrorRejectionIssue:ivb */
4594 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4595 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4597 g4x_disable_trickle_feed(dev_priv);
4599 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
4600 snpcr &= ~GEN6_MBC_SNPCR_MASK;
4601 snpcr |= GEN6_MBC_SNPCR_MED;
4602 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
4604 if (!HAS_PCH_NOP(dev_priv))
4605 cpt_init_clock_gating(dev_priv);
4607 gen6_check_mch_setup(dev_priv);
4610 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
4612 /* WaDisableBackToBackFlipFix:vlv */
4613 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4614 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4615 CHICKEN3_DGMG_DONE_FIX_DISABLE);
4617 /* WaDisableDopClockGating:vlv */
4618 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4619 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4621 /* This is required by WaCatErrorRejectionIssue:vlv */
4622 intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4623 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4626 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4627 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
4629 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4630 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4632 /* WaDisableL3Bank2xClockGate:vlv
4633 * Disabling L3 clock gating- MMIO 940c[25] = 1
4634 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
4635 intel_uncore_rmw(&dev_priv->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
4638 * WaDisableVLVClockGating_VBIIssue:vlv
4639 * Disable clock gating on th GCFG unit to prevent a delay
4640 * in the reporting of vblank events.
4642 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
4645 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
4647 /* WaVSRefCountFullforceMissDisable:chv */
4648 /* WaDSRefCountFullforceMissDisable:chv */
4649 intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4650 GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
4652 /* WaDisableSemaphoreAndSyncFlipWait:chv */
4653 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4654 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4656 /* WaDisableCSUnitClockGating:chv */
4657 intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4659 /* WaDisableSDEUnitClockGating:chv */
4660 intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4663 * WaProgramL3SqcReg1Default:chv
4664 * See gfxspecs/Related Documents/Performance Guide/
4665 * LSQC Setting Recommendations.
4667 gen8_set_l3sqc_credits(dev_priv, 38, 2);
4670 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
4674 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
4675 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
4676 GS_UNIT_CLOCK_GATE_DISABLE |
4677 CL_UNIT_CLOCK_GATE_DISABLE);
4678 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
4679 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
4680 OVRUNIT_CLOCK_GATE_DISABLE |
4681 OVCUNIT_CLOCK_GATE_DISABLE;
4682 if (IS_GM45(dev_priv))
4683 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
4684 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate);
4686 g4x_disable_trickle_feed(dev_priv);
4689 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
4691 struct intel_uncore *uncore = &dev_priv->uncore;
4693 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
4694 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
4695 intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0);
4696 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
4697 intel_uncore_write16(uncore, DEUC, 0);
4698 intel_uncore_write(uncore,
4700 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4703 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
4705 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
4706 I965_RCC_CLOCK_GATE_DISABLE |
4707 I965_RCPB_CLOCK_GATE_DISABLE |
4708 I965_ISC_CLOCK_GATE_DISABLE |
4709 I965_FBC_CLOCK_GATE_DISABLE);
4710 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
4711 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4712 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4715 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
4717 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
4719 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
4720 DSTATE_DOT_CLOCK_GATING;
4721 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
4723 if (IS_PINEVIEW(dev_priv))
4724 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4725 _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
4727 /* IIR "flip pending" means done if this bit is set */
4728 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4729 _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
4731 /* interrupts should cause a wake up from C3 */
4732 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
4734 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4735 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4737 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4738 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4741 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
4743 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
4745 /* interrupts should cause a wake up from C3 */
4746 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
4747 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
4749 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4750 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
4753 * Have FBC ignore 3D activity since we use software
4754 * render tracking, and otherwise a pure 3D workload
4755 * (even if it just renders a single frame and then does
4756 * abosultely nothing) would not allow FBC to recompress
4757 * until a 2D blit occurs.
4759 intel_uncore_write(&dev_priv->uncore, SCPD0,
4760 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
4763 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
4765 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4766 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
4767 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
4770 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
4772 dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
4775 void intel_suspend_hw(struct drm_i915_private *dev_priv)
4777 if (HAS_PCH_LPT(dev_priv))
4778 lpt_suspend_hw(dev_priv);
4781 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
4783 drm_dbg_kms(&dev_priv->drm,
4784 "No clock gating settings or workarounds applied.\n");
4787 #define CG_FUNCS(platform) \
4788 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
4789 .init_clock_gating = platform##_init_clock_gating, \
4821 * intel_init_clock_gating_hooks - setup the clock gating hooks
4822 * @dev_priv: device private
4824 * Setup the hooks that configure which clocks of a given platform can be
4825 * gated and also apply various GT and display specific workarounds for these
4826 * platforms. Note that some GT specific workarounds are applied separately
4827 * when GPU contexts or batchbuffers start their execution.
4829 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
4831 if (IS_PONTEVECCHIO(dev_priv))
4832 dev_priv->clock_gating_funcs = &pvc_clock_gating_funcs;
4833 else if (IS_DG2(dev_priv))
4834 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
4835 else if (IS_XEHPSDV(dev_priv))
4836 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
4837 else if (IS_ALDERLAKE_P(dev_priv))
4838 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
4839 else if (IS_DG1(dev_priv))
4840 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
4841 else if (GRAPHICS_VER(dev_priv) == 12)
4842 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
4843 else if (GRAPHICS_VER(dev_priv) == 11)
4844 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
4845 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
4846 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
4847 else if (IS_SKYLAKE(dev_priv))
4848 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
4849 else if (IS_KABYLAKE(dev_priv))
4850 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
4851 else if (IS_BROXTON(dev_priv))
4852 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
4853 else if (IS_GEMINILAKE(dev_priv))
4854 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
4855 else if (IS_BROADWELL(dev_priv))
4856 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
4857 else if (IS_CHERRYVIEW(dev_priv))
4858 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
4859 else if (IS_HASWELL(dev_priv))
4860 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
4861 else if (IS_IVYBRIDGE(dev_priv))
4862 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
4863 else if (IS_VALLEYVIEW(dev_priv))
4864 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
4865 else if (GRAPHICS_VER(dev_priv) == 6)
4866 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
4867 else if (GRAPHICS_VER(dev_priv) == 5)
4868 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
4869 else if (IS_G4X(dev_priv))
4870 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
4871 else if (IS_I965GM(dev_priv))
4872 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
4873 else if (IS_I965G(dev_priv))
4874 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
4875 else if (GRAPHICS_VER(dev_priv) == 3)
4876 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
4877 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
4878 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
4879 else if (GRAPHICS_VER(dev_priv) == 2)
4880 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
4882 MISSING_CASE(INTEL_DEVID(dev_priv));
4883 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
4887 static const struct intel_wm_funcs ilk_wm_funcs = {
4888 .compute_pipe_wm = ilk_compute_pipe_wm,
4889 .compute_intermediate_wm = ilk_compute_intermediate_wm,
4890 .initial_watermarks = ilk_initial_watermarks,
4891 .optimize_watermarks = ilk_optimize_watermarks,
4894 static const struct intel_wm_funcs vlv_wm_funcs = {
4895 .compute_pipe_wm = vlv_compute_pipe_wm,
4896 .compute_intermediate_wm = vlv_compute_intermediate_wm,
4897 .initial_watermarks = vlv_initial_watermarks,
4898 .optimize_watermarks = vlv_optimize_watermarks,
4899 .atomic_update_watermarks = vlv_atomic_update_fifo,
4902 static const struct intel_wm_funcs g4x_wm_funcs = {
4903 .compute_pipe_wm = g4x_compute_pipe_wm,
4904 .compute_intermediate_wm = g4x_compute_intermediate_wm,
4905 .initial_watermarks = g4x_initial_watermarks,
4906 .optimize_watermarks = g4x_optimize_watermarks,
4909 static const struct intel_wm_funcs pnv_wm_funcs = {
4910 .update_wm = pnv_update_wm,
4913 static const struct intel_wm_funcs i965_wm_funcs = {
4914 .update_wm = i965_update_wm,
4917 static const struct intel_wm_funcs i9xx_wm_funcs = {
4918 .update_wm = i9xx_update_wm,
4921 static const struct intel_wm_funcs i845_wm_funcs = {
4922 .update_wm = i845_update_wm,
4925 static const struct intel_wm_funcs nop_funcs = {
4928 /* Set up chip specific power management-related functions */
4929 void intel_init_pm(struct drm_i915_private *dev_priv)
4931 if (DISPLAY_VER(dev_priv) >= 9) {
4932 skl_wm_init(dev_priv);
4937 if (IS_PINEVIEW(dev_priv))
4938 pnv_get_mem_freq(dev_priv);
4939 else if (GRAPHICS_VER(dev_priv) == 5)
4940 ilk_get_mem_freq(dev_priv);
4942 /* For FIFO watermark updates */
4943 if (HAS_PCH_SPLIT(dev_priv)) {
4944 ilk_setup_wm_latency(dev_priv);
4946 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
4947 dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
4948 (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
4949 dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
4950 dev_priv->display.funcs.wm = &ilk_wm_funcs;
4952 drm_dbg_kms(&dev_priv->drm,
4953 "Failed to read display plane latency. "
4955 dev_priv->display.funcs.wm = &nop_funcs;
4957 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4958 vlv_setup_wm_latency(dev_priv);
4959 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4960 } else if (IS_G4X(dev_priv)) {
4961 g4x_setup_wm_latency(dev_priv);
4962 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4963 } else if (IS_PINEVIEW(dev_priv)) {
4964 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
4967 dev_priv->mem_freq)) {
4968 drm_info(&dev_priv->drm,
4969 "failed to find known CxSR latency "
4970 "(found ddr%s fsb freq %d, mem freq %d), "
4972 (dev_priv->is_ddr3 == 1) ? "3" : "2",
4973 dev_priv->fsb_freq, dev_priv->mem_freq);
4974 /* Disable CxSR and never update its watermark again */
4975 intel_set_memory_cxsr(dev_priv, false);
4976 dev_priv->display.funcs.wm = &nop_funcs;
4978 dev_priv->display.funcs.wm = &pnv_wm_funcs;
4979 } else if (DISPLAY_VER(dev_priv) == 4) {
4980 dev_priv->display.funcs.wm = &i965_wm_funcs;
4981 } else if (DISPLAY_VER(dev_priv) == 3) {
4982 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4983 } else if (DISPLAY_VER(dev_priv) == 2) {
4984 if (INTEL_NUM_PIPES(dev_priv) == 1)
4985 dev_priv->display.funcs.wm = &i845_wm_funcs;
4987 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4989 drm_err(&dev_priv->drm,
4990 "unexpected fall-through in %s\n", __func__);
4991 dev_priv->display.funcs.wm = &nop_funcs;
4995 void intel_pm_setup(struct drm_i915_private *dev_priv)
4997 dev_priv->runtime_pm.suspended = false;
4998 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);