2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
28 #include <linux/module.h>
29 #include <linux/pm_runtime.h>
31 #include <drm/drm_atomic_helper.h>
32 #include <drm/drm_fourcc.h>
33 #include <drm/drm_plane_helper.h>
35 #include "display/intel_atomic.h"
36 #include "display/intel_atomic_plane.h"
37 #include "display/intel_bw.h"
38 #include "display/intel_de.h"
39 #include "display/intel_display_trace.h"
40 #include "display/intel_display_types.h"
41 #include "display/intel_fb.h"
42 #include "display/intel_fbc.h"
43 #include "display/intel_sprite.h"
44 #include "display/skl_universal_plane.h"
46 #include "gt/intel_engine_regs.h"
47 #include "gt/intel_gt_regs.h"
48 #include "gt/intel_llc.h"
51 #include "i915_fixed.h"
53 #include "intel_mchbar_regs.h"
54 #include "intel_pcode.h"
56 #include "vlv_sideband.h"
57 #include "../../../platform/x86/intel_ips.h"
59 struct drm_i915_clock_gating_funcs {
60 void (*init_clock_gating)(struct drm_i915_private *i915);
63 /* Stores plane specific WM parameters */
64 struct skl_wm_params {
65 bool x_tiled, y_tiled;
72 u32 plane_bytes_per_line;
73 uint_fixed_16_16_t plane_blocks_per_line;
74 uint_fixed_16_16_t y_tile_minimum;
79 /* used in computing the new watermarks state */
80 struct intel_wm_config {
81 unsigned int num_pipes_active;
86 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
88 if (HAS_LLC(dev_priv)) {
90 * WaCompressedResourceDisplayNewHashMode:skl,kbl
91 * Display WA #0390: skl,kbl
93 * Must match Sampler, Pixel Back End, and Media. See
94 * WaCompressedResourceSamplerPbeMediaNewHashMode.
96 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
97 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
98 SKL_DE_COMPRESSED_HASH_MODE);
101 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
102 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
103 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
105 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
106 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
107 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
110 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
111 * Display WA #0859: skl,bxt,kbl,glk,cfl
113 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
114 DISP_FBC_MEMORY_WAKE);
117 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
119 gen9_init_clock_gating(dev_priv);
121 /* WaDisableSDEUnitClockGating:bxt */
122 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
123 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
127 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
129 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
130 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
133 * Wa: Backlight PWM may stop in the asserted state, causing backlight
136 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
137 PWM1_GATING_DIS | PWM2_GATING_DIS);
140 * Lower the display internal timeout.
141 * This is needed to avoid any hard hangs when DSI port PLL
142 * is off and a MMIO access is attempted by any privilege
143 * application, using batch buffers or any other means.
145 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
148 * WaFbcTurnOffFbcWatermark:bxt
149 * Display WA #0562: bxt
151 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
155 * WaFbcHighMemBwCorruptionAvoidance:bxt
156 * Display WA #0883: bxt
158 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
159 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
160 DPFC_DISABLE_DUMMY0);
163 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
165 gen9_init_clock_gating(dev_priv);
168 * WaDisablePWMClockGating:glk
169 * Backlight PWM may stop in the asserted state, causing backlight
172 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
173 PWM1_GATING_DIS | PWM2_GATING_DIS);
176 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
180 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
182 switch (tmp & CLKCFG_FSB_MASK) {
184 dev_priv->fsb_freq = 533; /* 133*4 */
187 dev_priv->fsb_freq = 800; /* 200*4 */
190 dev_priv->fsb_freq = 667; /* 167*4 */
193 dev_priv->fsb_freq = 400; /* 100*4 */
197 switch (tmp & CLKCFG_MEM_MASK) {
199 dev_priv->mem_freq = 533;
202 dev_priv->mem_freq = 667;
205 dev_priv->mem_freq = 800;
209 /* detect pineview DDR3 setting */
210 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
211 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
214 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
218 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
219 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
221 switch (ddrpll & 0xff) {
223 dev_priv->mem_freq = 800;
226 dev_priv->mem_freq = 1066;
229 dev_priv->mem_freq = 1333;
232 dev_priv->mem_freq = 1600;
235 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
237 dev_priv->mem_freq = 0;
241 switch (csipll & 0x3ff) {
243 dev_priv->fsb_freq = 3200;
246 dev_priv->fsb_freq = 3733;
249 dev_priv->fsb_freq = 4266;
252 dev_priv->fsb_freq = 4800;
255 dev_priv->fsb_freq = 5333;
258 dev_priv->fsb_freq = 5866;
261 dev_priv->fsb_freq = 6400;
264 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
266 dev_priv->fsb_freq = 0;
271 static const struct cxsr_latency cxsr_latency_table[] = {
272 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
273 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
274 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
275 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
276 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
278 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
279 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
280 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
281 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
282 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
284 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
285 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
286 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
287 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
288 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
290 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
291 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
292 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
293 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
294 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
296 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
297 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
298 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
299 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
300 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
302 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
303 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
304 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
305 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
306 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
309 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
314 const struct cxsr_latency *latency;
317 if (fsb == 0 || mem == 0)
320 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
321 latency = &cxsr_latency_table[i];
322 if (is_desktop == latency->is_desktop &&
323 is_ddr3 == latency->is_ddr3 &&
324 fsb == latency->fsb_freq && mem == latency->mem_freq)
328 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
333 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
337 vlv_punit_get(dev_priv);
339 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
341 val &= ~FORCE_DDR_HIGH_FREQ;
343 val |= FORCE_DDR_HIGH_FREQ;
344 val &= ~FORCE_DDR_LOW_FREQ;
345 val |= FORCE_DDR_FREQ_REQ_ACK;
346 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
348 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
349 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
350 drm_err(&dev_priv->drm,
351 "timed out waiting for Punit DDR DVFS request\n");
353 vlv_punit_put(dev_priv);
356 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
360 vlv_punit_get(dev_priv);
362 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
364 val |= DSP_MAXFIFO_PM5_ENABLE;
366 val &= ~DSP_MAXFIFO_PM5_ENABLE;
367 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
369 vlv_punit_put(dev_priv);
372 #define FW_WM(value, plane) \
373 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
375 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
380 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
381 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
382 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
383 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
384 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
385 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
386 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
387 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
388 } else if (IS_PINEVIEW(dev_priv)) {
389 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
390 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
392 val |= PINEVIEW_SELF_REFRESH_EN;
394 val &= ~PINEVIEW_SELF_REFRESH_EN;
395 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
396 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
397 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
398 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
399 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
400 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
401 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
402 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
403 } else if (IS_I915GM(dev_priv)) {
405 * FIXME can't find a bit like this for 915G, and
406 * and yet it does have the related watermark in
407 * FW_BLC_SELF. What's going on?
409 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
410 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
411 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
412 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
413 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
418 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
420 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
421 enableddisabled(enable),
422 enableddisabled(was_enabled));
428 * intel_set_memory_cxsr - Configure CxSR state
429 * @dev_priv: i915 device
430 * @enable: Allow vs. disallow CxSR
432 * Allow or disallow the system to enter a special CxSR
433 * (C-state self refresh) state. What typically happens in CxSR mode
434 * is that several display FIFOs may get combined into a single larger
435 * FIFO for a particular plane (so called max FIFO mode) to allow the
436 * system to defer memory fetches longer, and the memory will enter
439 * Note that enabling CxSR does not guarantee that the system enter
440 * this special mode, nor does it guarantee that the system stays
441 * in that mode once entered. So this just allows/disallows the system
442 * to autonomously utilize the CxSR mode. Other factors such as core
443 * C-states will affect when/if the system actually enters/exits the
446 * Note that on VLV/CHV this actually only controls the max FIFO mode,
447 * and the system is free to enter/exit memory self refresh at any time
448 * even when the use of CxSR has been disallowed.
450 * While the system is actually in the CxSR/max FIFO mode, some plane
451 * control registers will not get latched on vblank. Thus in order to
452 * guarantee the system will respond to changes in the plane registers
453 * we must always disallow CxSR prior to making changes to those registers.
454 * Unfortunately the system will re-evaluate the CxSR conditions at
455 * frame start which happens after vblank start (which is when the plane
456 * registers would get latched), so we can't proceed with the plane update
457 * during the same frame where we disallowed CxSR.
459 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
460 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
461 * the hardware w.r.t. HPLL SR when writing to plane registers.
462 * Disallowing just CxSR is sufficient.
464 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
468 mutex_lock(&dev_priv->wm.wm_mutex);
469 ret = _intel_set_memory_cxsr(dev_priv, enable);
470 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
471 dev_priv->wm.vlv.cxsr = enable;
472 else if (IS_G4X(dev_priv))
473 dev_priv->wm.g4x.cxsr = enable;
474 mutex_unlock(&dev_priv->wm.wm_mutex);
480 * Latency for FIFO fetches is dependent on several factors:
481 * - memory configuration (speed, channels)
483 * - current MCH state
484 * It can be fairly high in some situations, so here we assume a fairly
485 * pessimal value. It's a tradeoff between extra memory fetches (if we
486 * set this value too high, the FIFO will fetch frequently to stay full)
487 * and power consumption (set it too low to save power and we might see
488 * FIFO underruns and display "flicker").
490 * A value of 5us seems to be a good balance; safe for very low end
491 * platforms but not overly aggressive on lower latency configs.
493 static const int pessimal_latency_ns = 5000;
495 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
496 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
498 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
500 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
501 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
502 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
503 enum pipe pipe = crtc->pipe;
504 int sprite0_start, sprite1_start;
505 u32 dsparb, dsparb2, dsparb3;
509 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
510 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
511 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
512 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
515 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
516 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
517 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
518 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
521 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
522 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
523 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
524 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
531 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
532 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
533 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
534 fifo_state->plane[PLANE_CURSOR] = 63;
537 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
538 enum i9xx_plane_id i9xx_plane)
540 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
543 size = dsparb & 0x7f;
544 if (i9xx_plane == PLANE_B)
545 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
547 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
548 dsparb, plane_name(i9xx_plane), size);
553 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
554 enum i9xx_plane_id i9xx_plane)
556 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
559 size = dsparb & 0x1ff;
560 if (i9xx_plane == PLANE_B)
561 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
562 size >>= 1; /* Convert to cachelines */
564 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
565 dsparb, plane_name(i9xx_plane), size);
570 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
571 enum i9xx_plane_id i9xx_plane)
573 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
576 size = dsparb & 0x7f;
577 size >>= 2; /* Convert to cachelines */
579 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
580 dsparb, plane_name(i9xx_plane), size);
585 /* Pineview has different values for various configs */
586 static const struct intel_watermark_params pnv_display_wm = {
587 .fifo_size = PINEVIEW_DISPLAY_FIFO,
588 .max_wm = PINEVIEW_MAX_WM,
589 .default_wm = PINEVIEW_DFT_WM,
590 .guard_size = PINEVIEW_GUARD_WM,
591 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
594 static const struct intel_watermark_params pnv_display_hplloff_wm = {
595 .fifo_size = PINEVIEW_DISPLAY_FIFO,
596 .max_wm = PINEVIEW_MAX_WM,
597 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
598 .guard_size = PINEVIEW_GUARD_WM,
599 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
602 static const struct intel_watermark_params pnv_cursor_wm = {
603 .fifo_size = PINEVIEW_CURSOR_FIFO,
604 .max_wm = PINEVIEW_CURSOR_MAX_WM,
605 .default_wm = PINEVIEW_CURSOR_DFT_WM,
606 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
607 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
610 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
611 .fifo_size = PINEVIEW_CURSOR_FIFO,
612 .max_wm = PINEVIEW_CURSOR_MAX_WM,
613 .default_wm = PINEVIEW_CURSOR_DFT_WM,
614 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
615 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
618 static const struct intel_watermark_params i965_cursor_wm_info = {
619 .fifo_size = I965_CURSOR_FIFO,
620 .max_wm = I965_CURSOR_MAX_WM,
621 .default_wm = I965_CURSOR_DFT_WM,
623 .cacheline_size = I915_FIFO_LINE_SIZE,
626 static const struct intel_watermark_params i945_wm_info = {
627 .fifo_size = I945_FIFO_SIZE,
628 .max_wm = I915_MAX_WM,
631 .cacheline_size = I915_FIFO_LINE_SIZE,
634 static const struct intel_watermark_params i915_wm_info = {
635 .fifo_size = I915_FIFO_SIZE,
636 .max_wm = I915_MAX_WM,
639 .cacheline_size = I915_FIFO_LINE_SIZE,
642 static const struct intel_watermark_params i830_a_wm_info = {
643 .fifo_size = I855GM_FIFO_SIZE,
644 .max_wm = I915_MAX_WM,
647 .cacheline_size = I830_FIFO_LINE_SIZE,
650 static const struct intel_watermark_params i830_bc_wm_info = {
651 .fifo_size = I855GM_FIFO_SIZE,
652 .max_wm = I915_MAX_WM/2,
655 .cacheline_size = I830_FIFO_LINE_SIZE,
658 static const struct intel_watermark_params i845_wm_info = {
659 .fifo_size = I830_FIFO_SIZE,
660 .max_wm = I915_MAX_WM,
663 .cacheline_size = I830_FIFO_LINE_SIZE,
667 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
668 * @pixel_rate: Pipe pixel rate in kHz
669 * @cpp: Plane bytes per pixel
670 * @latency: Memory wakeup latency in 0.1us units
672 * Compute the watermark using the method 1 or "small buffer"
673 * formula. The caller may additonally add extra cachelines
674 * to account for TLB misses and clock crossings.
676 * This method is concerned with the short term drain rate
677 * of the FIFO, ie. it does not account for blanking periods
678 * which would effectively reduce the average drain rate across
679 * a longer period. The name "small" refers to the fact the
680 * FIFO is relatively small compared to the amount of data
683 * The FIFO level vs. time graph might look something like:
687 * __---__---__ (- plane active, _ blanking)
690 * or perhaps like this:
693 * __----__----__ (- plane active, _ blanking)
697 * The watermark in bytes
699 static unsigned int intel_wm_method1(unsigned int pixel_rate,
701 unsigned int latency)
705 ret = mul_u32_u32(pixel_rate, cpp * latency);
706 ret = DIV_ROUND_UP_ULL(ret, 10000);
712 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
713 * @pixel_rate: Pipe pixel rate in kHz
714 * @htotal: Pipe horizontal total
715 * @width: Plane width in pixels
716 * @cpp: Plane bytes per pixel
717 * @latency: Memory wakeup latency in 0.1us units
719 * Compute the watermark using the method 2 or "large buffer"
720 * formula. The caller may additonally add extra cachelines
721 * to account for TLB misses and clock crossings.
723 * This method is concerned with the long term drain rate
724 * of the FIFO, ie. it does account for blanking periods
725 * which effectively reduce the average drain rate across
726 * a longer period. The name "large" refers to the fact the
727 * FIFO is relatively large compared to the amount of data
730 * The FIFO level vs. time graph might look something like:
735 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
739 * The watermark in bytes
741 static unsigned int intel_wm_method2(unsigned int pixel_rate,
745 unsigned int latency)
750 * FIXME remove once all users are computing
751 * watermarks in the correct place.
753 if (WARN_ON_ONCE(htotal == 0))
756 ret = (latency * pixel_rate) / (htotal * 10000);
757 ret = (ret + 1) * width * cpp;
763 * intel_calculate_wm - calculate watermark level
764 * @pixel_rate: pixel clock
765 * @wm: chip FIFO params
766 * @fifo_size: size of the FIFO buffer
767 * @cpp: bytes per pixel
768 * @latency_ns: memory latency for the platform
770 * Calculate the watermark level (the level at which the display plane will
771 * start fetching from memory again). Each chip has a different display
772 * FIFO size and allocation, so the caller needs to figure that out and pass
773 * in the correct intel_watermark_params structure.
775 * As the pixel clock runs, the FIFO will be drained at a rate that depends
776 * on the pixel size. When it reaches the watermark level, it'll start
777 * fetching FIFO line sized based chunks from memory until the FIFO fills
778 * past the watermark point. If the FIFO drains completely, a FIFO underrun
779 * will occur, and a display engine hang could result.
781 static unsigned int intel_calculate_wm(int pixel_rate,
782 const struct intel_watermark_params *wm,
783 int fifo_size, int cpp,
784 unsigned int latency_ns)
786 int entries, wm_size;
789 * Note: we need to make sure we don't overflow for various clock &
791 * clocks go from a few thousand to several hundred thousand.
792 * latency is usually a few thousand
794 entries = intel_wm_method1(pixel_rate, cpp,
796 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
798 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
800 wm_size = fifo_size - entries;
801 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
803 /* Don't promote wm_size to unsigned... */
804 if (wm_size > wm->max_wm)
805 wm_size = wm->max_wm;
807 wm_size = wm->default_wm;
810 * Bspec seems to indicate that the value shouldn't be lower than
811 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
812 * Lets go for 8 which is the burst size since certain platforms
813 * already use a hardcoded 8 (which is what the spec says should be
822 static bool is_disabling(int old, int new, int threshold)
824 return old >= threshold && new < threshold;
827 static bool is_enabling(int old, int new, int threshold)
829 return old < threshold && new >= threshold;
832 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
834 return dev_priv->wm.max_level + 1;
837 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
838 const struct intel_plane_state *plane_state)
840 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
842 /* FIXME check the 'enable' instead */
843 if (!crtc_state->hw.active)
847 * Treat cursor with fb as always visible since cursor updates
848 * can happen faster than the vrefresh rate, and the current
849 * watermark code doesn't handle that correctly. Cursor updates
850 * which set/clear the fb or change the cursor size are going
851 * to get throttled by intel_legacy_cursor_update() to work
852 * around this problem with the watermark code.
854 if (plane->id == PLANE_CURSOR)
855 return plane_state->hw.fb != NULL;
857 return plane_state->uapi.visible;
860 static bool intel_crtc_active(struct intel_crtc *crtc)
862 /* Be paranoid as we can arrive here with only partial
863 * state retrieved from the hardware during setup.
865 * We can ditch the adjusted_mode.crtc_clock check as soon
866 * as Haswell has gained clock readout/fastboot support.
868 * We can ditch the crtc->primary->state->fb check as soon as we can
869 * properly reconstruct framebuffers.
871 * FIXME: The intel_crtc->active here should be switched to
872 * crtc->state->active once we have proper CRTC states wired up
875 return crtc && crtc->active && crtc->base.primary->state->fb &&
876 crtc->config->hw.adjusted_mode.crtc_clock;
879 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
881 struct intel_crtc *crtc, *enabled = NULL;
883 for_each_intel_crtc(&dev_priv->drm, crtc) {
884 if (intel_crtc_active(crtc)) {
894 static void pnv_update_wm(struct drm_i915_private *dev_priv)
896 struct intel_crtc *crtc;
897 const struct cxsr_latency *latency;
901 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
906 drm_dbg_kms(&dev_priv->drm,
907 "Unknown FSB/MEM found, disable CxSR\n");
908 intel_set_memory_cxsr(dev_priv, false);
912 crtc = single_enabled_crtc(dev_priv);
914 const struct drm_framebuffer *fb =
915 crtc->base.primary->state->fb;
916 int pixel_rate = crtc->config->pixel_rate;
917 int cpp = fb->format->cpp[0];
920 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
921 pnv_display_wm.fifo_size,
922 cpp, latency->display_sr);
923 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
924 reg &= ~DSPFW_SR_MASK;
925 reg |= FW_WM(wm, SR);
926 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
927 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
930 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
931 pnv_display_wm.fifo_size,
932 4, latency->cursor_sr);
933 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
934 reg &= ~DSPFW_CURSOR_SR_MASK;
935 reg |= FW_WM(wm, CURSOR_SR);
936 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
938 /* Display HPLL off SR */
939 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
940 pnv_display_hplloff_wm.fifo_size,
941 cpp, latency->display_hpll_disable);
942 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
943 reg &= ~DSPFW_HPLL_SR_MASK;
944 reg |= FW_WM(wm, HPLL_SR);
945 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
947 /* cursor HPLL off SR */
948 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
949 pnv_display_hplloff_wm.fifo_size,
950 4, latency->cursor_hpll_disable);
951 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
952 reg &= ~DSPFW_HPLL_CURSOR_MASK;
953 reg |= FW_WM(wm, HPLL_CURSOR);
954 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
955 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
957 intel_set_memory_cxsr(dev_priv, true);
959 intel_set_memory_cxsr(dev_priv, false);
964 * Documentation says:
965 * "If the line size is small, the TLB fetches can get in the way of the
966 * data fetches, causing some lag in the pixel data return which is not
967 * accounted for in the above formulas. The following adjustment only
968 * needs to be applied if eight whole lines fit in the buffer at once.
969 * The WM is adjusted upwards by the difference between the FIFO size
970 * and the size of 8 whole lines. This adjustment is always performed
971 * in the actual pixel depth regardless of whether FBC is enabled or not."
973 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
975 int tlb_miss = fifo_size * 64 - width * cpp * 8;
977 return max(0, tlb_miss);
980 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
981 const struct g4x_wm_values *wm)
985 for_each_pipe(dev_priv, pipe)
986 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
988 intel_uncore_write(&dev_priv->uncore, DSPFW1,
989 FW_WM(wm->sr.plane, SR) |
990 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
991 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
992 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
993 intel_uncore_write(&dev_priv->uncore, DSPFW2,
994 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
995 FW_WM(wm->sr.fbc, FBC_SR) |
996 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
997 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
998 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
999 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1000 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1001 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
1002 FW_WM(wm->sr.cursor, CURSOR_SR) |
1003 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1004 FW_WM(wm->hpll.plane, HPLL_SR));
1006 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1009 #define FW_WM_VLV(value, plane) \
1010 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1012 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1013 const struct vlv_wm_values *wm)
1017 for_each_pipe(dev_priv, pipe) {
1018 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
1020 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1021 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1022 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1023 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1024 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1028 * Zero the (unused) WM1 watermarks, and also clear all the
1029 * high order bits so that there are no out of bounds values
1030 * present in the registers during the reprogramming.
1032 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1033 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1034 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1035 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1036 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1038 intel_uncore_write(&dev_priv->uncore, DSPFW1,
1039 FW_WM(wm->sr.plane, SR) |
1040 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1041 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1042 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1043 intel_uncore_write(&dev_priv->uncore, DSPFW2,
1044 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1045 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1046 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1047 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1048 FW_WM(wm->sr.cursor, CURSOR_SR));
1050 if (IS_CHERRYVIEW(dev_priv)) {
1051 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1052 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1053 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1054 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1055 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1056 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1057 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1058 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1059 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1060 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1061 FW_WM(wm->sr.plane >> 9, SR_HI) |
1062 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1063 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1064 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1065 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1066 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1067 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1068 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1069 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1070 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1072 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1073 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1074 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1075 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1076 FW_WM(wm->sr.plane >> 9, SR_HI) |
1077 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1078 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1079 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1080 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1081 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1082 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1085 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1090 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1092 /* all latencies in usec */
1093 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1094 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1095 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1097 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1100 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1103 * DSPCNTR[13] supposedly controls whether the
1104 * primary plane can use the FIFO space otherwise
1105 * reserved for the sprite plane. It's not 100% clear
1106 * what the actual FIFO size is, but it looks like we
1107 * can happily set both primary and sprite watermarks
1108 * up to 127 cachelines. So that would seem to mean
1109 * that either DSPCNTR[13] doesn't do anything, or that
1110 * the total FIFO is >= 256 cachelines in size. Either
1111 * way, we don't seem to have to worry about this
1112 * repartitioning as the maximum watermark value the
1113 * register can hold for each plane is lower than the
1114 * minimum FIFO size.
1120 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1122 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1124 MISSING_CASE(plane_id);
1129 static int g4x_fbc_fifo_size(int level)
1132 case G4X_WM_LEVEL_SR:
1134 case G4X_WM_LEVEL_HPLL:
1137 MISSING_CASE(level);
1142 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1143 const struct intel_plane_state *plane_state,
1146 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1147 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1148 const struct drm_display_mode *pipe_mode =
1149 &crtc_state->hw.pipe_mode;
1150 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1151 unsigned int pixel_rate, htotal, cpp, width, wm;
1156 if (!intel_wm_plane_visible(crtc_state, plane_state))
1159 cpp = plane_state->hw.fb->format->cpp[0];
1162 * WaUse32BppForSRWM:ctg,elk
1164 * The spec fails to list this restriction for the
1165 * HPLL watermark, which seems a little strange.
1166 * Let's use 32bpp for the HPLL watermark as well.
1168 if (plane->id == PLANE_PRIMARY &&
1169 level != G4X_WM_LEVEL_NORMAL)
1172 pixel_rate = crtc_state->pixel_rate;
1173 htotal = pipe_mode->crtc_htotal;
1174 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1176 if (plane->id == PLANE_CURSOR) {
1177 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1178 } else if (plane->id == PLANE_PRIMARY &&
1179 level == G4X_WM_LEVEL_NORMAL) {
1180 wm = intel_wm_method1(pixel_rate, cpp, latency);
1182 unsigned int small, large;
1184 small = intel_wm_method1(pixel_rate, cpp, latency);
1185 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1187 wm = min(small, large);
1190 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1193 wm = DIV_ROUND_UP(wm, 64) + 2;
1195 return min_t(unsigned int, wm, USHRT_MAX);
1198 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1199 int level, enum plane_id plane_id, u16 value)
1201 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1204 for (; level < intel_wm_num_levels(dev_priv); level++) {
1205 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1207 dirty |= raw->plane[plane_id] != value;
1208 raw->plane[plane_id] = value;
1214 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1215 int level, u16 value)
1217 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1220 /* NORMAL level doesn't have an FBC watermark */
1221 level = max(level, G4X_WM_LEVEL_SR);
1223 for (; level < intel_wm_num_levels(dev_priv); level++) {
1224 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1226 dirty |= raw->fbc != value;
1233 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1234 const struct intel_plane_state *plane_state,
1237 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1238 const struct intel_plane_state *plane_state)
1240 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1241 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1242 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1243 enum plane_id plane_id = plane->id;
1247 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1248 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1249 if (plane_id == PLANE_PRIMARY)
1250 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1254 for (level = 0; level < num_levels; level++) {
1255 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1258 wm = g4x_compute_wm(crtc_state, plane_state, level);
1259 max_wm = g4x_plane_fifo_size(plane_id, level);
1264 dirty |= raw->plane[plane_id] != wm;
1265 raw->plane[plane_id] = wm;
1267 if (plane_id != PLANE_PRIMARY ||
1268 level == G4X_WM_LEVEL_NORMAL)
1271 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1272 raw->plane[plane_id]);
1273 max_wm = g4x_fbc_fifo_size(level);
1276 * FBC wm is not mandatory as we
1277 * can always just disable its use.
1282 dirty |= raw->fbc != wm;
1286 /* mark watermarks as invalid */
1287 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1289 if (plane_id == PLANE_PRIMARY)
1290 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1294 drm_dbg_kms(&dev_priv->drm,
1295 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1297 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1298 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1299 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1301 if (plane_id == PLANE_PRIMARY)
1302 drm_dbg_kms(&dev_priv->drm,
1303 "FBC watermarks: SR=%d, HPLL=%d\n",
1304 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1305 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1311 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1312 enum plane_id plane_id, int level)
1314 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1316 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1319 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1322 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1324 if (level > dev_priv->wm.max_level)
1327 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1328 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1329 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1332 /* mark all levels starting from 'level' as invalid */
1333 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1334 struct g4x_wm_state *wm_state, int level)
1336 if (level <= G4X_WM_LEVEL_NORMAL) {
1337 enum plane_id plane_id;
1339 for_each_plane_id_on_crtc(crtc, plane_id)
1340 wm_state->wm.plane[plane_id] = USHRT_MAX;
1343 if (level <= G4X_WM_LEVEL_SR) {
1344 wm_state->cxsr = false;
1345 wm_state->sr.cursor = USHRT_MAX;
1346 wm_state->sr.plane = USHRT_MAX;
1347 wm_state->sr.fbc = USHRT_MAX;
1350 if (level <= G4X_WM_LEVEL_HPLL) {
1351 wm_state->hpll_en = false;
1352 wm_state->hpll.cursor = USHRT_MAX;
1353 wm_state->hpll.plane = USHRT_MAX;
1354 wm_state->hpll.fbc = USHRT_MAX;
1358 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1361 if (level < G4X_WM_LEVEL_SR)
1364 if (level >= G4X_WM_LEVEL_SR &&
1365 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1368 if (level >= G4X_WM_LEVEL_HPLL &&
1369 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1375 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1376 struct intel_crtc *crtc)
1378 struct intel_crtc_state *crtc_state =
1379 intel_atomic_get_new_crtc_state(state, crtc);
1380 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1381 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1382 const struct g4x_pipe_wm *raw;
1383 const struct intel_plane_state *old_plane_state;
1384 const struct intel_plane_state *new_plane_state;
1385 struct intel_plane *plane;
1386 enum plane_id plane_id;
1388 unsigned int dirty = 0;
1390 for_each_oldnew_intel_plane_in_state(state, plane,
1392 new_plane_state, i) {
1393 if (new_plane_state->hw.crtc != &crtc->base &&
1394 old_plane_state->hw.crtc != &crtc->base)
1397 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1398 dirty |= BIT(plane->id);
1404 level = G4X_WM_LEVEL_NORMAL;
1405 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1408 raw = &crtc_state->wm.g4x.raw[level];
1409 for_each_plane_id_on_crtc(crtc, plane_id)
1410 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1412 level = G4X_WM_LEVEL_SR;
1413 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1416 raw = &crtc_state->wm.g4x.raw[level];
1417 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1418 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1419 wm_state->sr.fbc = raw->fbc;
1421 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1423 level = G4X_WM_LEVEL_HPLL;
1424 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1427 raw = &crtc_state->wm.g4x.raw[level];
1428 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1429 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1430 wm_state->hpll.fbc = raw->fbc;
1432 wm_state->hpll_en = wm_state->cxsr;
1437 if (level == G4X_WM_LEVEL_NORMAL)
1440 /* invalidate the higher levels */
1441 g4x_invalidate_wms(crtc, wm_state, level);
1444 * Determine if the FBC watermark(s) can be used. IF
1445 * this isn't the case we prefer to disable the FBC
1446 * watermark(s) rather than disable the SR/HPLL
1447 * level(s) entirely. 'level-1' is the highest valid
1450 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1455 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1456 struct intel_crtc *crtc)
1458 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1459 struct intel_crtc_state *new_crtc_state =
1460 intel_atomic_get_new_crtc_state(state, crtc);
1461 const struct intel_crtc_state *old_crtc_state =
1462 intel_atomic_get_old_crtc_state(state, crtc);
1463 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1464 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1465 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1466 enum plane_id plane_id;
1468 if (!new_crtc_state->hw.active ||
1469 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1470 *intermediate = *optimal;
1472 intermediate->cxsr = false;
1473 intermediate->hpll_en = false;
1477 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1478 !new_crtc_state->disable_cxsr;
1479 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1480 !new_crtc_state->disable_cxsr;
1481 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1483 for_each_plane_id_on_crtc(crtc, plane_id) {
1484 intermediate->wm.plane[plane_id] =
1485 max(optimal->wm.plane[plane_id],
1486 active->wm.plane[plane_id]);
1488 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1489 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1492 intermediate->sr.plane = max(optimal->sr.plane,
1494 intermediate->sr.cursor = max(optimal->sr.cursor,
1496 intermediate->sr.fbc = max(optimal->sr.fbc,
1499 intermediate->hpll.plane = max(optimal->hpll.plane,
1500 active->hpll.plane);
1501 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1502 active->hpll.cursor);
1503 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1506 drm_WARN_ON(&dev_priv->drm,
1507 (intermediate->sr.plane >
1508 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1509 intermediate->sr.cursor >
1510 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1511 intermediate->cxsr);
1512 drm_WARN_ON(&dev_priv->drm,
1513 (intermediate->sr.plane >
1514 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1515 intermediate->sr.cursor >
1516 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1517 intermediate->hpll_en);
1519 drm_WARN_ON(&dev_priv->drm,
1520 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1521 intermediate->fbc_en && intermediate->cxsr);
1522 drm_WARN_ON(&dev_priv->drm,
1523 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1524 intermediate->fbc_en && intermediate->hpll_en);
1528 * If our intermediate WM are identical to the final WM, then we can
1529 * omit the post-vblank programming; only update if it's different.
1531 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1532 new_crtc_state->wm.need_postvbl_update = true;
1537 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1538 struct g4x_wm_values *wm)
1540 struct intel_crtc *crtc;
1541 int num_active_pipes = 0;
1547 for_each_intel_crtc(&dev_priv->drm, crtc) {
1548 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1553 if (!wm_state->cxsr)
1555 if (!wm_state->hpll_en)
1556 wm->hpll_en = false;
1557 if (!wm_state->fbc_en)
1563 if (num_active_pipes != 1) {
1565 wm->hpll_en = false;
1569 for_each_intel_crtc(&dev_priv->drm, crtc) {
1570 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1571 enum pipe pipe = crtc->pipe;
1573 wm->pipe[pipe] = wm_state->wm;
1574 if (crtc->active && wm->cxsr)
1575 wm->sr = wm_state->sr;
1576 if (crtc->active && wm->hpll_en)
1577 wm->hpll = wm_state->hpll;
1581 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1583 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1584 struct g4x_wm_values new_wm = {};
1586 g4x_merge_wm(dev_priv, &new_wm);
1588 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1591 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1592 _intel_set_memory_cxsr(dev_priv, false);
1594 g4x_write_wm_values(dev_priv, &new_wm);
1596 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1597 _intel_set_memory_cxsr(dev_priv, true);
1602 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1603 struct intel_crtc *crtc)
1605 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1606 const struct intel_crtc_state *crtc_state =
1607 intel_atomic_get_new_crtc_state(state, crtc);
1609 mutex_lock(&dev_priv->wm.wm_mutex);
1610 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1611 g4x_program_watermarks(dev_priv);
1612 mutex_unlock(&dev_priv->wm.wm_mutex);
1615 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1616 struct intel_crtc *crtc)
1618 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1619 const struct intel_crtc_state *crtc_state =
1620 intel_atomic_get_new_crtc_state(state, crtc);
1622 if (!crtc_state->wm.need_postvbl_update)
1625 mutex_lock(&dev_priv->wm.wm_mutex);
1626 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1627 g4x_program_watermarks(dev_priv);
1628 mutex_unlock(&dev_priv->wm.wm_mutex);
1631 /* latency must be in 0.1us units. */
1632 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1633 unsigned int htotal,
1636 unsigned int latency)
1640 ret = intel_wm_method2(pixel_rate, htotal,
1641 width, cpp, latency);
1642 ret = DIV_ROUND_UP(ret, 64);
1647 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1649 /* all latencies in usec */
1650 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1652 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1654 if (IS_CHERRYVIEW(dev_priv)) {
1655 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1656 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1658 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1662 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1663 const struct intel_plane_state *plane_state,
1666 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1667 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1668 const struct drm_display_mode *pipe_mode =
1669 &crtc_state->hw.pipe_mode;
1670 unsigned int pixel_rate, htotal, cpp, width, wm;
1672 if (dev_priv->wm.pri_latency[level] == 0)
1675 if (!intel_wm_plane_visible(crtc_state, plane_state))
1678 cpp = plane_state->hw.fb->format->cpp[0];
1679 pixel_rate = crtc_state->pixel_rate;
1680 htotal = pipe_mode->crtc_htotal;
1681 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1683 if (plane->id == PLANE_CURSOR) {
1685 * FIXME the formula gives values that are
1686 * too big for the cursor FIFO, and hence we
1687 * would never be able to use cursors. For
1688 * now just hardcode the watermark.
1692 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1693 dev_priv->wm.pri_latency[level] * 10);
1696 return min_t(unsigned int, wm, USHRT_MAX);
1699 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1701 return (active_planes & (BIT(PLANE_SPRITE0) |
1702 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1705 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1707 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1708 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1709 const struct g4x_pipe_wm *raw =
1710 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1711 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1712 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1713 int num_active_planes = hweight8(active_planes);
1714 const int fifo_size = 511;
1715 int fifo_extra, fifo_left = fifo_size;
1716 int sprite0_fifo_extra = 0;
1717 unsigned int total_rate;
1718 enum plane_id plane_id;
1721 * When enabling sprite0 after sprite1 has already been enabled
1722 * we tend to get an underrun unless sprite0 already has some
1723 * FIFO space allcoated. Hence we always allocate at least one
1724 * cacheline for sprite0 whenever sprite1 is enabled.
1726 * All other plane enable sequences appear immune to this problem.
1728 if (vlv_need_sprite0_fifo_workaround(active_planes))
1729 sprite0_fifo_extra = 1;
1731 total_rate = raw->plane[PLANE_PRIMARY] +
1732 raw->plane[PLANE_SPRITE0] +
1733 raw->plane[PLANE_SPRITE1] +
1736 if (total_rate > fifo_size)
1739 if (total_rate == 0)
1742 for_each_plane_id_on_crtc(crtc, plane_id) {
1745 if ((active_planes & BIT(plane_id)) == 0) {
1746 fifo_state->plane[plane_id] = 0;
1750 rate = raw->plane[plane_id];
1751 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1752 fifo_left -= fifo_state->plane[plane_id];
1755 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1756 fifo_left -= sprite0_fifo_extra;
1758 fifo_state->plane[PLANE_CURSOR] = 63;
1760 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1762 /* spread the remainder evenly */
1763 for_each_plane_id_on_crtc(crtc, plane_id) {
1769 if ((active_planes & BIT(plane_id)) == 0)
1772 plane_extra = min(fifo_extra, fifo_left);
1773 fifo_state->plane[plane_id] += plane_extra;
1774 fifo_left -= plane_extra;
1777 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1779 /* give it all to the first plane if none are active */
1780 if (active_planes == 0) {
1781 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1782 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1788 /* mark all levels starting from 'level' as invalid */
1789 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1790 struct vlv_wm_state *wm_state, int level)
1792 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1794 for (; level < intel_wm_num_levels(dev_priv); level++) {
1795 enum plane_id plane_id;
1797 for_each_plane_id_on_crtc(crtc, plane_id)
1798 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1800 wm_state->sr[level].cursor = USHRT_MAX;
1801 wm_state->sr[level].plane = USHRT_MAX;
1805 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1810 return fifo_size - wm;
1814 * Starting from 'level' set all higher
1815 * levels to 'value' in the "raw" watermarks.
1817 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1818 int level, enum plane_id plane_id, u16 value)
1820 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1821 int num_levels = intel_wm_num_levels(dev_priv);
1824 for (; level < num_levels; level++) {
1825 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1827 dirty |= raw->plane[plane_id] != value;
1828 raw->plane[plane_id] = value;
1834 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1835 const struct intel_plane_state *plane_state)
1837 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1838 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1839 enum plane_id plane_id = plane->id;
1840 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1844 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1845 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1849 for (level = 0; level < num_levels; level++) {
1850 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1851 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1852 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1857 dirty |= raw->plane[plane_id] != wm;
1858 raw->plane[plane_id] = wm;
1861 /* mark all higher levels as invalid */
1862 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1866 drm_dbg_kms(&dev_priv->drm,
1867 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1869 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1870 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1871 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1876 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1877 enum plane_id plane_id, int level)
1879 const struct g4x_pipe_wm *raw =
1880 &crtc_state->wm.vlv.raw[level];
1881 const struct vlv_fifo_state *fifo_state =
1882 &crtc_state->wm.vlv.fifo_state;
1884 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1887 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1889 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1890 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1891 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1892 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1895 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1896 struct intel_crtc *crtc)
1898 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1899 struct intel_crtc_state *crtc_state =
1900 intel_atomic_get_new_crtc_state(state, crtc);
1901 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1902 const struct vlv_fifo_state *fifo_state =
1903 &crtc_state->wm.vlv.fifo_state;
1904 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1905 int num_active_planes = hweight8(active_planes);
1906 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1907 const struct intel_plane_state *old_plane_state;
1908 const struct intel_plane_state *new_plane_state;
1909 struct intel_plane *plane;
1910 enum plane_id plane_id;
1912 unsigned int dirty = 0;
1914 for_each_oldnew_intel_plane_in_state(state, plane,
1916 new_plane_state, i) {
1917 if (new_plane_state->hw.crtc != &crtc->base &&
1918 old_plane_state->hw.crtc != &crtc->base)
1921 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1922 dirty |= BIT(plane->id);
1926 * DSPARB registers may have been reset due to the
1927 * power well being turned off. Make sure we restore
1928 * them to a consistent state even if no primary/sprite
1929 * planes are initially active.
1932 crtc_state->fifo_changed = true;
1937 /* cursor changes don't warrant a FIFO recompute */
1938 if (dirty & ~BIT(PLANE_CURSOR)) {
1939 const struct intel_crtc_state *old_crtc_state =
1940 intel_atomic_get_old_crtc_state(state, crtc);
1941 const struct vlv_fifo_state *old_fifo_state =
1942 &old_crtc_state->wm.vlv.fifo_state;
1944 ret = vlv_compute_fifo(crtc_state);
1948 if (needs_modeset ||
1949 memcmp(old_fifo_state, fifo_state,
1950 sizeof(*fifo_state)) != 0)
1951 crtc_state->fifo_changed = true;
1954 /* initially allow all levels */
1955 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1957 * Note that enabling cxsr with no primary/sprite planes
1958 * enabled can wedge the pipe. Hence we only allow cxsr
1959 * with exactly one enabled primary/sprite plane.
1961 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1963 for (level = 0; level < wm_state->num_levels; level++) {
1964 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1965 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1967 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1970 for_each_plane_id_on_crtc(crtc, plane_id) {
1971 wm_state->wm[level].plane[plane_id] =
1972 vlv_invert_wm_value(raw->plane[plane_id],
1973 fifo_state->plane[plane_id]);
1976 wm_state->sr[level].plane =
1977 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1978 raw->plane[PLANE_SPRITE0],
1979 raw->plane[PLANE_SPRITE1]),
1982 wm_state->sr[level].cursor =
1983 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1990 /* limit to only levels we can actually handle */
1991 wm_state->num_levels = level;
1993 /* invalidate the higher levels */
1994 vlv_invalidate_wms(crtc, wm_state, level);
1999 #define VLV_FIFO(plane, value) \
2000 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2002 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2003 struct intel_crtc *crtc)
2005 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2006 struct intel_uncore *uncore = &dev_priv->uncore;
2007 const struct intel_crtc_state *crtc_state =
2008 intel_atomic_get_new_crtc_state(state, crtc);
2009 const struct vlv_fifo_state *fifo_state =
2010 &crtc_state->wm.vlv.fifo_state;
2011 int sprite0_start, sprite1_start, fifo_size;
2012 u32 dsparb, dsparb2, dsparb3;
2014 if (!crtc_state->fifo_changed)
2017 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2018 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2019 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2021 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2022 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2024 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2027 * uncore.lock serves a double purpose here. It allows us to
2028 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2029 * it protects the DSPARB registers from getting clobbered by
2030 * parallel updates from multiple pipes.
2032 * intel_pipe_update_start() has already disabled interrupts
2033 * for us, so a plain spin_lock() is sufficient here.
2035 spin_lock(&uncore->lock);
2037 switch (crtc->pipe) {
2039 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2040 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2042 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2043 VLV_FIFO(SPRITEB, 0xff));
2044 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2045 VLV_FIFO(SPRITEB, sprite1_start));
2047 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2048 VLV_FIFO(SPRITEB_HI, 0x1));
2049 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2050 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2052 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2053 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2056 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2057 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2059 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2060 VLV_FIFO(SPRITED, 0xff));
2061 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2062 VLV_FIFO(SPRITED, sprite1_start));
2064 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2065 VLV_FIFO(SPRITED_HI, 0xff));
2066 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2067 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2069 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2070 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2073 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2074 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2076 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2077 VLV_FIFO(SPRITEF, 0xff));
2078 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2079 VLV_FIFO(SPRITEF, sprite1_start));
2081 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2082 VLV_FIFO(SPRITEF_HI, 0xff));
2083 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2084 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2086 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2087 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2093 intel_uncore_posting_read_fw(uncore, DSPARB);
2095 spin_unlock(&uncore->lock);
2100 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2101 struct intel_crtc *crtc)
2103 struct intel_crtc_state *new_crtc_state =
2104 intel_atomic_get_new_crtc_state(state, crtc);
2105 const struct intel_crtc_state *old_crtc_state =
2106 intel_atomic_get_old_crtc_state(state, crtc);
2107 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2108 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2109 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2112 if (!new_crtc_state->hw.active ||
2113 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2114 *intermediate = *optimal;
2116 intermediate->cxsr = false;
2120 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2121 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2122 !new_crtc_state->disable_cxsr;
2124 for (level = 0; level < intermediate->num_levels; level++) {
2125 enum plane_id plane_id;
2127 for_each_plane_id_on_crtc(crtc, plane_id) {
2128 intermediate->wm[level].plane[plane_id] =
2129 min(optimal->wm[level].plane[plane_id],
2130 active->wm[level].plane[plane_id]);
2133 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2134 active->sr[level].plane);
2135 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2136 active->sr[level].cursor);
2139 vlv_invalidate_wms(crtc, intermediate, level);
2143 * If our intermediate WM are identical to the final WM, then we can
2144 * omit the post-vblank programming; only update if it's different.
2146 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2147 new_crtc_state->wm.need_postvbl_update = true;
2152 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2153 struct vlv_wm_values *wm)
2155 struct intel_crtc *crtc;
2156 int num_active_pipes = 0;
2158 wm->level = dev_priv->wm.max_level;
2161 for_each_intel_crtc(&dev_priv->drm, crtc) {
2162 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2167 if (!wm_state->cxsr)
2171 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2174 if (num_active_pipes != 1)
2177 if (num_active_pipes > 1)
2178 wm->level = VLV_WM_LEVEL_PM2;
2180 for_each_intel_crtc(&dev_priv->drm, crtc) {
2181 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2182 enum pipe pipe = crtc->pipe;
2184 wm->pipe[pipe] = wm_state->wm[wm->level];
2185 if (crtc->active && wm->cxsr)
2186 wm->sr = wm_state->sr[wm->level];
2188 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2189 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2190 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2191 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2195 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2197 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2198 struct vlv_wm_values new_wm = {};
2200 vlv_merge_wm(dev_priv, &new_wm);
2202 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2205 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2206 chv_set_memory_dvfs(dev_priv, false);
2208 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2209 chv_set_memory_pm5(dev_priv, false);
2211 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2212 _intel_set_memory_cxsr(dev_priv, false);
2214 vlv_write_wm_values(dev_priv, &new_wm);
2216 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2217 _intel_set_memory_cxsr(dev_priv, true);
2219 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2220 chv_set_memory_pm5(dev_priv, true);
2222 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2223 chv_set_memory_dvfs(dev_priv, true);
2228 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2229 struct intel_crtc *crtc)
2231 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2232 const struct intel_crtc_state *crtc_state =
2233 intel_atomic_get_new_crtc_state(state, crtc);
2235 mutex_lock(&dev_priv->wm.wm_mutex);
2236 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2237 vlv_program_watermarks(dev_priv);
2238 mutex_unlock(&dev_priv->wm.wm_mutex);
2241 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2242 struct intel_crtc *crtc)
2244 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2245 const struct intel_crtc_state *crtc_state =
2246 intel_atomic_get_new_crtc_state(state, crtc);
2248 if (!crtc_state->wm.need_postvbl_update)
2251 mutex_lock(&dev_priv->wm.wm_mutex);
2252 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2253 vlv_program_watermarks(dev_priv);
2254 mutex_unlock(&dev_priv->wm.wm_mutex);
2257 static void i965_update_wm(struct drm_i915_private *dev_priv)
2259 struct intel_crtc *crtc;
2264 /* Calc sr entries for one plane configs */
2265 crtc = single_enabled_crtc(dev_priv);
2267 /* self-refresh has much higher latency */
2268 static const int sr_latency_ns = 12000;
2269 const struct drm_display_mode *pipe_mode =
2270 &crtc->config->hw.pipe_mode;
2271 const struct drm_framebuffer *fb =
2272 crtc->base.primary->state->fb;
2273 int pixel_rate = crtc->config->pixel_rate;
2274 int htotal = pipe_mode->crtc_htotal;
2275 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2276 int cpp = fb->format->cpp[0];
2279 entries = intel_wm_method2(pixel_rate, htotal,
2280 width, cpp, sr_latency_ns / 100);
2281 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2282 srwm = I965_FIFO_SIZE - entries;
2286 drm_dbg_kms(&dev_priv->drm,
2287 "self-refresh entries: %d, wm: %d\n",
2290 entries = intel_wm_method2(pixel_rate, htotal,
2291 crtc->base.cursor->state->crtc_w, 4,
2292 sr_latency_ns / 100);
2293 entries = DIV_ROUND_UP(entries,
2294 i965_cursor_wm_info.cacheline_size) +
2295 i965_cursor_wm_info.guard_size;
2297 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2298 if (cursor_sr > i965_cursor_wm_info.max_wm)
2299 cursor_sr = i965_cursor_wm_info.max_wm;
2301 drm_dbg_kms(&dev_priv->drm,
2302 "self-refresh watermark: display plane %d "
2303 "cursor %d\n", srwm, cursor_sr);
2305 cxsr_enabled = true;
2307 cxsr_enabled = false;
2308 /* Turn off self refresh if both pipes are enabled */
2309 intel_set_memory_cxsr(dev_priv, false);
2312 drm_dbg_kms(&dev_priv->drm,
2313 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2316 /* 965 has limitations... */
2317 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2321 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2322 FW_WM(8, PLANEC_OLD));
2323 /* update cursor SR watermark */
2324 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2327 intel_set_memory_cxsr(dev_priv, true);
2332 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2333 enum i9xx_plane_id i9xx_plane)
2335 struct intel_plane *plane;
2337 for_each_intel_plane(&i915->drm, plane) {
2338 if (plane->id == PLANE_PRIMARY &&
2339 plane->i9xx_plane == i9xx_plane)
2340 return intel_crtc_for_pipe(i915, plane->pipe);
2346 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2348 const struct intel_watermark_params *wm_info;
2353 int planea_wm, planeb_wm;
2354 struct intel_crtc *crtc;
2356 if (IS_I945GM(dev_priv))
2357 wm_info = &i945_wm_info;
2358 else if (DISPLAY_VER(dev_priv) != 2)
2359 wm_info = &i915_wm_info;
2361 wm_info = &i830_a_wm_info;
2363 if (DISPLAY_VER(dev_priv) == 2)
2364 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2366 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2367 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2368 if (intel_crtc_active(crtc)) {
2369 const struct drm_framebuffer *fb =
2370 crtc->base.primary->state->fb;
2373 if (DISPLAY_VER(dev_priv) == 2)
2376 cpp = fb->format->cpp[0];
2378 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2379 wm_info, fifo_size, cpp,
2380 pessimal_latency_ns);
2382 planea_wm = fifo_size - wm_info->guard_size;
2383 if (planea_wm > (long)wm_info->max_wm)
2384 planea_wm = wm_info->max_wm;
2387 if (DISPLAY_VER(dev_priv) == 2)
2388 wm_info = &i830_bc_wm_info;
2390 if (DISPLAY_VER(dev_priv) == 2)
2391 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2393 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2394 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2395 if (intel_crtc_active(crtc)) {
2396 const struct drm_framebuffer *fb =
2397 crtc->base.primary->state->fb;
2400 if (DISPLAY_VER(dev_priv) == 2)
2403 cpp = fb->format->cpp[0];
2405 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2406 wm_info, fifo_size, cpp,
2407 pessimal_latency_ns);
2409 planeb_wm = fifo_size - wm_info->guard_size;
2410 if (planeb_wm > (long)wm_info->max_wm)
2411 planeb_wm = wm_info->max_wm;
2414 drm_dbg_kms(&dev_priv->drm,
2415 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2417 crtc = single_enabled_crtc(dev_priv);
2418 if (IS_I915GM(dev_priv) && crtc) {
2419 struct drm_i915_gem_object *obj;
2421 obj = intel_fb_obj(crtc->base.primary->state->fb);
2423 /* self-refresh seems busted with untiled */
2424 if (!i915_gem_object_is_tiled(obj))
2429 * Overlay gets an aggressive default since video jitter is bad.
2433 /* Play safe and disable self-refresh before adjusting watermarks. */
2434 intel_set_memory_cxsr(dev_priv, false);
2436 /* Calc sr entries for one plane configs */
2437 if (HAS_FW_BLC(dev_priv) && crtc) {
2438 /* self-refresh has much higher latency */
2439 static const int sr_latency_ns = 6000;
2440 const struct drm_display_mode *pipe_mode =
2441 &crtc->config->hw.pipe_mode;
2442 const struct drm_framebuffer *fb =
2443 crtc->base.primary->state->fb;
2444 int pixel_rate = crtc->config->pixel_rate;
2445 int htotal = pipe_mode->crtc_htotal;
2446 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2450 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2453 cpp = fb->format->cpp[0];
2455 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2456 sr_latency_ns / 100);
2457 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2458 drm_dbg_kms(&dev_priv->drm,
2459 "self-refresh entries: %d\n", entries);
2460 srwm = wm_info->fifo_size - entries;
2464 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2465 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2466 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2468 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2471 drm_dbg_kms(&dev_priv->drm,
2472 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2473 planea_wm, planeb_wm, cwm, srwm);
2475 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2476 fwater_hi = (cwm & 0x1f);
2478 /* Set request length to 8 cachelines per fetch */
2479 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2480 fwater_hi = fwater_hi | (1 << 8);
2482 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2483 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2486 intel_set_memory_cxsr(dev_priv, true);
2489 static void i845_update_wm(struct drm_i915_private *dev_priv)
2491 struct intel_crtc *crtc;
2495 crtc = single_enabled_crtc(dev_priv);
2499 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2501 i845_get_fifo_size(dev_priv, PLANE_A),
2502 4, pessimal_latency_ns);
2503 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2504 fwater_lo |= (3<<8) | planea_wm;
2506 drm_dbg_kms(&dev_priv->drm,
2507 "Setting FIFO watermarks - A: %d\n", planea_wm);
2509 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2512 /* latency must be in 0.1us units. */
2513 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2515 unsigned int latency)
2519 ret = intel_wm_method1(pixel_rate, cpp, latency);
2520 ret = DIV_ROUND_UP(ret, 64) + 2;
2525 /* latency must be in 0.1us units. */
2526 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2527 unsigned int htotal,
2530 unsigned int latency)
2534 ret = intel_wm_method2(pixel_rate, htotal,
2535 width, cpp, latency);
2536 ret = DIV_ROUND_UP(ret, 64) + 2;
2541 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2544 * Neither of these should be possible since this function shouldn't be
2545 * called if the CRTC is off or the plane is invisible. But let's be
2546 * extra paranoid to avoid a potential divide-by-zero if we screw up
2547 * elsewhere in the driver.
2551 if (WARN_ON(!horiz_pixels))
2554 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2557 struct ilk_wm_maximums {
2565 * For both WM_PIPE and WM_LP.
2566 * mem_value must be in 0.1us units.
2568 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2569 const struct intel_plane_state *plane_state,
2570 u32 mem_value, bool is_lp)
2572 u32 method1, method2;
2578 if (!intel_wm_plane_visible(crtc_state, plane_state))
2581 cpp = plane_state->hw.fb->format->cpp[0];
2583 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2588 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2589 crtc_state->hw.pipe_mode.crtc_htotal,
2590 drm_rect_width(&plane_state->uapi.src) >> 16,
2593 return min(method1, method2);
2597 * For both WM_PIPE and WM_LP.
2598 * mem_value must be in 0.1us units.
2600 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2601 const struct intel_plane_state *plane_state,
2604 u32 method1, method2;
2610 if (!intel_wm_plane_visible(crtc_state, plane_state))
2613 cpp = plane_state->hw.fb->format->cpp[0];
2615 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2616 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2617 crtc_state->hw.pipe_mode.crtc_htotal,
2618 drm_rect_width(&plane_state->uapi.src) >> 16,
2620 return min(method1, method2);
2624 * For both WM_PIPE and WM_LP.
2625 * mem_value must be in 0.1us units.
2627 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2628 const struct intel_plane_state *plane_state,
2636 if (!intel_wm_plane_visible(crtc_state, plane_state))
2639 cpp = plane_state->hw.fb->format->cpp[0];
2641 return ilk_wm_method2(crtc_state->pixel_rate,
2642 crtc_state->hw.pipe_mode.crtc_htotal,
2643 drm_rect_width(&plane_state->uapi.src) >> 16,
2647 /* Only for WM_LP. */
2648 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2649 const struct intel_plane_state *plane_state,
2654 if (!intel_wm_plane_visible(crtc_state, plane_state))
2657 cpp = plane_state->hw.fb->format->cpp[0];
2659 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2664 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2666 if (DISPLAY_VER(dev_priv) >= 8)
2668 else if (DISPLAY_VER(dev_priv) >= 7)
2675 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2676 int level, bool is_sprite)
2678 if (DISPLAY_VER(dev_priv) >= 8)
2679 /* BDW primary/sprite plane watermarks */
2680 return level == 0 ? 255 : 2047;
2681 else if (DISPLAY_VER(dev_priv) >= 7)
2682 /* IVB/HSW primary/sprite plane watermarks */
2683 return level == 0 ? 127 : 1023;
2684 else if (!is_sprite)
2685 /* ILK/SNB primary plane watermarks */
2686 return level == 0 ? 127 : 511;
2688 /* ILK/SNB sprite plane watermarks */
2689 return level == 0 ? 63 : 255;
2693 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2695 if (DISPLAY_VER(dev_priv) >= 7)
2696 return level == 0 ? 63 : 255;
2698 return level == 0 ? 31 : 63;
2701 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2703 if (DISPLAY_VER(dev_priv) >= 8)
2709 /* Calculate the maximum primary/sprite plane watermark */
2710 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2712 const struct intel_wm_config *config,
2713 enum intel_ddb_partitioning ddb_partitioning,
2716 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2718 /* if sprites aren't enabled, sprites get nothing */
2719 if (is_sprite && !config->sprites_enabled)
2722 /* HSW allows LP1+ watermarks even with multiple pipes */
2723 if (level == 0 || config->num_pipes_active > 1) {
2724 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2727 * For some reason the non self refresh
2728 * FIFO size is only half of the self
2729 * refresh FIFO size on ILK/SNB.
2731 if (DISPLAY_VER(dev_priv) <= 6)
2735 if (config->sprites_enabled) {
2736 /* level 0 is always calculated with 1:1 split */
2737 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2746 /* clamp to max that the registers can hold */
2747 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2750 /* Calculate the maximum cursor plane watermark */
2751 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2753 const struct intel_wm_config *config)
2755 /* HSW LP1+ watermarks w/ multiple pipes */
2756 if (level > 0 && config->num_pipes_active > 1)
2759 /* otherwise just report max that registers can hold */
2760 return ilk_cursor_wm_reg_max(dev_priv, level);
2763 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2765 const struct intel_wm_config *config,
2766 enum intel_ddb_partitioning ddb_partitioning,
2767 struct ilk_wm_maximums *max)
2769 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2770 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2771 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2772 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2775 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2777 struct ilk_wm_maximums *max)
2779 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2780 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2781 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2782 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2785 static bool ilk_validate_wm_level(int level,
2786 const struct ilk_wm_maximums *max,
2787 struct intel_wm_level *result)
2791 /* already determined to be invalid? */
2792 if (!result->enable)
2795 result->enable = result->pri_val <= max->pri &&
2796 result->spr_val <= max->spr &&
2797 result->cur_val <= max->cur;
2799 ret = result->enable;
2802 * HACK until we can pre-compute everything,
2803 * and thus fail gracefully if LP0 watermarks
2806 if (level == 0 && !result->enable) {
2807 if (result->pri_val > max->pri)
2808 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2809 level, result->pri_val, max->pri);
2810 if (result->spr_val > max->spr)
2811 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2812 level, result->spr_val, max->spr);
2813 if (result->cur_val > max->cur)
2814 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2815 level, result->cur_val, max->cur);
2817 result->pri_val = min_t(u32, result->pri_val, max->pri);
2818 result->spr_val = min_t(u32, result->spr_val, max->spr);
2819 result->cur_val = min_t(u32, result->cur_val, max->cur);
2820 result->enable = true;
2826 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2827 const struct intel_crtc *crtc,
2829 struct intel_crtc_state *crtc_state,
2830 const struct intel_plane_state *pristate,
2831 const struct intel_plane_state *sprstate,
2832 const struct intel_plane_state *curstate,
2833 struct intel_wm_level *result)
2835 u16 pri_latency = dev_priv->wm.pri_latency[level];
2836 u16 spr_latency = dev_priv->wm.spr_latency[level];
2837 u16 cur_latency = dev_priv->wm.cur_latency[level];
2839 /* WM1+ latency values stored in 0.5us units */
2847 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2848 pri_latency, level);
2849 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2853 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2856 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2858 result->enable = true;
2861 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2864 struct intel_uncore *uncore = &dev_priv->uncore;
2866 if (DISPLAY_VER(dev_priv) >= 9) {
2869 int level, max_level = ilk_wm_max_level(dev_priv);
2870 int mult = IS_DG2(dev_priv) ? 2 : 1;
2872 /* read the first set of memory latencies[0:3] */
2873 val = 0; /* data0 to be programmed to 0 for first set */
2874 ret = snb_pcode_read(dev_priv, GEN9_PCODE_READ_MEM_LATENCY,
2878 drm_err(&dev_priv->drm,
2879 "SKL Mailbox read error = %d\n", ret);
2883 wm[0] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2884 wm[1] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2885 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2886 wm[2] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2887 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2888 wm[3] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2889 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2891 /* read the second set of memory latencies[4:7] */
2892 val = 1; /* data0 to be programmed to 1 for second set */
2893 ret = snb_pcode_read(dev_priv, GEN9_PCODE_READ_MEM_LATENCY,
2896 drm_err(&dev_priv->drm,
2897 "SKL Mailbox read error = %d\n", ret);
2901 wm[4] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2902 wm[5] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2903 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2904 wm[6] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2905 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2906 wm[7] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2907 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2910 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2911 * need to be disabled. We make sure to sanitize the values out
2912 * of the punit to satisfy this requirement.
2914 for (level = 1; level <= max_level; level++) {
2915 if (wm[level] == 0) {
2916 for (i = level + 1; i <= max_level; i++)
2919 max_level = level - 1;
2926 * WaWmMemoryReadLatency
2928 * punit doesn't take into account the read latency so we need
2929 * to add proper adjustement to each valid level we retrieve
2930 * from the punit when level 0 response data is 0us.
2933 u8 adjust = DISPLAY_VER(dev_priv) >= 12 ? 3 : 2;
2935 for (level = 0; level <= max_level; level++)
2936 wm[level] += adjust;
2940 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2941 * If we could not get dimm info enable this WA to prevent from
2942 * any underrun. If not able to get Dimm info assume 16GB dimm
2943 * to avoid any underrun.
2945 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2947 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2948 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2950 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2952 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2953 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2954 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2955 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2956 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2957 } else if (DISPLAY_VER(dev_priv) >= 6) {
2958 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2960 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2961 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2962 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2963 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2964 } else if (DISPLAY_VER(dev_priv) >= 5) {
2965 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2967 /* ILK primary LP0 latency is 700 ns */
2969 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2970 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2972 MISSING_CASE(INTEL_DEVID(dev_priv));
2976 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2979 /* ILK sprite LP0 latency is 1300 ns */
2980 if (DISPLAY_VER(dev_priv) == 5)
2984 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2987 /* ILK cursor LP0 latency is 1300 ns */
2988 if (DISPLAY_VER(dev_priv) == 5)
2992 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2994 /* how many WM levels are we expecting */
2995 if (HAS_HW_SAGV_WM(dev_priv))
2997 else if (DISPLAY_VER(dev_priv) >= 9)
2999 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3001 else if (DISPLAY_VER(dev_priv) >= 6)
3007 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3011 int level, max_level = ilk_wm_max_level(dev_priv);
3013 for (level = 0; level <= max_level; level++) {
3014 unsigned int latency = wm[level];
3017 drm_dbg_kms(&dev_priv->drm,
3018 "%s WM%d latency not provided\n",
3024 * - latencies are in us on gen9.
3025 * - before then, WM1+ latency values are in 0.5us units
3027 if (DISPLAY_VER(dev_priv) >= 9)
3032 drm_dbg_kms(&dev_priv->drm,
3033 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3034 wm[level], latency / 10, latency % 10);
3038 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3041 int level, max_level = ilk_wm_max_level(dev_priv);
3046 wm[0] = max(wm[0], min);
3047 for (level = 1; level <= max_level; level++)
3048 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3053 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3058 * The BIOS provided WM memory latency values are often
3059 * inadequate for high resolution displays. Adjust them.
3061 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12);
3062 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12);
3063 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3068 drm_dbg_kms(&dev_priv->drm,
3069 "WM latency values increased to avoid potential underruns\n");
3070 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3071 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3072 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3075 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3078 * On some SNB machines (Thinkpad X220 Tablet at least)
3079 * LP3 usage can cause vblank interrupts to be lost.
3080 * The DEIIR bit will go high but it looks like the CPU
3081 * never gets interrupted.
3083 * It's not clear whether other interrupt source could
3084 * be affected or if this is somehow limited to vblank
3085 * interrupts only. To play it safe we disable LP3
3086 * watermarks entirely.
3088 if (dev_priv->wm.pri_latency[3] == 0 &&
3089 dev_priv->wm.spr_latency[3] == 0 &&
3090 dev_priv->wm.cur_latency[3] == 0)
3093 dev_priv->wm.pri_latency[3] = 0;
3094 dev_priv->wm.spr_latency[3] = 0;
3095 dev_priv->wm.cur_latency[3] = 0;
3097 drm_dbg_kms(&dev_priv->drm,
3098 "LP3 watermarks disabled due to potential for lost interrupts\n");
3099 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3100 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3101 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3104 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3106 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3108 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3109 sizeof(dev_priv->wm.pri_latency));
3110 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3111 sizeof(dev_priv->wm.pri_latency));
3113 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3114 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3116 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3117 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3118 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3120 if (DISPLAY_VER(dev_priv) == 6) {
3121 snb_wm_latency_quirk(dev_priv);
3122 snb_wm_lp3_irq_quirk(dev_priv);
3126 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3128 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3129 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3132 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3133 struct intel_pipe_wm *pipe_wm)
3135 /* LP0 watermark maximums depend on this pipe alone */
3136 const struct intel_wm_config config = {
3137 .num_pipes_active = 1,
3138 .sprites_enabled = pipe_wm->sprites_enabled,
3139 .sprites_scaled = pipe_wm->sprites_scaled,
3141 struct ilk_wm_maximums max;
3143 /* LP0 watermarks always use 1/2 DDB partitioning */
3144 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3146 /* At least LP0 must be valid */
3147 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3148 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3155 /* Compute new watermarks for the pipe */
3156 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3157 struct intel_crtc *crtc)
3159 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3160 struct intel_crtc_state *crtc_state =
3161 intel_atomic_get_new_crtc_state(state, crtc);
3162 struct intel_pipe_wm *pipe_wm;
3163 struct intel_plane *plane;
3164 const struct intel_plane_state *plane_state;
3165 const struct intel_plane_state *pristate = NULL;
3166 const struct intel_plane_state *sprstate = NULL;
3167 const struct intel_plane_state *curstate = NULL;
3168 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3169 struct ilk_wm_maximums max;
3171 pipe_wm = &crtc_state->wm.ilk.optimal;
3173 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3174 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3175 pristate = plane_state;
3176 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3177 sprstate = plane_state;
3178 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3179 curstate = plane_state;
3182 pipe_wm->pipe_enabled = crtc_state->hw.active;
3183 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3184 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3186 usable_level = max_level;
3188 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3189 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3192 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3193 if (pipe_wm->sprites_scaled)
3196 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3197 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3198 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3200 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3203 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3205 for (level = 1; level <= usable_level; level++) {
3206 struct intel_wm_level *wm = &pipe_wm->wm[level];
3208 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3209 pristate, sprstate, curstate, wm);
3212 * Disable any watermark level that exceeds the
3213 * register maximums since such watermarks are
3216 if (!ilk_validate_wm_level(level, &max, wm)) {
3217 memset(wm, 0, sizeof(*wm));
3226 * Build a set of 'intermediate' watermark values that satisfy both the old
3227 * state and the new state. These can be programmed to the hardware
3230 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3231 struct intel_crtc *crtc)
3233 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3234 struct intel_crtc_state *new_crtc_state =
3235 intel_atomic_get_new_crtc_state(state, crtc);
3236 const struct intel_crtc_state *old_crtc_state =
3237 intel_atomic_get_old_crtc_state(state, crtc);
3238 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3239 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3240 int level, max_level = ilk_wm_max_level(dev_priv);
3243 * Start with the final, target watermarks, then combine with the
3244 * currently active watermarks to get values that are safe both before
3245 * and after the vblank.
3247 *a = new_crtc_state->wm.ilk.optimal;
3248 if (!new_crtc_state->hw.active ||
3249 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) ||
3250 state->skip_intermediate_wm)
3253 a->pipe_enabled |= b->pipe_enabled;
3254 a->sprites_enabled |= b->sprites_enabled;
3255 a->sprites_scaled |= b->sprites_scaled;
3257 for (level = 0; level <= max_level; level++) {
3258 struct intel_wm_level *a_wm = &a->wm[level];
3259 const struct intel_wm_level *b_wm = &b->wm[level];
3261 a_wm->enable &= b_wm->enable;
3262 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3263 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3264 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3265 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3269 * We need to make sure that these merged watermark values are
3270 * actually a valid configuration themselves. If they're not,
3271 * there's no safe way to transition from the old state to
3272 * the new state, so we need to fail the atomic transaction.
3274 if (!ilk_validate_pipe_wm(dev_priv, a))
3278 * If our intermediate WM are identical to the final WM, then we can
3279 * omit the post-vblank programming; only update if it's different.
3281 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3282 new_crtc_state->wm.need_postvbl_update = true;
3288 * Merge the watermarks from all active pipes for a specific level.
3290 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3292 struct intel_wm_level *ret_wm)
3294 const struct intel_crtc *crtc;
3296 ret_wm->enable = true;
3298 for_each_intel_crtc(&dev_priv->drm, crtc) {
3299 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3300 const struct intel_wm_level *wm = &active->wm[level];
3302 if (!active->pipe_enabled)
3306 * The watermark values may have been used in the past,
3307 * so we must maintain them in the registers for some
3308 * time even if the level is now disabled.
3311 ret_wm->enable = false;
3313 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3314 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3315 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3316 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3321 * Merge all low power watermarks for all active pipes.
3323 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3324 const struct intel_wm_config *config,
3325 const struct ilk_wm_maximums *max,
3326 struct intel_pipe_wm *merged)
3328 int level, max_level = ilk_wm_max_level(dev_priv);
3329 int last_enabled_level = max_level;
3331 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3332 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3333 config->num_pipes_active > 1)
3334 last_enabled_level = 0;
3336 /* ILK: FBC WM must be disabled always */
3337 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3339 /* merge each WM1+ level */
3340 for (level = 1; level <= max_level; level++) {
3341 struct intel_wm_level *wm = &merged->wm[level];
3343 ilk_merge_wm_level(dev_priv, level, wm);
3345 if (level > last_enabled_level)
3347 else if (!ilk_validate_wm_level(level, max, wm))
3348 /* make sure all following levels get disabled */
3349 last_enabled_level = level - 1;
3352 * The spec says it is preferred to disable
3353 * FBC WMs instead of disabling a WM level.
3355 if (wm->fbc_val > max->fbc) {
3357 merged->fbc_wm_enabled = false;
3362 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3363 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3364 dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3365 for (level = 2; level <= max_level; level++) {
3366 struct intel_wm_level *wm = &merged->wm[level];
3373 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3375 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3376 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3379 /* The value we need to program into the WM_LPx latency field */
3380 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3383 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3386 return dev_priv->wm.pri_latency[level];
3389 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3390 const struct intel_pipe_wm *merged,
3391 enum intel_ddb_partitioning partitioning,
3392 struct ilk_wm_values *results)
3394 struct intel_crtc *crtc;
3397 results->enable_fbc_wm = merged->fbc_wm_enabled;
3398 results->partitioning = partitioning;
3400 /* LP1+ register values */
3401 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3402 const struct intel_wm_level *r;
3404 level = ilk_wm_lp_to_level(wm_lp, merged);
3406 r = &merged->wm[level];
3409 * Maintain the watermark values even if the level is
3410 * disabled. Doing otherwise could cause underruns.
3412 results->wm_lp[wm_lp - 1] =
3413 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3414 WM_LP_PRIMARY(r->pri_val) |
3415 WM_LP_CURSOR(r->cur_val);
3418 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3420 if (DISPLAY_VER(dev_priv) >= 8)
3421 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3423 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3425 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3428 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3429 * level is disabled. Doing otherwise could cause underruns.
3431 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3432 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3433 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3437 /* LP0 register values */
3438 for_each_intel_crtc(&dev_priv->drm, crtc) {
3439 enum pipe pipe = crtc->pipe;
3440 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3441 const struct intel_wm_level *r = &pipe_wm->wm[0];
3443 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3446 results->wm_pipe[pipe] =
3447 WM0_PIPE_PRIMARY(r->pri_val) |
3448 WM0_PIPE_SPRITE(r->spr_val) |
3449 WM0_PIPE_CURSOR(r->cur_val);
3453 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3454 * case both are at the same level. Prefer r1 in case they're the same. */
3455 static struct intel_pipe_wm *
3456 ilk_find_best_result(struct drm_i915_private *dev_priv,
3457 struct intel_pipe_wm *r1,
3458 struct intel_pipe_wm *r2)
3460 int level, max_level = ilk_wm_max_level(dev_priv);
3461 int level1 = 0, level2 = 0;
3463 for (level = 1; level <= max_level; level++) {
3464 if (r1->wm[level].enable)
3466 if (r2->wm[level].enable)
3470 if (level1 == level2) {
3471 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3475 } else if (level1 > level2) {
3482 /* dirty bits used to track which watermarks need changes */
3483 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3484 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3485 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3486 #define WM_DIRTY_FBC (1 << 24)
3487 #define WM_DIRTY_DDB (1 << 25)
3489 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3490 const struct ilk_wm_values *old,
3491 const struct ilk_wm_values *new)
3493 unsigned int dirty = 0;
3497 for_each_pipe(dev_priv, pipe) {
3498 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3499 dirty |= WM_DIRTY_PIPE(pipe);
3500 /* Must disable LP1+ watermarks too */
3501 dirty |= WM_DIRTY_LP_ALL;
3505 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3506 dirty |= WM_DIRTY_FBC;
3507 /* Must disable LP1+ watermarks too */
3508 dirty |= WM_DIRTY_LP_ALL;
3511 if (old->partitioning != new->partitioning) {
3512 dirty |= WM_DIRTY_DDB;
3513 /* Must disable LP1+ watermarks too */
3514 dirty |= WM_DIRTY_LP_ALL;
3517 /* LP1+ watermarks already deemed dirty, no need to continue */
3518 if (dirty & WM_DIRTY_LP_ALL)
3521 /* Find the lowest numbered LP1+ watermark in need of an update... */
3522 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3523 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3524 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3528 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3529 for (; wm_lp <= 3; wm_lp++)
3530 dirty |= WM_DIRTY_LP(wm_lp);
3535 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3538 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3539 bool changed = false;
3541 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3542 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3543 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3546 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3547 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3548 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3551 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3552 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3553 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3558 * Don't touch WM_LP_SPRITE_ENABLE here.
3559 * Doing so could cause underruns.
3566 * The spec says we shouldn't write when we don't need, because every write
3567 * causes WMs to be re-evaluated, expending some power.
3569 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3570 struct ilk_wm_values *results)
3572 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3576 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3580 _ilk_disable_lp_wm(dev_priv, dirty);
3582 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3583 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3584 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3585 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3586 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3587 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3589 if (dirty & WM_DIRTY_DDB) {
3590 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3591 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3592 if (results->partitioning == INTEL_DDB_PART_1_2)
3593 val &= ~WM_MISC_DATA_PARTITION_5_6;
3595 val |= WM_MISC_DATA_PARTITION_5_6;
3596 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3598 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3599 if (results->partitioning == INTEL_DDB_PART_1_2)
3600 val &= ~DISP_DATA_PARTITION_5_6;
3602 val |= DISP_DATA_PARTITION_5_6;
3603 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3607 if (dirty & WM_DIRTY_FBC) {
3608 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3609 if (results->enable_fbc_wm)
3610 val &= ~DISP_FBC_WM_DIS;
3612 val |= DISP_FBC_WM_DIS;
3613 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3616 if (dirty & WM_DIRTY_LP(1) &&
3617 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3618 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3620 if (DISPLAY_VER(dev_priv) >= 7) {
3621 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3622 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3623 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3624 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3627 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3628 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3629 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3630 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3631 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3632 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3634 dev_priv->wm.hw = *results;
3637 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3639 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3642 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3644 u8 enabled_slices = 0;
3645 enum dbuf_slice slice;
3647 for_each_dbuf_slice(dev_priv, slice) {
3648 if (intel_uncore_read(&dev_priv->uncore,
3649 DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
3650 enabled_slices |= BIT(slice);
3653 return enabled_slices;
3657 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3658 * so assume we'll always need it in order to avoid underruns.
3660 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3662 return DISPLAY_VER(dev_priv) == 9;
3666 intel_has_sagv(struct drm_i915_private *dev_priv)
3668 return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
3669 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3673 skl_setup_sagv_block_time(struct drm_i915_private *dev_priv)
3675 if (DISPLAY_VER(dev_priv) >= 12) {
3679 ret = snb_pcode_read(dev_priv,
3680 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3683 dev_priv->sagv_block_time_us = val;
3687 drm_dbg(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3688 } else if (DISPLAY_VER(dev_priv) == 11) {
3689 dev_priv->sagv_block_time_us = 10;
3691 } else if (DISPLAY_VER(dev_priv) == 10) {
3692 dev_priv->sagv_block_time_us = 20;
3694 } else if (DISPLAY_VER(dev_priv) == 9) {
3695 dev_priv->sagv_block_time_us = 30;
3698 MISSING_CASE(DISPLAY_VER(dev_priv));
3701 dev_priv->sagv_block_time_us = 0;
3705 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3706 * depending on power and performance requirements. The display engine access
3707 * to system memory is blocked during the adjustment time. Because of the
3708 * blocking time, having this enabled can cause full system hangs and/or pipe
3709 * underruns if we don't meet all of the following requirements:
3711 * - <= 1 pipe enabled
3712 * - All planes can enable watermarks for latencies >= SAGV engine block time
3713 * - We're not using an interlaced display configuration
3716 intel_enable_sagv(struct drm_i915_private *dev_priv)
3720 if (!intel_has_sagv(dev_priv))
3723 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3726 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3727 ret = snb_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3730 /* We don't need to wait for SAGV when enabling */
3733 * Some skl systems, pre-release machines in particular,
3734 * don't actually have SAGV.
3736 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3737 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3738 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3740 } else if (ret < 0) {
3741 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3745 dev_priv->sagv_status = I915_SAGV_ENABLED;
3750 intel_disable_sagv(struct drm_i915_private *dev_priv)
3754 if (!intel_has_sagv(dev_priv))
3757 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3760 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3761 /* bspec says to keep retrying for at least 1 ms */
3762 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3764 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3767 * Some skl systems, pre-release machines in particular,
3768 * don't actually have SAGV.
3770 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3771 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3772 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3774 } else if (ret < 0) {
3775 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3779 dev_priv->sagv_status = I915_SAGV_DISABLED;
3783 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
3785 struct drm_i915_private *i915 = to_i915(state->base.dev);
3786 const struct intel_bw_state *new_bw_state =
3787 intel_atomic_get_new_bw_state(state);
3792 if (!intel_can_enable_sagv(i915, new_bw_state))
3793 intel_disable_sagv(i915);
3796 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
3798 struct drm_i915_private *i915 = to_i915(state->base.dev);
3799 const struct intel_bw_state *new_bw_state =
3800 intel_atomic_get_new_bw_state(state);
3805 if (intel_can_enable_sagv(i915, new_bw_state))
3806 intel_enable_sagv(i915);
3809 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
3811 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3812 const struct intel_bw_state *old_bw_state =
3813 intel_atomic_get_old_bw_state(state);
3814 const struct intel_bw_state *new_bw_state =
3815 intel_atomic_get_new_bw_state(state);
3816 u16 old_mask, new_mask;
3821 old_mask = old_bw_state->qgv_points_mask;
3822 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3824 if (old_mask == new_mask)
3827 WARN_ON(!new_bw_state->base.changed);
3829 drm_dbg_kms(&dev_priv->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
3830 old_mask, new_mask);
3833 * Restrict required qgv points before updating the configuration.
3834 * According to BSpec we can't mask and unmask qgv points at the same
3835 * time. Also masking should be done before updating the configuration
3836 * and unmasking afterwards.
3838 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3841 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
3843 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3844 const struct intel_bw_state *old_bw_state =
3845 intel_atomic_get_old_bw_state(state);
3846 const struct intel_bw_state *new_bw_state =
3847 intel_atomic_get_new_bw_state(state);
3848 u16 old_mask, new_mask;
3853 old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3854 new_mask = new_bw_state->qgv_points_mask;
3856 if (old_mask == new_mask)
3859 WARN_ON(!new_bw_state->base.changed);
3861 drm_dbg_kms(&dev_priv->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
3862 old_mask, new_mask);
3865 * Allow required qgv points after updating the configuration.
3866 * According to BSpec we can't mask and unmask qgv points at the same
3867 * time. Also masking should be done before updating the configuration
3868 * and unmasking afterwards.
3870 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3873 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3875 struct drm_i915_private *i915 = to_i915(state->base.dev);
3878 * Just return if we can't control SAGV or don't have it.
3879 * This is different from situation when we have SAGV but just can't
3880 * afford it due to DBuf limitation - in case if SAGV is completely
3881 * disabled in a BIOS, we are not even allowed to send a PCode request,
3882 * as it will throw an error. So have to check it here.
3884 if (!intel_has_sagv(i915))
3887 if (DISPLAY_VER(i915) >= 11)
3888 icl_sagv_pre_plane_update(state);
3890 skl_sagv_pre_plane_update(state);
3893 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3895 struct drm_i915_private *i915 = to_i915(state->base.dev);
3898 * Just return if we can't control SAGV or don't have it.
3899 * This is different from situation when we have SAGV but just can't
3900 * afford it due to DBuf limitation - in case if SAGV is completely
3901 * disabled in a BIOS, we are not even allowed to send a PCode request,
3902 * as it will throw an error. So have to check it here.
3904 if (!intel_has_sagv(i915))
3907 if (DISPLAY_VER(i915) >= 11)
3908 icl_sagv_post_plane_update(state);
3910 skl_sagv_post_plane_update(state);
3913 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3915 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3916 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3917 enum plane_id plane_id;
3918 int max_level = INT_MAX;
3920 if (!intel_has_sagv(dev_priv))
3923 if (!crtc_state->hw.active)
3926 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3929 for_each_plane_id_on_crtc(crtc, plane_id) {
3930 const struct skl_plane_wm *wm =
3931 &crtc_state->wm.skl.optimal.planes[plane_id];
3934 /* Skip this plane if it's not enabled */
3935 if (!wm->wm[0].enable)
3938 /* Find the highest enabled wm level for this plane */
3939 for (level = ilk_wm_max_level(dev_priv);
3940 !wm->wm[level].enable; --level)
3943 /* Highest common enabled wm level for all planes */
3944 max_level = min(level, max_level);
3947 /* No enabled planes? */
3948 if (max_level == INT_MAX)
3951 for_each_plane_id_on_crtc(crtc, plane_id) {
3952 const struct skl_plane_wm *wm =
3953 &crtc_state->wm.skl.optimal.planes[plane_id];
3956 * All enabled planes must have enabled a common wm level that
3957 * can tolerate memory latencies higher than sagv_block_time_us
3959 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3966 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3968 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3969 enum plane_id plane_id;
3971 if (!crtc_state->hw.active)
3974 for_each_plane_id_on_crtc(crtc, plane_id) {
3975 const struct skl_plane_wm *wm =
3976 &crtc_state->wm.skl.optimal.planes[plane_id];
3978 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
3985 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3987 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3988 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3990 if (DISPLAY_VER(dev_priv) >= 12)
3991 return tgl_crtc_can_enable_sagv(crtc_state);
3993 return skl_crtc_can_enable_sagv(crtc_state);
3996 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
3997 const struct intel_bw_state *bw_state)
3999 if (DISPLAY_VER(dev_priv) < 11 &&
4000 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
4003 return bw_state->pipe_sagv_reject == 0;
4006 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
4008 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
4010 struct intel_crtc *crtc;
4011 struct intel_crtc_state *new_crtc_state;
4012 struct intel_bw_state *new_bw_state = NULL;
4013 const struct intel_bw_state *old_bw_state = NULL;
4016 for_each_new_intel_crtc_in_state(state, crtc,
4017 new_crtc_state, i) {
4018 new_bw_state = intel_atomic_get_bw_state(state);
4019 if (IS_ERR(new_bw_state))
4020 return PTR_ERR(new_bw_state);
4022 old_bw_state = intel_atomic_get_old_bw_state(state);
4024 if (intel_crtc_can_enable_sagv(new_crtc_state))
4025 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
4027 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
4033 new_bw_state->active_pipes =
4034 intel_calc_active_pipes(state, old_bw_state->active_pipes);
4036 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4037 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4042 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4043 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4044 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4047 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4048 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4053 for_each_new_intel_crtc_in_state(state, crtc,
4054 new_crtc_state, i) {
4055 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4058 * We store use_sagv_wm in the crtc state rather than relying on
4059 * that bw state since we have no convenient way to get at the
4060 * latter from the plane commit hooks (especially in the legacy
4063 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
4064 DISPLAY_VER(dev_priv) >= 12 &&
4065 intel_can_enable_sagv(dev_priv, new_bw_state);
4071 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
4074 entry->start = start;
4080 static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4082 return INTEL_INFO(dev_priv)->dbuf.size /
4083 hweight8(INTEL_INFO(dev_priv)->dbuf.slice_mask);
4087 skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4088 struct skl_ddb_entry *ddb)
4090 int slice_size = intel_dbuf_slice_size(dev_priv);
4098 ddb->start = (ffs(slice_mask) - 1) * slice_size;
4099 ddb->end = fls(slice_mask) * slice_size;
4101 WARN_ON(ddb->start >= ddb->end);
4102 WARN_ON(ddb->end > INTEL_INFO(dev_priv)->dbuf.size);
4105 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
4107 struct skl_ddb_entry ddb;
4109 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
4110 slice_mask = BIT(DBUF_S1);
4111 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
4112 slice_mask = BIT(DBUF_S3);
4114 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
4119 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4120 const struct skl_ddb_entry *entry)
4122 int slice_size = intel_dbuf_slice_size(dev_priv);
4123 enum dbuf_slice start_slice, end_slice;
4126 if (!skl_ddb_entry_size(entry))
4129 start_slice = entry->start / slice_size;
4130 end_slice = (entry->end - 1) / slice_size;
4133 * Per plane DDB entry can in a really worst case be on multiple slices
4134 * but single entry is anyway contigious.
4136 while (start_slice <= end_slice) {
4137 slice_mask |= BIT(start_slice);
4144 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4146 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4147 int hdisplay, vdisplay;
4149 if (!crtc_state->hw.active)
4153 * Watermark/ddb requirement highly depends upon width of the
4154 * framebuffer, So instead of allocating DDB equally among pipes
4155 * distribute DDB based on resolution/width of the display.
4157 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4162 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4164 unsigned int *weight_start,
4165 unsigned int *weight_end,
4166 unsigned int *weight_total)
4168 struct drm_i915_private *dev_priv =
4169 to_i915(dbuf_state->base.state->base.dev);
4176 for_each_pipe(dev_priv, pipe) {
4177 int weight = dbuf_state->weight[pipe];
4180 * Do not account pipes using other slice sets
4181 * luckily as of current BSpec slice sets do not partially
4182 * intersect(pipes share either same one slice or same slice set
4183 * i.e no partial intersection), so it is enough to check for
4186 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4189 *weight_total += weight;
4190 if (pipe < for_pipe) {
4191 *weight_start += weight;
4192 *weight_end += weight;
4193 } else if (pipe == for_pipe) {
4194 *weight_end += weight;
4200 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4202 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4203 unsigned int weight_total, weight_start, weight_end;
4204 const struct intel_dbuf_state *old_dbuf_state =
4205 intel_atomic_get_old_dbuf_state(state);
4206 struct intel_dbuf_state *new_dbuf_state =
4207 intel_atomic_get_new_dbuf_state(state);
4208 struct intel_crtc_state *crtc_state;
4209 struct skl_ddb_entry ddb_slices;
4210 enum pipe pipe = crtc->pipe;
4211 unsigned int mbus_offset = 0;
4213 u32 dbuf_slice_mask;
4217 if (new_dbuf_state->weight[pipe] == 0) {
4218 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
4222 dbuf_slice_mask = new_dbuf_state->slices[pipe];
4224 skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4225 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
4226 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4228 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4229 &weight_start, &weight_end, &weight_total);
4231 start = ddb_range_size * weight_start / weight_total;
4232 end = ddb_range_size * weight_end / weight_total;
4234 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
4235 ddb_slices.start - mbus_offset + start,
4236 ddb_slices.start - mbus_offset + end);
4239 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
4240 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4241 &new_dbuf_state->ddb[pipe]))
4244 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4248 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4249 if (IS_ERR(crtc_state))
4250 return PTR_ERR(crtc_state);
4253 * Used for checking overlaps, so we need absolute
4254 * offsets instead of MBUS relative offsets.
4256 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
4257 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
4259 drm_dbg_kms(&dev_priv->drm,
4260 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4261 crtc->base.base.id, crtc->base.name,
4262 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4263 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4264 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4265 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4270 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4271 int width, const struct drm_format_info *format,
4272 u64 modifier, unsigned int rotation,
4273 u32 plane_pixel_rate, struct skl_wm_params *wp,
4276 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4277 struct intel_plane *plane,
4279 unsigned int latency,
4280 const struct skl_wm_params *wp,
4281 const struct skl_wm_level *result_prev,
4282 struct skl_wm_level *result /* out */);
4285 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4288 struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
4289 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4290 int level, max_level = ilk_wm_max_level(dev_priv);
4291 struct skl_wm_level wm = {};
4292 int ret, min_ddb_alloc = 0;
4293 struct skl_wm_params wp;
4295 ret = skl_compute_wm_params(crtc_state, 256,
4296 drm_format_info(DRM_FORMAT_ARGB8888),
4297 DRM_FORMAT_MOD_LINEAR,
4299 crtc_state->pixel_rate, &wp, 0);
4300 drm_WARN_ON(&dev_priv->drm, ret);
4302 for (level = 0; level <= max_level; level++) {
4303 unsigned int latency = dev_priv->wm.skl_latency[level];
4305 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
4306 if (wm.min_ddb_alloc == U16_MAX)
4309 min_ddb_alloc = wm.min_ddb_alloc;
4312 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4315 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
4317 skl_ddb_entry_init(entry,
4318 REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
4319 REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
4325 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4326 const enum pipe pipe,
4327 const enum plane_id plane_id,
4328 struct skl_ddb_entry *ddb_y,
4329 struct skl_ddb_entry *ddb_uv)
4334 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4335 if (plane_id == PLANE_CURSOR) {
4336 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4337 skl_ddb_entry_init_from_hw(ddb_y, val);
4341 val = intel_uncore_read(&dev_priv->uncore, PLANE_CTL(pipe, plane_id));
4343 /* No DDB allocated for disabled planes */
4344 if (val & PLANE_CTL_ENABLE)
4345 fourcc = skl_format_to_fourcc(val & PLANE_CTL_FORMAT_MASK_SKL,
4346 val & PLANE_CTL_ORDER_RGBX,
4347 val & PLANE_CTL_ALPHA_MASK);
4349 if (DISPLAY_VER(dev_priv) >= 11) {
4350 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4351 skl_ddb_entry_init_from_hw(ddb_y, val);
4353 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4354 val2 = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4357 drm_format_info_is_yuv_semiplanar(drm_format_info(fourcc)))
4360 skl_ddb_entry_init_from_hw(ddb_y, val);
4361 skl_ddb_entry_init_from_hw(ddb_uv, val2);
4365 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4366 struct skl_ddb_entry *ddb_y,
4367 struct skl_ddb_entry *ddb_uv)
4369 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4370 enum intel_display_power_domain power_domain;
4371 enum pipe pipe = crtc->pipe;
4372 intel_wakeref_t wakeref;
4373 enum plane_id plane_id;
4375 power_domain = POWER_DOMAIN_PIPE(pipe);
4376 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4380 for_each_plane_id_on_crtc(crtc, plane_id)
4381 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4386 intel_display_power_put(dev_priv, power_domain, wakeref);
4389 struct dbuf_slice_conf_entry {
4391 u8 dbuf_mask[I915_MAX_PIPES];
4396 * Table taken from Bspec 12716
4397 * Pipes do have some preferred DBuf slice affinity,
4398 * plus there are some hardcoded requirements on how
4399 * those should be distributed for multipipe scenarios.
4400 * For more DBuf slices algorithm can get even more messy
4401 * and less readable, so decided to use a table almost
4402 * as is from BSpec itself - that way it is at least easier
4403 * to compare, change and check.
4405 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4406 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4409 .active_pipes = BIT(PIPE_A),
4411 [PIPE_A] = BIT(DBUF_S1),
4415 .active_pipes = BIT(PIPE_B),
4417 [PIPE_B] = BIT(DBUF_S1),
4421 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4423 [PIPE_A] = BIT(DBUF_S1),
4424 [PIPE_B] = BIT(DBUF_S2),
4428 .active_pipes = BIT(PIPE_C),
4430 [PIPE_C] = BIT(DBUF_S2),
4434 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4436 [PIPE_A] = BIT(DBUF_S1),
4437 [PIPE_C] = BIT(DBUF_S2),
4441 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4443 [PIPE_B] = BIT(DBUF_S1),
4444 [PIPE_C] = BIT(DBUF_S2),
4448 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4450 [PIPE_A] = BIT(DBUF_S1),
4451 [PIPE_B] = BIT(DBUF_S1),
4452 [PIPE_C] = BIT(DBUF_S2),
4459 * Table taken from Bspec 49255
4460 * Pipes do have some preferred DBuf slice affinity,
4461 * plus there are some hardcoded requirements on how
4462 * those should be distributed for multipipe scenarios.
4463 * For more DBuf slices algorithm can get even more messy
4464 * and less readable, so decided to use a table almost
4465 * as is from BSpec itself - that way it is at least easier
4466 * to compare, change and check.
4468 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4469 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4472 .active_pipes = BIT(PIPE_A),
4474 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4478 .active_pipes = BIT(PIPE_B),
4480 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4484 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4486 [PIPE_A] = BIT(DBUF_S2),
4487 [PIPE_B] = BIT(DBUF_S1),
4491 .active_pipes = BIT(PIPE_C),
4493 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4497 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4499 [PIPE_A] = BIT(DBUF_S1),
4500 [PIPE_C] = BIT(DBUF_S2),
4504 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4506 [PIPE_B] = BIT(DBUF_S1),
4507 [PIPE_C] = BIT(DBUF_S2),
4511 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4513 [PIPE_A] = BIT(DBUF_S1),
4514 [PIPE_B] = BIT(DBUF_S1),
4515 [PIPE_C] = BIT(DBUF_S2),
4519 .active_pipes = BIT(PIPE_D),
4521 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4525 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4527 [PIPE_A] = BIT(DBUF_S1),
4528 [PIPE_D] = BIT(DBUF_S2),
4532 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4534 [PIPE_B] = BIT(DBUF_S1),
4535 [PIPE_D] = BIT(DBUF_S2),
4539 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4541 [PIPE_A] = BIT(DBUF_S1),
4542 [PIPE_B] = BIT(DBUF_S1),
4543 [PIPE_D] = BIT(DBUF_S2),
4547 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4549 [PIPE_C] = BIT(DBUF_S1),
4550 [PIPE_D] = BIT(DBUF_S2),
4554 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4556 [PIPE_A] = BIT(DBUF_S1),
4557 [PIPE_C] = BIT(DBUF_S2),
4558 [PIPE_D] = BIT(DBUF_S2),
4562 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4564 [PIPE_B] = BIT(DBUF_S1),
4565 [PIPE_C] = BIT(DBUF_S2),
4566 [PIPE_D] = BIT(DBUF_S2),
4570 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4572 [PIPE_A] = BIT(DBUF_S1),
4573 [PIPE_B] = BIT(DBUF_S1),
4574 [PIPE_C] = BIT(DBUF_S2),
4575 [PIPE_D] = BIT(DBUF_S2),
4581 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
4583 .active_pipes = BIT(PIPE_A),
4585 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4589 .active_pipes = BIT(PIPE_B),
4591 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4595 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4597 [PIPE_A] = BIT(DBUF_S1),
4598 [PIPE_B] = BIT(DBUF_S2),
4602 .active_pipes = BIT(PIPE_C),
4604 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4608 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4610 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4611 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4615 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4617 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4618 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4622 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4624 [PIPE_A] = BIT(DBUF_S1),
4625 [PIPE_B] = BIT(DBUF_S2),
4626 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4630 .active_pipes = BIT(PIPE_D),
4632 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4636 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4638 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4639 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4643 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4645 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4646 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4650 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4652 [PIPE_A] = BIT(DBUF_S1),
4653 [PIPE_B] = BIT(DBUF_S2),
4654 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4658 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4660 [PIPE_C] = BIT(DBUF_S3),
4661 [PIPE_D] = BIT(DBUF_S4),
4665 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4667 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4668 [PIPE_C] = BIT(DBUF_S3),
4669 [PIPE_D] = BIT(DBUF_S4),
4673 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4675 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4676 [PIPE_C] = BIT(DBUF_S3),
4677 [PIPE_D] = BIT(DBUF_S4),
4681 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4683 [PIPE_A] = BIT(DBUF_S1),
4684 [PIPE_B] = BIT(DBUF_S2),
4685 [PIPE_C] = BIT(DBUF_S3),
4686 [PIPE_D] = BIT(DBUF_S4),
4692 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
4694 * Keep the join_mbus cases first so check_mbus_joined()
4695 * will prefer them over the !join_mbus cases.
4698 .active_pipes = BIT(PIPE_A),
4700 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4705 .active_pipes = BIT(PIPE_B),
4707 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4712 .active_pipes = BIT(PIPE_A),
4714 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4719 .active_pipes = BIT(PIPE_B),
4721 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4726 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4728 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4729 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4733 .active_pipes = BIT(PIPE_C),
4735 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4739 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4741 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4742 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4746 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4748 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4749 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4753 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4755 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4756 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4757 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4761 .active_pipes = BIT(PIPE_D),
4763 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4767 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4769 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4770 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4774 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4776 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4777 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4781 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4783 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4784 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4785 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4789 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4791 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4792 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4796 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4798 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4799 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4800 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4804 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4806 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4807 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4808 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4812 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4814 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4815 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4816 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4817 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4824 static bool check_mbus_joined(u8 active_pipes,
4825 const struct dbuf_slice_conf_entry *dbuf_slices)
4829 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
4830 if (dbuf_slices[i].active_pipes == active_pipes)
4831 return dbuf_slices[i].join_mbus;
4836 static bool adlp_check_mbus_joined(u8 active_pipes)
4838 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
4841 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
4842 const struct dbuf_slice_conf_entry *dbuf_slices)
4846 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
4847 if (dbuf_slices[i].active_pipes == active_pipes &&
4848 dbuf_slices[i].join_mbus == join_mbus)
4849 return dbuf_slices[i].dbuf_mask[pipe];
4855 * This function finds an entry with same enabled pipe configuration and
4856 * returns correspondent DBuf slice mask as stated in BSpec for particular
4859 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4862 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4863 * required calculating "pipe ratio" in order to determine
4864 * if one or two slices can be used for single pipe configurations
4865 * as additional constraint to the existing table.
4866 * However based on recent info, it should be not "pipe ratio"
4867 * but rather ratio between pixel_rate and cdclk with additional
4868 * constants, so for now we are using only table until this is
4869 * clarified. Also this is the reason why crtc_state param is
4870 * still here - we will need it once those additional constraints
4873 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4877 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4879 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4883 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4885 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4886 adlp_allowed_dbufs);
4889 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4891 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4895 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
4897 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4898 enum pipe pipe = crtc->pipe;
4900 if (IS_DG2(dev_priv))
4901 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4902 else if (IS_ALDERLAKE_P(dev_priv))
4903 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4904 else if (DISPLAY_VER(dev_priv) == 12)
4905 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4906 else if (DISPLAY_VER(dev_priv) == 11)
4907 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4909 * For anything else just return one slice yet.
4910 * Should be extended for other platforms.
4912 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4916 use_min_ddb(const struct intel_crtc_state *crtc_state,
4917 struct intel_plane *plane)
4919 struct drm_i915_private *i915 = to_i915(plane->base.dev);
4921 return DISPLAY_VER(i915) >= 13 &&
4922 crtc_state->uapi.async_flip &&
4927 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
4928 struct intel_plane *plane)
4930 struct drm_i915_private *i915 = to_i915(plane->base.dev);
4932 return DISPLAY_VER(i915) >= 13 &&
4933 crtc_state->uapi.async_flip &&
4938 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
4939 const struct intel_plane_state *plane_state,
4942 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
4943 const struct drm_framebuffer *fb = plane_state->hw.fb;
4946 if (!plane_state->uapi.visible)
4949 if (plane->id == PLANE_CURSOR)
4953 * We calculate extra ddb based on ratio plane rate/total data rate
4954 * in case, in some cases we should not allocate extra ddb for the plane,
4955 * so do not count its data rate, if this is the case.
4957 if (use_min_ddb(crtc_state, plane))
4960 if (color_plane == 1 &&
4961 !intel_format_info_is_yuv_semiplanar(fb->format, fb->modifier))
4965 * Src coordinates are already rotated by 270 degrees for
4966 * the 90/270 degree plane rotation cases (to match the
4967 * GTT mapping), hence no need to account for rotation here.
4969 width = drm_rect_width(&plane_state->uapi.src) >> 16;
4970 height = drm_rect_height(&plane_state->uapi.src) >> 16;
4972 /* UV plane does 1/2 pixel sub-sampling */
4973 if (color_plane == 1) {
4978 return width * height * fb->format->cpp[color_plane];
4982 skl_get_total_relative_data_rate(struct intel_atomic_state *state,
4983 struct intel_crtc *crtc)
4985 struct intel_crtc_state *crtc_state =
4986 intel_atomic_get_new_crtc_state(state, crtc);
4987 const struct intel_plane_state *plane_state;
4988 struct intel_plane *plane;
4989 u64 total_data_rate = 0;
4990 enum plane_id plane_id;
4993 /* Calculate and cache data rate for each plane */
4994 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
4995 if (plane->pipe != crtc->pipe)
4998 plane_id = plane->id;
5001 crtc_state->plane_data_rate[plane_id] =
5002 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5005 crtc_state->uv_plane_data_rate[plane_id] =
5006 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
5009 for_each_plane_id_on_crtc(crtc, plane_id) {
5010 total_data_rate += crtc_state->plane_data_rate[plane_id];
5011 total_data_rate += crtc_state->uv_plane_data_rate[plane_id];
5014 return total_data_rate;
5018 icl_get_total_relative_data_rate(struct intel_atomic_state *state,
5019 struct intel_crtc *crtc)
5021 struct intel_crtc_state *crtc_state =
5022 intel_atomic_get_new_crtc_state(state, crtc);
5023 const struct intel_plane_state *plane_state;
5024 struct intel_plane *plane;
5025 u64 total_data_rate = 0;
5026 enum plane_id plane_id;
5029 /* Calculate and cache data rate for each plane */
5030 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5031 if (plane->pipe != crtc->pipe)
5034 plane_id = plane->id;
5036 if (!plane_state->planar_linked_plane) {
5037 crtc_state->plane_data_rate[plane_id] =
5038 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5040 enum plane_id y_plane_id;
5043 * The slave plane might not iterate in
5044 * intel_atomic_crtc_state_for_each_plane_state(),
5045 * and needs the master plane state which may be
5046 * NULL if we try get_new_plane_state(), so we
5047 * always calculate from the master.
5049 if (plane_state->planar_slave)
5052 /* Y plane rate is calculated on the slave */
5053 y_plane_id = plane_state->planar_linked_plane->id;
5054 crtc_state->plane_data_rate[y_plane_id] =
5055 skl_plane_relative_data_rate(crtc_state, plane_state, 0);
5057 crtc_state->plane_data_rate[plane_id] =
5058 skl_plane_relative_data_rate(crtc_state, plane_state, 1);
5062 for_each_plane_id_on_crtc(crtc, plane_id)
5063 total_data_rate += crtc_state->plane_data_rate[plane_id];
5065 return total_data_rate;
5068 const struct skl_wm_level *
5069 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
5070 enum plane_id plane_id,
5073 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5075 if (level == 0 && pipe_wm->use_sagv_wm)
5076 return &wm->sagv.wm0;
5078 return &wm->wm[level];
5081 const struct skl_wm_level *
5082 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
5083 enum plane_id plane_id)
5085 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5087 if (pipe_wm->use_sagv_wm)
5088 return &wm->sagv.trans_wm;
5090 return &wm->trans_wm;
5094 * We only disable the watermarks for each plane if
5095 * they exceed the ddb allocation of said plane. This
5096 * is done so that we don't end up touching cursor
5097 * watermarks needlessly when some other plane reduces
5098 * our max possible watermark level.
5100 * Bspec has this to say about the PLANE_WM enable bit:
5101 * "All the watermarks at this level for all enabled
5102 * planes must be enabled before the level will be used."
5103 * So this is actually safe to do.
5106 skl_check_wm_level(struct skl_wm_level *wm, u64 total)
5108 if (wm->min_ddb_alloc > total)
5109 memset(wm, 0, sizeof(*wm));
5113 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
5114 u64 total, u64 uv_total)
5116 if (wm->min_ddb_alloc > total ||
5117 uv_wm->min_ddb_alloc > uv_total) {
5118 memset(wm, 0, sizeof(*wm));
5119 memset(uv_wm, 0, sizeof(*uv_wm));
5123 static bool icl_need_wm1_wa(struct drm_i915_private *i915,
5124 enum plane_id plane_id)
5127 * Wa_1408961008:icl, ehl
5128 * Wa_14012656716:tgl, adl
5129 * Underruns with WM1+ disabled
5131 return DISPLAY_VER(i915) == 11 ||
5132 (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR);
5135 struct skl_plane_ddb_iter {
5137 u16 total[I915_MAX_PLANES];
5138 u16 uv_total[I915_MAX_PLANES];
5143 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
5144 const struct skl_wm_level *wm,
5150 extra = min_t(u16, iter->size,
5151 DIV64_U64_ROUND_UP(iter->size * data_rate,
5153 iter->size -= extra;
5154 iter->data_rate -= data_rate;
5157 return wm->min_ddb_alloc + extra;
5161 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
5162 struct intel_crtc *crtc)
5164 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5165 struct intel_crtc_state *crtc_state =
5166 intel_atomic_get_new_crtc_state(state, crtc);
5167 const struct intel_dbuf_state *dbuf_state =
5168 intel_atomic_get_new_dbuf_state(state);
5169 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
5170 int num_active = hweight8(dbuf_state->active_pipes);
5171 struct skl_plane_ddb_iter iter = {};
5172 enum plane_id plane_id;
5176 /* Clear the partitioning for disabled planes. */
5177 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
5178 memset(crtc_state->wm.skl.plane_ddb_uv, 0, sizeof(crtc_state->wm.skl.plane_ddb_uv));
5180 if (!crtc_state->hw.active)
5183 if (DISPLAY_VER(dev_priv) >= 11)
5184 iter.data_rate = icl_get_total_relative_data_rate(state, crtc);
5186 iter.data_rate = skl_get_total_relative_data_rate(state, crtc);
5188 iter.size = skl_ddb_entry_size(alloc);
5192 /* Allocate fixed number of blocks for cursor. */
5193 iter.total[PLANE_CURSOR] = skl_cursor_allocation(crtc_state, num_active);
5194 iter.size -= iter.total[PLANE_CURSOR];
5195 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb_y[PLANE_CURSOR],
5196 alloc->end - iter.total[PLANE_CURSOR], alloc->end);
5199 * Find the highest watermark level for which we can satisfy the block
5200 * requirement of active planes.
5202 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
5204 for_each_plane_id_on_crtc(crtc, plane_id) {
5205 const struct skl_plane_wm *wm =
5206 &crtc_state->wm.skl.optimal.planes[plane_id];
5208 if (plane_id == PLANE_CURSOR) {
5209 if (wm->wm[level].min_ddb_alloc > iter.total[PLANE_CURSOR]) {
5210 drm_WARN_ON(&dev_priv->drm,
5211 wm->wm[level].min_ddb_alloc != U16_MAX);
5218 blocks += wm->wm[level].min_ddb_alloc;
5219 blocks += wm->uv_wm[level].min_ddb_alloc;
5222 if (blocks <= iter.size) {
5223 iter.size -= blocks;
5229 drm_dbg_kms(&dev_priv->drm,
5230 "Requested display configuration exceeds system DDB limitations");
5231 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
5236 /* avoid the WARN later when we don't allocate any extra DDB */
5237 if (iter.data_rate == 0)
5241 * Grant each plane the blocks it requires at the highest achievable
5242 * watermark level, plus an extra share of the leftover blocks
5243 * proportional to its relative data rate.
5245 for_each_plane_id_on_crtc(crtc, plane_id) {
5246 const struct skl_plane_wm *wm =
5247 &crtc_state->wm.skl.optimal.planes[plane_id];
5249 if (plane_id == PLANE_CURSOR)
5252 iter.total[plane_id] =
5253 skl_allocate_plane_ddb(&iter, &wm->wm[level],
5254 crtc_state->plane_data_rate[plane_id]);
5256 iter.uv_total[plane_id] =
5257 skl_allocate_plane_ddb(&iter, &wm->uv_wm[level],
5258 crtc_state->uv_plane_data_rate[plane_id]);
5260 drm_WARN_ON(&dev_priv->drm, iter.size != 0 || iter.data_rate != 0);
5262 /* Set the actual DDB start/end points for each plane */
5263 iter.start = alloc->start;
5264 for_each_plane_id_on_crtc(crtc, plane_id) {
5265 struct skl_ddb_entry *plane_alloc =
5266 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5267 struct skl_ddb_entry *uv_plane_alloc =
5268 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5270 if (plane_id == PLANE_CURSOR)
5273 /* Gen11+ uses a separate plane for UV watermarks */
5274 drm_WARN_ON(&dev_priv->drm,
5275 DISPLAY_VER(dev_priv) >= 11 && iter.uv_total[plane_id]);
5277 /* Leave disabled planes at (0,0) */
5278 if (iter.total[plane_id])
5279 iter.start = skl_ddb_entry_init(plane_alloc, iter.start,
5280 iter.start + iter.total[plane_id]);
5282 if (iter.uv_total[plane_id])
5283 iter.start = skl_ddb_entry_init(uv_plane_alloc, iter.start,
5284 iter.start + iter.uv_total[plane_id]);
5288 * When we calculated watermark values we didn't know how high
5289 * of a level we'd actually be able to hit, so we just marked
5290 * all levels as "enabled." Go back now and disable the ones
5291 * that aren't actually possible.
5293 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
5294 for_each_plane_id_on_crtc(crtc, plane_id) {
5295 struct skl_plane_wm *wm =
5296 &crtc_state->wm.skl.optimal.planes[plane_id];
5298 skl_check_nv12_wm_level(&wm->wm[level], &wm->uv_wm[level],
5299 iter.total[plane_id],
5300 iter.uv_total[plane_id]);
5302 if (icl_need_wm1_wa(dev_priv, plane_id) &&
5303 level == 1 && wm->wm[0].enable) {
5304 wm->wm[level].blocks = wm->wm[0].blocks;
5305 wm->wm[level].lines = wm->wm[0].lines;
5306 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5312 * Go back and disable the transition and SAGV watermarks
5313 * if it turns out we don't have enough DDB blocks for them.
5315 for_each_plane_id_on_crtc(crtc, plane_id) {
5316 struct skl_plane_wm *wm =
5317 &crtc_state->wm.skl.optimal.planes[plane_id];
5319 skl_check_wm_level(&wm->trans_wm, iter.total[plane_id]);
5320 skl_check_wm_level(&wm->sagv.wm0, iter.total[plane_id]);
5321 skl_check_wm_level(&wm->sagv.trans_wm, iter.total[plane_id]);
5328 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5329 * for the read latency) and cpp should always be <= 8, so that
5330 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5331 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5333 static uint_fixed_16_16_t
5334 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5335 u8 cpp, u32 latency, u32 dbuf_block_size)
5337 u32 wm_intermediate_val;
5338 uint_fixed_16_16_t ret;
5341 return FP_16_16_MAX;
5343 wm_intermediate_val = latency * pixel_rate * cpp;
5344 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5346 if (DISPLAY_VER(dev_priv) >= 10)
5347 ret = add_fixed16_u32(ret, 1);
5352 static uint_fixed_16_16_t
5353 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5354 uint_fixed_16_16_t plane_blocks_per_line)
5356 u32 wm_intermediate_val;
5357 uint_fixed_16_16_t ret;
5360 return FP_16_16_MAX;
5362 wm_intermediate_val = latency * pixel_rate;
5363 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5364 pipe_htotal * 1000);
5365 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5369 static uint_fixed_16_16_t
5370 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5372 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5375 uint_fixed_16_16_t linetime_us;
5377 if (!crtc_state->hw.active)
5378 return u32_to_fixed16(0);
5380 pixel_rate = crtc_state->pixel_rate;
5382 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5383 return u32_to_fixed16(0);
5385 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5386 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5392 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5393 int width, const struct drm_format_info *format,
5394 u64 modifier, unsigned int rotation,
5395 u32 plane_pixel_rate, struct skl_wm_params *wp,
5398 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5399 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5402 /* only planar format has two planes */
5403 if (color_plane == 1 &&
5404 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5405 drm_dbg_kms(&dev_priv->drm,
5406 "Non planar format have single plane\n");
5410 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5411 modifier == I915_FORMAT_MOD_Yf_TILED ||
5412 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5413 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5414 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5415 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5416 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5417 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5420 if (color_plane == 1 && wp->is_planar)
5423 wp->cpp = format->cpp[color_plane];
5424 wp->plane_pixel_rate = plane_pixel_rate;
5426 if (DISPLAY_VER(dev_priv) >= 11 &&
5427 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5428 wp->dbuf_block_size = 256;
5430 wp->dbuf_block_size = 512;
5432 if (drm_rotation_90_or_270(rotation)) {
5435 wp->y_min_scanlines = 16;
5438 wp->y_min_scanlines = 8;
5441 wp->y_min_scanlines = 4;
5444 MISSING_CASE(wp->cpp);
5448 wp->y_min_scanlines = 4;
5451 if (skl_needs_memory_bw_wa(dev_priv))
5452 wp->y_min_scanlines *= 2;
5454 wp->plane_bytes_per_line = wp->width * wp->cpp;
5456 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5457 wp->y_min_scanlines,
5458 wp->dbuf_block_size);
5460 if (DISPLAY_VER(dev_priv) >= 10)
5463 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5464 wp->y_min_scanlines);
5466 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5467 wp->dbuf_block_size);
5469 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5472 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5475 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5476 wp->plane_blocks_per_line);
5478 wp->linetime_us = fixed16_to_u32_round_up(
5479 intel_get_linetime_us(crtc_state));
5485 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5486 const struct intel_plane_state *plane_state,
5487 struct skl_wm_params *wp, int color_plane)
5489 const struct drm_framebuffer *fb = plane_state->hw.fb;
5493 * Src coordinates are already rotated by 270 degrees for
5494 * the 90/270 degree plane rotation cases (to match the
5495 * GTT mapping), hence no need to account for rotation here.
5497 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5499 return skl_compute_wm_params(crtc_state, width,
5500 fb->format, fb->modifier,
5501 plane_state->hw.rotation,
5502 intel_plane_pixel_rate(crtc_state, plane_state),
5506 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5508 if (DISPLAY_VER(dev_priv) >= 10)
5511 /* The number of lines are ignored for the level 0 watermark. */
5515 static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
5517 if (DISPLAY_VER(dev_priv) >= 13)
5523 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5524 struct intel_plane *plane,
5526 unsigned int latency,
5527 const struct skl_wm_params *wp,
5528 const struct skl_wm_level *result_prev,
5529 struct skl_wm_level *result /* out */)
5531 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5532 uint_fixed_16_16_t method1, method2;
5533 uint_fixed_16_16_t selected_result;
5534 u32 blocks, lines, min_ddb_alloc = 0;
5537 (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
5539 result->min_ddb_alloc = U16_MAX;
5544 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5545 * Display WA #1141: kbl,cfl
5547 if ((IS_KABYLAKE(dev_priv) ||
5548 IS_COFFEELAKE(dev_priv) ||
5549 IS_COMETLAKE(dev_priv)) &&
5550 dev_priv->ipc_enabled)
5553 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5556 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5557 wp->cpp, latency, wp->dbuf_block_size);
5558 method2 = skl_wm_method2(wp->plane_pixel_rate,
5559 crtc_state->hw.pipe_mode.crtc_htotal,
5561 wp->plane_blocks_per_line);
5564 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5566 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5567 wp->dbuf_block_size < 1) &&
5568 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5569 selected_result = method2;
5570 } else if (latency >= wp->linetime_us) {
5571 if (DISPLAY_VER(dev_priv) == 9)
5572 selected_result = min_fixed16(method1, method2);
5574 selected_result = method2;
5576 selected_result = method1;
5580 blocks = fixed16_to_u32_round_up(selected_result) + 1;
5581 lines = div_round_up_fixed16(selected_result,
5582 wp->plane_blocks_per_line);
5584 if (DISPLAY_VER(dev_priv) == 9) {
5585 /* Display WA #1125: skl,bxt,kbl */
5586 if (level == 0 && wp->rc_surface)
5587 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5589 /* Display WA #1126: skl,bxt,kbl */
5590 if (level >= 1 && level <= 7) {
5592 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5593 lines += wp->y_min_scanlines;
5599 * Make sure result blocks for higher latency levels are
5600 * atleast as high as level below the current level.
5601 * Assumption in DDB algorithm optimization for special
5602 * cases. Also covers Display WA #1125 for RC.
5604 if (result_prev->blocks > blocks)
5605 blocks = result_prev->blocks;
5609 if (DISPLAY_VER(dev_priv) >= 11) {
5613 if (lines % wp->y_min_scanlines == 0)
5614 extra_lines = wp->y_min_scanlines;
5616 extra_lines = wp->y_min_scanlines * 2 -
5617 lines % wp->y_min_scanlines;
5619 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5620 wp->plane_blocks_per_line);
5622 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5626 if (!skl_wm_has_lines(dev_priv, level))
5629 if (lines > skl_wm_max_lines(dev_priv)) {
5631 result->min_ddb_alloc = U16_MAX;
5636 * If lines is valid, assume we can use this watermark level
5637 * for now. We'll come back and disable it after we calculate the
5638 * DDB allocation if it turns out we don't actually have enough
5639 * blocks to satisfy it.
5641 result->blocks = blocks;
5642 result->lines = lines;
5643 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5644 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5645 result->enable = true;
5647 if (DISPLAY_VER(dev_priv) < 12 && dev_priv->sagv_block_time_us)
5648 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5652 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5653 struct intel_plane *plane,
5654 const struct skl_wm_params *wm_params,
5655 struct skl_wm_level *levels)
5657 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5658 int level, max_level = ilk_wm_max_level(dev_priv);
5659 struct skl_wm_level *result_prev = &levels[0];
5661 for (level = 0; level <= max_level; level++) {
5662 struct skl_wm_level *result = &levels[level];
5663 unsigned int latency = dev_priv->wm.skl_latency[level];
5665 skl_compute_plane_wm(crtc_state, plane, level, latency,
5666 wm_params, result_prev, result);
5668 result_prev = result;
5672 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5673 struct intel_plane *plane,
5674 const struct skl_wm_params *wm_params,
5675 struct skl_plane_wm *plane_wm)
5677 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5678 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5679 struct skl_wm_level *levels = plane_wm->wm;
5680 unsigned int latency = 0;
5682 if (dev_priv->sagv_block_time_us)
5683 latency = dev_priv->sagv_block_time_us + dev_priv->wm.skl_latency[0];
5685 skl_compute_plane_wm(crtc_state, plane, 0, latency,
5686 wm_params, &levels[0],
5690 static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5691 struct skl_wm_level *trans_wm,
5692 const struct skl_wm_level *wm0,
5693 const struct skl_wm_params *wp)
5695 u16 trans_min, trans_amount, trans_y_tile_min;
5696 u16 wm0_blocks, trans_offset, blocks;
5698 /* Transition WM don't make any sense if ipc is disabled */
5699 if (!dev_priv->ipc_enabled)
5703 * WaDisableTWM:skl,kbl,cfl,bxt
5704 * Transition WM are not recommended by HW team for GEN9
5706 if (DISPLAY_VER(dev_priv) == 9)
5709 if (DISPLAY_VER(dev_priv) >= 11)
5714 /* Display WA #1140: glk,cnl */
5715 if (DISPLAY_VER(dev_priv) == 10)
5718 trans_amount = 10; /* This is configurable amount */
5720 trans_offset = trans_min + trans_amount;
5723 * The spec asks for Selected Result Blocks for wm0 (the real value),
5724 * not Result Blocks (the integer value). Pay attention to the capital
5725 * letters. The value wm_l0->blocks is actually Result Blocks, but
5726 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5727 * and since we later will have to get the ceiling of the sum in the
5728 * transition watermarks calculation, we can just pretend Selected
5729 * Result Blocks is Result Blocks minus 1 and it should work for the
5730 * current platforms.
5732 wm0_blocks = wm0->blocks - 1;
5736 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5737 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5739 blocks = wm0_blocks + trans_offset;
5744 * Just assume we can enable the transition watermark. After
5745 * computing the DDB we'll come back and disable it if that
5746 * assumption turns out to be false.
5748 trans_wm->blocks = blocks;
5749 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5750 trans_wm->enable = true;
5753 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5754 const struct intel_plane_state *plane_state,
5755 struct intel_plane *plane, int color_plane)
5757 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5758 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5759 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
5760 struct skl_wm_params wm_params;
5763 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5764 &wm_params, color_plane);
5768 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
5770 skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5771 &wm->wm[0], &wm_params);
5773 if (DISPLAY_VER(dev_priv) >= 12) {
5774 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
5776 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5777 &wm->sagv.wm0, &wm_params);
5783 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5784 const struct intel_plane_state *plane_state,
5785 struct intel_plane *plane)
5787 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
5788 struct skl_wm_params wm_params;
5791 wm->is_planar = true;
5793 /* uv plane watermarks must also be validated for NV12/Planar */
5794 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5799 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
5804 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5805 const struct intel_plane_state *plane_state)
5807 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5808 enum plane_id plane_id = plane->id;
5809 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5810 const struct drm_framebuffer *fb = plane_state->hw.fb;
5813 memset(wm, 0, sizeof(*wm));
5815 if (!intel_wm_plane_visible(crtc_state, plane_state))
5818 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5823 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5824 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5833 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5834 const struct intel_plane_state *plane_state)
5836 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5837 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5838 enum plane_id plane_id = plane->id;
5839 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5842 /* Watermarks calculated in master */
5843 if (plane_state->planar_slave)
5846 memset(wm, 0, sizeof(*wm));
5848 if (plane_state->planar_linked_plane) {
5849 const struct drm_framebuffer *fb = plane_state->hw.fb;
5851 drm_WARN_ON(&dev_priv->drm,
5852 !intel_wm_plane_visible(crtc_state, plane_state));
5853 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5854 fb->format->num_planes == 1);
5856 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5857 plane_state->planar_linked_plane, 0);
5861 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5865 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5866 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5875 static int skl_build_pipe_wm(struct intel_atomic_state *state,
5876 struct intel_crtc *crtc)
5878 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5879 struct intel_crtc_state *crtc_state =
5880 intel_atomic_get_new_crtc_state(state, crtc);
5881 const struct intel_plane_state *plane_state;
5882 struct intel_plane *plane;
5885 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5887 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5888 * instead but we don't populate that correctly for NV12 Y
5889 * planes so for now hack this.
5891 if (plane->pipe != crtc->pipe)
5894 if (DISPLAY_VER(dev_priv) >= 11)
5895 ret = icl_build_plane_wm(crtc_state, plane_state);
5897 ret = skl_build_plane_wm(crtc_state, plane_state);
5902 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5907 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5909 const struct skl_ddb_entry *entry)
5912 intel_de_write_fw(dev_priv, reg,
5913 PLANE_BUF_END(entry->end - 1) |
5914 PLANE_BUF_START(entry->start));
5916 intel_de_write_fw(dev_priv, reg, 0);
5919 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5921 const struct skl_wm_level *level)
5927 if (level->ignore_lines)
5928 val |= PLANE_WM_IGNORE_LINES;
5929 val |= level->blocks;
5930 val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
5932 intel_de_write_fw(dev_priv, reg, val);
5935 void skl_write_plane_wm(struct intel_plane *plane,
5936 const struct intel_crtc_state *crtc_state)
5938 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5939 int level, max_level = ilk_wm_max_level(dev_priv);
5940 enum plane_id plane_id = plane->id;
5941 enum pipe pipe = plane->pipe;
5942 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5943 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5944 const struct skl_ddb_entry *ddb_y =
5945 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5946 const struct skl_ddb_entry *ddb_uv =
5947 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
5949 for (level = 0; level <= max_level; level++)
5950 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5951 skl_plane_wm_level(pipe_wm, plane_id, level));
5953 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5954 skl_plane_trans_wm(pipe_wm, plane_id));
5956 if (HAS_HW_SAGV_WM(dev_priv)) {
5957 skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
5959 skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
5960 &wm->sagv.trans_wm);
5963 if (DISPLAY_VER(dev_priv) >= 11) {
5964 skl_ddb_entry_write(dev_priv,
5965 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5970 swap(ddb_y, ddb_uv);
5972 skl_ddb_entry_write(dev_priv,
5973 PLANE_BUF_CFG(pipe, plane_id), ddb_y);
5974 skl_ddb_entry_write(dev_priv,
5975 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_uv);
5978 void skl_write_cursor_wm(struct intel_plane *plane,
5979 const struct intel_crtc_state *crtc_state)
5981 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5982 int level, max_level = ilk_wm_max_level(dev_priv);
5983 enum plane_id plane_id = plane->id;
5984 enum pipe pipe = plane->pipe;
5985 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5986 const struct skl_ddb_entry *ddb =
5987 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5989 for (level = 0; level <= max_level; level++)
5990 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5991 skl_plane_wm_level(pipe_wm, plane_id, level));
5993 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5994 skl_plane_trans_wm(pipe_wm, plane_id));
5996 if (HAS_HW_SAGV_WM(dev_priv)) {
5997 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5999 skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
6001 skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
6002 &wm->sagv.trans_wm);
6005 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
6008 bool skl_wm_level_equals(const struct skl_wm_level *l1,
6009 const struct skl_wm_level *l2)
6011 return l1->enable == l2->enable &&
6012 l1->ignore_lines == l2->ignore_lines &&
6013 l1->lines == l2->lines &&
6014 l1->blocks == l2->blocks;
6017 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
6018 const struct skl_plane_wm *wm1,
6019 const struct skl_plane_wm *wm2)
6021 int level, max_level = ilk_wm_max_level(dev_priv);
6023 for (level = 0; level <= max_level; level++) {
6025 * We don't check uv_wm as the hardware doesn't actually
6026 * use it. It only gets used for calculating the required
6029 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
6033 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
6034 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
6035 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
6038 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
6039 const struct skl_ddb_entry *b)
6041 return a->start < b->end && b->start < a->end;
6044 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
6045 const struct skl_ddb_entry *b)
6047 if (a->end && b->end) {
6048 a->start = min(a->start, b->start);
6049 a->end = max(a->end, b->end);
6050 } else if (b->end) {
6051 a->start = b->start;
6056 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
6057 const struct skl_ddb_entry *entries,
6058 int num_entries, int ignore_idx)
6062 for (i = 0; i < num_entries; i++) {
6063 if (i != ignore_idx &&
6064 skl_ddb_entries_overlap(ddb, &entries[i]))
6072 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
6073 struct intel_crtc_state *new_crtc_state)
6075 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
6076 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
6077 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6078 struct intel_plane *plane;
6080 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6081 struct intel_plane_state *plane_state;
6082 enum plane_id plane_id = plane->id;
6084 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
6085 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]) &&
6086 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_uv[plane_id],
6087 &new_crtc_state->wm.skl.plane_ddb_uv[plane_id]))
6090 plane_state = intel_atomic_get_plane_state(state, plane);
6091 if (IS_ERR(plane_state))
6092 return PTR_ERR(plane_state);
6094 new_crtc_state->update_planes |= BIT(plane_id);
6100 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
6102 struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
6107 * FIXME: For now we always enable slice S1 as per
6108 * the Bspec display initialization sequence.
6110 enabled_slices = BIT(DBUF_S1);
6112 for_each_pipe(dev_priv, pipe)
6113 enabled_slices |= dbuf_state->slices[pipe];
6115 return enabled_slices;
6119 skl_compute_ddb(struct intel_atomic_state *state)
6121 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6122 const struct intel_dbuf_state *old_dbuf_state;
6123 struct intel_dbuf_state *new_dbuf_state = NULL;
6124 const struct intel_crtc_state *old_crtc_state;
6125 struct intel_crtc_state *new_crtc_state;
6126 struct intel_crtc *crtc;
6129 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6130 new_dbuf_state = intel_atomic_get_dbuf_state(state);
6131 if (IS_ERR(new_dbuf_state))
6132 return PTR_ERR(new_dbuf_state);
6134 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6138 if (!new_dbuf_state)
6141 new_dbuf_state->active_pipes =
6142 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6144 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
6145 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6150 if (IS_ALDERLAKE_P(dev_priv))
6151 new_dbuf_state->joined_mbus =
6152 adlp_check_mbus_joined(new_dbuf_state->active_pipes);
6154 for_each_intel_crtc(&dev_priv->drm, crtc) {
6155 enum pipe pipe = crtc->pipe;
6157 new_dbuf_state->slices[pipe] =
6158 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
6159 new_dbuf_state->joined_mbus);
6161 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
6164 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6169 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
6171 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
6172 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6173 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
6177 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6178 /* TODO: Implement vblank synchronized MBUS joining changes */
6179 ret = intel_modeset_all_pipes(state);
6184 drm_dbg_kms(&dev_priv->drm,
6185 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
6186 old_dbuf_state->enabled_slices,
6187 new_dbuf_state->enabled_slices,
6188 INTEL_INFO(dev_priv)->dbuf.slice_mask,
6189 yesno(old_dbuf_state->joined_mbus),
6190 yesno(new_dbuf_state->joined_mbus));
6193 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6194 enum pipe pipe = crtc->pipe;
6196 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
6198 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
6201 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6206 for_each_intel_crtc(&dev_priv->drm, crtc) {
6207 ret = skl_crtc_allocate_ddb(state, crtc);
6212 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6213 new_crtc_state, i) {
6214 ret = skl_crtc_allocate_plane_ddb(state, crtc);
6218 ret = skl_ddb_add_affected_planes(old_crtc_state,
6227 static char enast(bool enable)
6229 return enable ? '*' : ' ';
6233 skl_print_wm_changes(struct intel_atomic_state *state)
6235 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6236 const struct intel_crtc_state *old_crtc_state;
6237 const struct intel_crtc_state *new_crtc_state;
6238 struct intel_plane *plane;
6239 struct intel_crtc *crtc;
6242 if (!drm_debug_enabled(DRM_UT_KMS))
6245 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6246 new_crtc_state, i) {
6247 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
6249 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
6250 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
6252 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6253 enum plane_id plane_id = plane->id;
6254 const struct skl_ddb_entry *old, *new;
6256 old = &old_crtc_state->wm.skl.plane_ddb_y[plane_id];
6257 new = &new_crtc_state->wm.skl.plane_ddb_y[plane_id];
6259 if (skl_ddb_entry_equal(old, new))
6262 drm_dbg_kms(&dev_priv->drm,
6263 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
6264 plane->base.base.id, plane->base.name,
6265 old->start, old->end, new->start, new->end,
6266 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
6269 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6270 enum plane_id plane_id = plane->id;
6271 const struct skl_plane_wm *old_wm, *new_wm;
6273 old_wm = &old_pipe_wm->planes[plane_id];
6274 new_wm = &new_pipe_wm->planes[plane_id];
6276 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
6279 drm_dbg_kms(&dev_priv->drm,
6280 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
6281 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
6282 plane->base.base.id, plane->base.name,
6283 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
6284 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
6285 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
6286 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
6287 enast(old_wm->trans_wm.enable),
6288 enast(old_wm->sagv.wm0.enable),
6289 enast(old_wm->sagv.trans_wm.enable),
6290 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
6291 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
6292 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
6293 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
6294 enast(new_wm->trans_wm.enable),
6295 enast(new_wm->sagv.wm0.enable),
6296 enast(new_wm->sagv.trans_wm.enable));
6298 drm_dbg_kms(&dev_priv->drm,
6299 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
6300 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
6301 plane->base.base.id, plane->base.name,
6302 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
6303 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
6304 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
6305 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
6306 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
6307 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
6308 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
6309 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
6310 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
6311 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
6312 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
6313 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
6314 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
6315 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
6316 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
6317 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
6318 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
6319 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
6320 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
6321 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
6322 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
6323 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
6325 drm_dbg_kms(&dev_priv->drm,
6326 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6327 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6328 plane->base.base.id, plane->base.name,
6329 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
6330 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
6331 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
6332 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
6333 old_wm->trans_wm.blocks,
6334 old_wm->sagv.wm0.blocks,
6335 old_wm->sagv.trans_wm.blocks,
6336 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
6337 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
6338 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
6339 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
6340 new_wm->trans_wm.blocks,
6341 new_wm->sagv.wm0.blocks,
6342 new_wm->sagv.trans_wm.blocks);
6344 drm_dbg_kms(&dev_priv->drm,
6345 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6346 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6347 plane->base.base.id, plane->base.name,
6348 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
6349 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
6350 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
6351 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
6352 old_wm->trans_wm.min_ddb_alloc,
6353 old_wm->sagv.wm0.min_ddb_alloc,
6354 old_wm->sagv.trans_wm.min_ddb_alloc,
6355 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
6356 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
6357 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
6358 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6359 new_wm->trans_wm.min_ddb_alloc,
6360 new_wm->sagv.wm0.min_ddb_alloc,
6361 new_wm->sagv.trans_wm.min_ddb_alloc);
6366 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6367 const struct skl_pipe_wm *old_pipe_wm,
6368 const struct skl_pipe_wm *new_pipe_wm)
6370 struct drm_i915_private *i915 = to_i915(plane->base.dev);
6371 int level, max_level = ilk_wm_max_level(i915);
6373 for (level = 0; level <= max_level; level++) {
6375 * We don't check uv_wm as the hardware doesn't actually
6376 * use it. It only gets used for calculating the required
6379 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6380 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6384 if (HAS_HW_SAGV_WM(i915)) {
6385 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
6386 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
6388 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
6389 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
6393 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6394 skl_plane_trans_wm(new_pipe_wm, plane->id));
6398 * To make sure the cursor watermark registers are always consistent
6399 * with our computed state the following scenario needs special
6403 * 2. move cursor entirely offscreen
6406 * Step 2. does call .disable_plane() but does not zero the watermarks
6407 * (since we consider an offscreen cursor still active for the purposes
6408 * of watermarks). Step 3. would not normally call .disable_plane()
6409 * because the actual plane visibility isn't changing, and we don't
6410 * deallocate the cursor ddb until the pipe gets disabled. So we must
6411 * force step 3. to call .disable_plane() to update the watermark
6412 * registers properly.
6414 * Other planes do not suffer from this issues as their watermarks are
6415 * calculated based on the actual plane visibility. The only time this
6416 * can trigger for the other planes is during the initial readout as the
6417 * default value of the watermarks registers is not zero.
6419 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6420 struct intel_crtc *crtc)
6422 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6423 const struct intel_crtc_state *old_crtc_state =
6424 intel_atomic_get_old_crtc_state(state, crtc);
6425 struct intel_crtc_state *new_crtc_state =
6426 intel_atomic_get_new_crtc_state(state, crtc);
6427 struct intel_plane *plane;
6429 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6430 struct intel_plane_state *plane_state;
6431 enum plane_id plane_id = plane->id;
6434 * Force a full wm update for every plane on modeset.
6435 * Required because the reset value of the wm registers
6436 * is non-zero, whereas we want all disabled planes to
6437 * have zero watermarks. So if we turn off the relevant
6438 * power well the hardware state will go out of sync
6439 * with the software state.
6441 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6442 skl_plane_selected_wm_equals(plane,
6443 &old_crtc_state->wm.skl.optimal,
6444 &new_crtc_state->wm.skl.optimal))
6447 plane_state = intel_atomic_get_plane_state(state, plane);
6448 if (IS_ERR(plane_state))
6449 return PTR_ERR(plane_state);
6451 new_crtc_state->update_planes |= BIT(plane_id);
6458 skl_compute_wm(struct intel_atomic_state *state)
6460 struct intel_crtc *crtc;
6461 struct intel_crtc_state *new_crtc_state;
6464 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6465 ret = skl_build_pipe_wm(state, crtc);
6470 ret = skl_compute_ddb(state);
6474 ret = intel_compute_sagv_mask(state);
6479 * skl_compute_ddb() will have adjusted the final watermarks
6480 * based on how much ddb is available. Now we can actually
6481 * check if the final watermarks changed.
6483 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6484 ret = skl_wm_add_affected_planes(state, crtc);
6489 skl_print_wm_changes(state);
6494 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6495 struct intel_wm_config *config)
6497 struct intel_crtc *crtc;
6499 /* Compute the currently _active_ config */
6500 for_each_intel_crtc(&dev_priv->drm, crtc) {
6501 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6503 if (!wm->pipe_enabled)
6506 config->sprites_enabled |= wm->sprites_enabled;
6507 config->sprites_scaled |= wm->sprites_scaled;
6508 config->num_pipes_active++;
6512 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6514 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6515 struct ilk_wm_maximums max;
6516 struct intel_wm_config config = {};
6517 struct ilk_wm_values results = {};
6518 enum intel_ddb_partitioning partitioning;
6520 ilk_compute_wm_config(dev_priv, &config);
6522 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6523 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6525 /* 5/6 split only in single pipe config on IVB+ */
6526 if (DISPLAY_VER(dev_priv) >= 7 &&
6527 config.num_pipes_active == 1 && config.sprites_enabled) {
6528 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6529 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6531 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6533 best_lp_wm = &lp_wm_1_2;
6536 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6537 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6539 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6541 ilk_write_wm_values(dev_priv, &results);
6544 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6545 struct intel_crtc *crtc)
6547 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6548 const struct intel_crtc_state *crtc_state =
6549 intel_atomic_get_new_crtc_state(state, crtc);
6551 mutex_lock(&dev_priv->wm.wm_mutex);
6552 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6553 ilk_program_watermarks(dev_priv);
6554 mutex_unlock(&dev_priv->wm.wm_mutex);
6557 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6558 struct intel_crtc *crtc)
6560 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6561 const struct intel_crtc_state *crtc_state =
6562 intel_atomic_get_new_crtc_state(state, crtc);
6564 if (!crtc_state->wm.need_postvbl_update)
6567 mutex_lock(&dev_priv->wm.wm_mutex);
6568 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6569 ilk_program_watermarks(dev_priv);
6570 mutex_unlock(&dev_priv->wm.wm_mutex);
6573 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6575 level->enable = val & PLANE_WM_EN;
6576 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6577 level->blocks = val & PLANE_WM_BLOCKS_MASK;
6578 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
6581 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6582 struct skl_pipe_wm *out)
6584 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6585 enum pipe pipe = crtc->pipe;
6586 int level, max_level;
6587 enum plane_id plane_id;
6590 max_level = ilk_wm_max_level(dev_priv);
6592 for_each_plane_id_on_crtc(crtc, plane_id) {
6593 struct skl_plane_wm *wm = &out->planes[plane_id];
6595 for (level = 0; level <= max_level; level++) {
6596 if (plane_id != PLANE_CURSOR)
6597 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6599 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6601 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6604 if (plane_id != PLANE_CURSOR)
6605 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6607 val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6609 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6611 if (HAS_HW_SAGV_WM(dev_priv)) {
6612 if (plane_id != PLANE_CURSOR)
6613 val = intel_uncore_read(&dev_priv->uncore,
6614 PLANE_WM_SAGV(pipe, plane_id));
6616 val = intel_uncore_read(&dev_priv->uncore,
6619 skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
6621 if (plane_id != PLANE_CURSOR)
6622 val = intel_uncore_read(&dev_priv->uncore,
6623 PLANE_WM_SAGV_TRANS(pipe, plane_id));
6625 val = intel_uncore_read(&dev_priv->uncore,
6626 CUR_WM_SAGV_TRANS(pipe));
6628 skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
6629 } else if (DISPLAY_VER(dev_priv) >= 12) {
6630 wm->sagv.wm0 = wm->wm[0];
6631 wm->sagv.trans_wm = wm->trans_wm;
6636 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6638 struct intel_dbuf_state *dbuf_state =
6639 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6640 struct intel_crtc *crtc;
6642 if (IS_ALDERLAKE_P(dev_priv))
6643 dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
6645 for_each_intel_crtc(&dev_priv->drm, crtc) {
6646 struct intel_crtc_state *crtc_state =
6647 to_intel_crtc_state(crtc->base.state);
6648 enum pipe pipe = crtc->pipe;
6649 unsigned int mbus_offset;
6650 enum plane_id plane_id;
6653 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6654 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6656 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6658 for_each_plane_id_on_crtc(crtc, plane_id) {
6659 struct skl_ddb_entry *ddb_y =
6660 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6661 struct skl_ddb_entry *ddb_uv =
6662 &crtc_state->wm.skl.plane_ddb_uv[plane_id];
6664 skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6665 plane_id, ddb_y, ddb_uv);
6667 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6668 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_uv);
6671 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6674 * Used for checking overlaps, so we need absolute
6675 * offsets instead of MBUS relative offsets.
6677 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
6678 dbuf_state->joined_mbus);
6679 mbus_offset = mbus_ddb_offset(dev_priv, slices);
6680 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
6681 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
6683 /* The slices actually used by the planes on the pipe */
6684 dbuf_state->slices[pipe] =
6685 skl_ddb_dbuf_slice_mask(dev_priv, &crtc_state->wm.skl.ddb);
6687 drm_dbg_kms(&dev_priv->drm,
6688 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
6689 crtc->base.base.id, crtc->base.name,
6690 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6691 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
6692 yesno(dbuf_state->joined_mbus));
6695 dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6698 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
6700 const struct intel_dbuf_state *dbuf_state =
6701 to_intel_dbuf_state(i915->dbuf.obj.state);
6702 struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
6703 struct intel_crtc *crtc;
6705 for_each_intel_crtc(&i915->drm, crtc) {
6706 const struct intel_crtc_state *crtc_state =
6707 to_intel_crtc_state(crtc->base.state);
6709 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
6712 for_each_intel_crtc(&i915->drm, crtc) {
6713 const struct intel_crtc_state *crtc_state =
6714 to_intel_crtc_state(crtc->base.state);
6717 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
6718 dbuf_state->joined_mbus);
6719 if (dbuf_state->slices[crtc->pipe] & ~slices)
6722 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
6723 I915_MAX_PIPES, crtc->pipe))
6730 void skl_wm_sanitize(struct drm_i915_private *i915)
6732 struct intel_crtc *crtc;
6735 * On TGL/RKL (at least) the BIOS likes to assign the planes
6736 * to the wrong DBUF slices. This will cause an infinite loop
6737 * in skl_commit_modeset_enables() as it can't find a way to
6738 * transition between the old bogus DBUF layout to the new
6739 * proper DBUF layout without DBUF allocation overlaps between
6740 * the planes (which cannot be allowed or else the hardware
6741 * may hang). If we detect a bogus DBUF layout just turn off
6742 * all the planes so that skl_commit_modeset_enables() can
6743 * simply ignore them.
6745 if (!skl_dbuf_is_misconfigured(i915))
6748 drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
6750 for_each_intel_crtc(&i915->drm, crtc) {
6751 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6752 const struct intel_plane_state *plane_state =
6753 to_intel_plane_state(plane->base.state);
6754 struct intel_crtc_state *crtc_state =
6755 to_intel_crtc_state(crtc->base.state);
6757 if (plane_state->uapi.visible)
6758 intel_plane_disable_noatomic(crtc, plane);
6760 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
6762 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
6766 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6768 struct drm_device *dev = crtc->base.dev;
6769 struct drm_i915_private *dev_priv = to_i915(dev);
6770 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6771 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6772 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6773 enum pipe pipe = crtc->pipe;
6775 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6777 memset(active, 0, sizeof(*active));
6779 active->pipe_enabled = crtc->active;
6781 if (active->pipe_enabled) {
6782 u32 tmp = hw->wm_pipe[pipe];
6785 * For active pipes LP0 watermark is marked as
6786 * enabled, and LP1+ watermaks as disabled since
6787 * we can't really reverse compute them in case
6788 * multiple pipes are active.
6790 active->wm[0].enable = true;
6791 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
6792 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
6793 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
6795 int level, max_level = ilk_wm_max_level(dev_priv);
6798 * For inactive pipes, all watermark levels
6799 * should be marked as enabled but zeroed,
6800 * which is what we'd compute them to.
6802 for (level = 0; level <= max_level; level++)
6803 active->wm[level].enable = true;
6806 crtc->wm.active.ilk = *active;
6809 #define _FW_WM(value, plane) \
6810 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6811 #define _FW_WM_VLV(value, plane) \
6812 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6814 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6815 struct g4x_wm_values *wm)
6819 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6820 wm->sr.plane = _FW_WM(tmp, SR);
6821 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6822 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6823 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6825 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6826 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6827 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6828 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6829 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6830 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6831 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6833 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6834 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6835 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6836 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6837 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6840 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6841 struct vlv_wm_values *wm)
6846 for_each_pipe(dev_priv, pipe) {
6847 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6849 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6850 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6851 wm->ddl[pipe].plane[PLANE_CURSOR] =
6852 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6853 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6854 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6855 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6856 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6859 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6860 wm->sr.plane = _FW_WM(tmp, SR);
6861 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6862 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6863 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6865 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6866 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6867 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6868 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6870 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6871 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6873 if (IS_CHERRYVIEW(dev_priv)) {
6874 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6875 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6876 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6878 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6879 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6880 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6882 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6883 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6884 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6886 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6887 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6888 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6889 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6890 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6891 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6892 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6893 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6894 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6895 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6896 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6898 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6899 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6900 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6902 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6903 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6904 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6905 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6906 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6907 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6908 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6909 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6916 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6918 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6919 struct intel_crtc *crtc;
6921 g4x_read_wm_values(dev_priv, wm);
6923 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6925 for_each_intel_crtc(&dev_priv->drm, crtc) {
6926 struct intel_crtc_state *crtc_state =
6927 to_intel_crtc_state(crtc->base.state);
6928 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6929 struct g4x_pipe_wm *raw;
6930 enum pipe pipe = crtc->pipe;
6931 enum plane_id plane_id;
6932 int level, max_level;
6934 active->cxsr = wm->cxsr;
6935 active->hpll_en = wm->hpll_en;
6936 active->fbc_en = wm->fbc_en;
6938 active->sr = wm->sr;
6939 active->hpll = wm->hpll;
6941 for_each_plane_id_on_crtc(crtc, plane_id) {
6942 active->wm.plane[plane_id] =
6943 wm->pipe[pipe].plane[plane_id];
6946 if (wm->cxsr && wm->hpll_en)
6947 max_level = G4X_WM_LEVEL_HPLL;
6949 max_level = G4X_WM_LEVEL_SR;
6951 max_level = G4X_WM_LEVEL_NORMAL;
6953 level = G4X_WM_LEVEL_NORMAL;
6954 raw = &crtc_state->wm.g4x.raw[level];
6955 for_each_plane_id_on_crtc(crtc, plane_id)
6956 raw->plane[plane_id] = active->wm.plane[plane_id];
6958 level = G4X_WM_LEVEL_SR;
6959 if (level > max_level)
6962 raw = &crtc_state->wm.g4x.raw[level];
6963 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6964 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6965 raw->plane[PLANE_SPRITE0] = 0;
6966 raw->fbc = active->sr.fbc;
6968 level = G4X_WM_LEVEL_HPLL;
6969 if (level > max_level)
6972 raw = &crtc_state->wm.g4x.raw[level];
6973 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6974 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6975 raw->plane[PLANE_SPRITE0] = 0;
6976 raw->fbc = active->hpll.fbc;
6980 for_each_plane_id_on_crtc(crtc, plane_id)
6981 g4x_raw_plane_wm_set(crtc_state, level,
6982 plane_id, USHRT_MAX);
6983 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6985 crtc_state->wm.g4x.optimal = *active;
6986 crtc_state->wm.g4x.intermediate = *active;
6988 drm_dbg_kms(&dev_priv->drm,
6989 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6991 wm->pipe[pipe].plane[PLANE_PRIMARY],
6992 wm->pipe[pipe].plane[PLANE_CURSOR],
6993 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6996 drm_dbg_kms(&dev_priv->drm,
6997 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6998 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6999 drm_dbg_kms(&dev_priv->drm,
7000 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
7001 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
7002 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
7003 yesno(wm->cxsr), yesno(wm->hpll_en), yesno(wm->fbc_en));
7006 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
7008 struct intel_plane *plane;
7009 struct intel_crtc *crtc;
7011 mutex_lock(&dev_priv->wm.wm_mutex);
7013 for_each_intel_plane(&dev_priv->drm, plane) {
7014 struct intel_crtc *crtc =
7015 intel_crtc_for_pipe(dev_priv, plane->pipe);
7016 struct intel_crtc_state *crtc_state =
7017 to_intel_crtc_state(crtc->base.state);
7018 struct intel_plane_state *plane_state =
7019 to_intel_plane_state(plane->base.state);
7020 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
7021 enum plane_id plane_id = plane->id;
7024 if (plane_state->uapi.visible)
7027 for (level = 0; level < 3; level++) {
7028 struct g4x_pipe_wm *raw =
7029 &crtc_state->wm.g4x.raw[level];
7031 raw->plane[plane_id] = 0;
7032 wm_state->wm.plane[plane_id] = 0;
7035 if (plane_id == PLANE_PRIMARY) {
7036 for (level = 0; level < 3; level++) {
7037 struct g4x_pipe_wm *raw =
7038 &crtc_state->wm.g4x.raw[level];
7042 wm_state->sr.fbc = 0;
7043 wm_state->hpll.fbc = 0;
7044 wm_state->fbc_en = false;
7048 for_each_intel_crtc(&dev_priv->drm, crtc) {
7049 struct intel_crtc_state *crtc_state =
7050 to_intel_crtc_state(crtc->base.state);
7052 crtc_state->wm.g4x.intermediate =
7053 crtc_state->wm.g4x.optimal;
7054 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
7057 g4x_program_watermarks(dev_priv);
7059 mutex_unlock(&dev_priv->wm.wm_mutex);
7062 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
7064 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
7065 struct intel_crtc *crtc;
7068 vlv_read_wm_values(dev_priv, wm);
7070 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
7071 wm->level = VLV_WM_LEVEL_PM2;
7073 if (IS_CHERRYVIEW(dev_priv)) {
7074 vlv_punit_get(dev_priv);
7076 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
7077 if (val & DSP_MAXFIFO_PM5_ENABLE)
7078 wm->level = VLV_WM_LEVEL_PM5;
7081 * If DDR DVFS is disabled in the BIOS, Punit
7082 * will never ack the request. So if that happens
7083 * assume we don't have to enable/disable DDR DVFS
7084 * dynamically. To test that just set the REQ_ACK
7085 * bit to poke the Punit, but don't change the
7086 * HIGH/LOW bits so that we don't actually change
7087 * the current state.
7089 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
7090 val |= FORCE_DDR_FREQ_REQ_ACK;
7091 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
7093 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
7094 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
7095 drm_dbg_kms(&dev_priv->drm,
7096 "Punit not acking DDR DVFS request, "
7097 "assuming DDR DVFS is disabled\n");
7098 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
7100 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
7101 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
7102 wm->level = VLV_WM_LEVEL_DDR_DVFS;
7105 vlv_punit_put(dev_priv);
7108 for_each_intel_crtc(&dev_priv->drm, crtc) {
7109 struct intel_crtc_state *crtc_state =
7110 to_intel_crtc_state(crtc->base.state);
7111 struct vlv_wm_state *active = &crtc->wm.active.vlv;
7112 const struct vlv_fifo_state *fifo_state =
7113 &crtc_state->wm.vlv.fifo_state;
7114 enum pipe pipe = crtc->pipe;
7115 enum plane_id plane_id;
7118 vlv_get_fifo_size(crtc_state);
7120 active->num_levels = wm->level + 1;
7121 active->cxsr = wm->cxsr;
7123 for (level = 0; level < active->num_levels; level++) {
7124 struct g4x_pipe_wm *raw =
7125 &crtc_state->wm.vlv.raw[level];
7127 active->sr[level].plane = wm->sr.plane;
7128 active->sr[level].cursor = wm->sr.cursor;
7130 for_each_plane_id_on_crtc(crtc, plane_id) {
7131 active->wm[level].plane[plane_id] =
7132 wm->pipe[pipe].plane[plane_id];
7134 raw->plane[plane_id] =
7135 vlv_invert_wm_value(active->wm[level].plane[plane_id],
7136 fifo_state->plane[plane_id]);
7140 for_each_plane_id_on_crtc(crtc, plane_id)
7141 vlv_raw_plane_wm_set(crtc_state, level,
7142 plane_id, USHRT_MAX);
7143 vlv_invalidate_wms(crtc, active, level);
7145 crtc_state->wm.vlv.optimal = *active;
7146 crtc_state->wm.vlv.intermediate = *active;
7148 drm_dbg_kms(&dev_priv->drm,
7149 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
7151 wm->pipe[pipe].plane[PLANE_PRIMARY],
7152 wm->pipe[pipe].plane[PLANE_CURSOR],
7153 wm->pipe[pipe].plane[PLANE_SPRITE0],
7154 wm->pipe[pipe].plane[PLANE_SPRITE1]);
7157 drm_dbg_kms(&dev_priv->drm,
7158 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
7159 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
7162 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
7164 struct intel_plane *plane;
7165 struct intel_crtc *crtc;
7167 mutex_lock(&dev_priv->wm.wm_mutex);
7169 for_each_intel_plane(&dev_priv->drm, plane) {
7170 struct intel_crtc *crtc =
7171 intel_crtc_for_pipe(dev_priv, plane->pipe);
7172 struct intel_crtc_state *crtc_state =
7173 to_intel_crtc_state(crtc->base.state);
7174 struct intel_plane_state *plane_state =
7175 to_intel_plane_state(plane->base.state);
7176 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
7177 const struct vlv_fifo_state *fifo_state =
7178 &crtc_state->wm.vlv.fifo_state;
7179 enum plane_id plane_id = plane->id;
7182 if (plane_state->uapi.visible)
7185 for (level = 0; level < wm_state->num_levels; level++) {
7186 struct g4x_pipe_wm *raw =
7187 &crtc_state->wm.vlv.raw[level];
7189 raw->plane[plane_id] = 0;
7191 wm_state->wm[level].plane[plane_id] =
7192 vlv_invert_wm_value(raw->plane[plane_id],
7193 fifo_state->plane[plane_id]);
7197 for_each_intel_crtc(&dev_priv->drm, crtc) {
7198 struct intel_crtc_state *crtc_state =
7199 to_intel_crtc_state(crtc->base.state);
7201 crtc_state->wm.vlv.intermediate =
7202 crtc_state->wm.vlv.optimal;
7203 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
7206 vlv_program_watermarks(dev_priv);
7208 mutex_unlock(&dev_priv->wm.wm_mutex);
7212 * FIXME should probably kill this and improve
7213 * the real watermark readout/sanitation instead
7215 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
7217 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM_LP_ENABLE);
7218 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM_LP_ENABLE);
7219 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM_LP_ENABLE);
7222 * Don't touch WM_LP_SPRITE_ENABLE here.
7223 * Doing so could cause underruns.
7227 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
7229 struct ilk_wm_values *hw = &dev_priv->wm.hw;
7230 struct intel_crtc *crtc;
7232 ilk_init_lp_watermarks(dev_priv);
7234 for_each_intel_crtc(&dev_priv->drm, crtc)
7235 ilk_pipe_wm_get_hw_state(crtc);
7237 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
7238 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
7239 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
7241 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
7242 if (DISPLAY_VER(dev_priv) >= 7) {
7243 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
7244 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
7247 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7248 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
7249 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7250 else if (IS_IVYBRIDGE(dev_priv))
7251 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
7252 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7255 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
7258 void intel_enable_ipc(struct drm_i915_private *dev_priv)
7262 if (!HAS_IPC(dev_priv))
7265 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
7267 if (dev_priv->ipc_enabled)
7268 val |= DISP_IPC_ENABLE;
7270 val &= ~DISP_IPC_ENABLE;
7272 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
7275 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
7277 /* Display WA #0477 WaDisableIPC: skl */
7278 if (IS_SKYLAKE(dev_priv))
7281 /* Display WA #1141: SKL:all KBL:all CFL */
7282 if (IS_KABYLAKE(dev_priv) ||
7283 IS_COFFEELAKE(dev_priv) ||
7284 IS_COMETLAKE(dev_priv))
7285 return dev_priv->dram_info.symmetric_memory;
7290 void intel_init_ipc(struct drm_i915_private *dev_priv)
7292 if (!HAS_IPC(dev_priv))
7295 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
7297 intel_enable_ipc(dev_priv);
7300 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
7303 * On Ibex Peak and Cougar Point, we need to disable clock
7304 * gating for the panel power sequencer or it will fail to
7305 * start up when no ports are active.
7307 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7310 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7314 for_each_pipe(dev_priv, pipe) {
7315 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
7316 intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
7317 DISP_TRICKLE_FEED_DISABLE);
7319 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
7320 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
7324 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
7326 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7330 * WaFbcDisableDpfcClockGating:ilk
7332 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
7333 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
7334 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7336 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
7337 MARIUNIT_CLOCK_GATE_DISABLE |
7338 SVSMUNIT_CLOCK_GATE_DISABLE);
7339 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
7340 VFMUNIT_CLOCK_GATE_DISABLE);
7343 * According to the spec the following bits should be set in
7344 * order to enable memory self-refresh
7345 * The bit 22/21 of 0x42004
7346 * The bit 5 of 0x42020
7347 * The bit 15 of 0x45000
7349 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7350 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7351 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
7352 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7353 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
7354 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7358 * Based on the document from hardware guys the following bits
7359 * should be set unconditionally in order to enable FBC.
7360 * The bit 22 of 0x42000
7361 * The bit 22 of 0x42004
7362 * The bit 7,8,9 of 0x42020.
7364 if (IS_IRONLAKE_M(dev_priv)) {
7365 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
7366 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7367 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7369 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7370 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7374 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7376 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7377 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7378 ILK_ELPIN_409_SELECT);
7380 g4x_disable_trickle_feed(dev_priv);
7382 ibx_init_clock_gating(dev_priv);
7385 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7391 * On Ibex Peak and Cougar Point, we need to disable clock
7392 * gating for the panel power sequencer or it will fail to
7393 * start up when no ports are active.
7395 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7396 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
7397 PCH_CPUNIT_CLOCK_GATE_DISABLE);
7398 intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
7399 DPLS_EDP_PPS_FIX_DIS);
7400 /* The below fixes the weird display corruption, a few pixels shifted
7401 * downward, on (only) LVDS of some HP laptops with IVY.
7403 for_each_pipe(dev_priv, pipe) {
7404 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
7405 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
7406 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7407 if (dev_priv->vbt.fdi_rx_polarity_inverted)
7408 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7409 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
7410 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7411 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
7413 /* WADP0ClockGatingDisable */
7414 for_each_pipe(dev_priv, pipe) {
7415 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
7416 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7420 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7424 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
7425 if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
7426 drm_dbg_kms(&dev_priv->drm,
7427 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
7431 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7433 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7435 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7437 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7438 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7439 ILK_ELPIN_409_SELECT);
7441 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7442 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7443 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7444 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7446 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7447 * gating disable must be set. Failure to set it results in
7448 * flickering pixels due to Z write ordering failures after
7449 * some amount of runtime in the Mesa "fire" demo, and Unigine
7450 * Sanctuary and Tropics, and apparently anything else with
7451 * alpha test or pixel discard.
7453 * According to the spec, bit 11 (RCCUNIT) must also be set,
7454 * but we didn't debug actual testcases to find it out.
7456 * WaDisableRCCUnitClockGating:snb
7457 * WaDisableRCPBUnitClockGating:snb
7459 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7460 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7461 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7464 * According to the spec the following bits should be
7465 * set in order to enable memory self-refresh and fbc:
7466 * The bit21 and bit22 of 0x42000
7467 * The bit21 and bit22 of 0x42004
7468 * The bit5 and bit7 of 0x42020
7469 * The bit14 of 0x70180
7470 * The bit14 of 0x71180
7472 * WaFbcAsynchFlipDisableFbcQueue:snb
7474 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7475 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7476 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7477 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7478 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7479 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7480 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7481 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7482 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7483 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7485 g4x_disable_trickle_feed(dev_priv);
7487 cpt_init_clock_gating(dev_priv);
7489 gen6_check_mch_setup(dev_priv);
7492 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7495 * TODO: this bit should only be enabled when really needed, then
7496 * disabled when not needed anymore in order to save power.
7498 if (HAS_PCH_LPT_LP(dev_priv))
7499 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7500 intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7501 PCH_LP_PARTITION_LEVEL_DISABLE);
7503 /* WADPOClockGatingDisable:hsw */
7504 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7505 intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7506 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7509 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7511 if (HAS_PCH_LPT_LP(dev_priv)) {
7512 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7514 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7515 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7519 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7520 int general_prio_credits,
7521 int high_prio_credits)
7526 /* WaTempDisableDOPClkGating:bdw */
7527 misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7528 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7530 val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7531 val &= ~L3_PRIO_CREDITS_MASK;
7532 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7533 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7534 intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7537 * Wait at least 100 clocks before re-enabling clock gating.
7538 * See the definition of L3SQCREG1 in BSpec.
7540 intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7542 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7545 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7547 /* Wa_1409120013:icl,ehl */
7548 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7549 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7551 /*Wa_14010594013:icl, ehl */
7552 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7553 0, ICL_DELAY_PMRSP);
7556 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7558 /* Wa_1409120013:tgl,rkl,adl-s,dg1,dg2 */
7559 if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) ||
7560 IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv) || IS_DG2(dev_priv))
7561 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7562 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7564 /* Wa_1409825376:tgl (pre-prod)*/
7565 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
7566 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7567 TGL_VRH_GATING_DIS);
7569 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
7570 if (DISPLAY_VER(dev_priv) == 12)
7571 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
7572 CLKREQ_POLICY_MEM_UP_OVRD, 0);
7575 static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
7577 gen12lp_init_clock_gating(dev_priv);
7579 /* Wa_22011091694:adlp */
7580 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
7583 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7585 gen12lp_init_clock_gating(dev_priv);
7587 /* Wa_1409836686:dg1[a0] */
7588 if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7589 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7593 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
7595 /* Wa_22010146351:xehpsdv */
7596 if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7597 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
7600 static void dg2_init_clock_gating(struct drm_i915_private *i915)
7602 /* Wa_22010954014:dg2_g10 */
7603 if (IS_DG2_G10(i915))
7604 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7608 * Wa_14010733611:dg2_g10
7609 * Wa_22010146351:dg2_g10
7611 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
7612 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7613 SGR_DIS | SGGI_DIS);
7616 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7618 if (!HAS_PCH_CNP(dev_priv))
7621 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7622 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7623 CNP_PWM_CGE_GATING_DISABLE);
7626 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7628 cnp_init_clock_gating(dev_priv);
7629 gen9_init_clock_gating(dev_priv);
7631 /* WAC6entrylatency:cfl */
7632 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7633 FBC_LLC_FULLY_OPEN);
7636 * WaFbcTurnOffFbcWatermark:cfl
7637 * Display WA #0562: cfl
7639 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7643 * WaFbcNukeOnHostModify:cfl
7644 * Display WA #0873: cfl
7646 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7647 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7648 DPFC_NUKE_ON_ANY_MODIFICATION);
7651 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7653 gen9_init_clock_gating(dev_priv);
7655 /* WAC6entrylatency:kbl */
7656 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7657 FBC_LLC_FULLY_OPEN);
7659 /* WaDisableSDEUnitClockGating:kbl */
7660 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7661 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7662 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7664 /* WaDisableGamClockGating:kbl */
7665 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7666 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7667 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7670 * WaFbcTurnOffFbcWatermark:kbl
7671 * Display WA #0562: kbl
7673 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7677 * WaFbcNukeOnHostModify:kbl
7678 * Display WA #0873: kbl
7680 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7681 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7682 DPFC_NUKE_ON_ANY_MODIFICATION);
7685 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7687 gen9_init_clock_gating(dev_priv);
7689 /* WaDisableDopClockGating:skl */
7690 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7691 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7693 /* WAC6entrylatency:skl */
7694 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7695 FBC_LLC_FULLY_OPEN);
7698 * WaFbcTurnOffFbcWatermark:skl
7699 * Display WA #0562: skl
7701 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7705 * WaFbcNukeOnHostModify:skl
7706 * Display WA #0873: skl
7708 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7709 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7710 DPFC_NUKE_ON_ANY_MODIFICATION);
7713 * WaFbcHighMemBwCorruptionAvoidance:skl
7714 * Display WA #0883: skl
7716 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7717 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7718 DPFC_DISABLE_DUMMY0);
7721 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7725 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7726 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7727 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7730 /* WaSwitchSolVfFArbitrationPriority:bdw */
7731 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7733 /* WaPsrDPAMaskVBlankInSRD:bdw */
7734 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7735 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7737 for_each_pipe(dev_priv, pipe) {
7738 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7739 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7740 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7741 BDW_DPRS_MASK_VBLANK_SRD);
7744 /* WaVSRefCountFullforceMissDisable:bdw */
7745 /* WaDSRefCountFullforceMissDisable:bdw */
7746 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7747 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7748 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7750 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
7751 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7753 /* WaDisableSDEUnitClockGating:bdw */
7754 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7755 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7757 /* WaProgramL3SqcReg1Default:bdw */
7758 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7760 /* WaKVMNotificationOnConfigChange:bdw */
7761 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7762 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7764 lpt_init_clock_gating(dev_priv);
7766 /* WaDisableDopClockGating:bdw
7768 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7771 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7772 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7775 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7777 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7778 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7779 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7782 /* This is required by WaCatErrorRejectionIssue:hsw */
7783 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7784 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7785 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7787 /* WaSwitchSolVfFArbitrationPriority:hsw */
7788 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7790 lpt_init_clock_gating(dev_priv);
7793 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7797 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7799 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7800 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7801 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7804 /* WaDisableBackToBackFlipFix:ivb */
7805 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7806 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7807 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7809 if (IS_IVB_GT1(dev_priv))
7810 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7811 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7813 /* must write both registers */
7814 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7815 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7816 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7817 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7821 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7822 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7824 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7825 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7827 /* This is required by WaCatErrorRejectionIssue:ivb */
7828 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7829 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7830 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7832 g4x_disable_trickle_feed(dev_priv);
7834 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7835 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7836 snpcr |= GEN6_MBC_SNPCR_MED;
7837 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7839 if (!HAS_PCH_NOP(dev_priv))
7840 cpt_init_clock_gating(dev_priv);
7842 gen6_check_mch_setup(dev_priv);
7845 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7847 /* WaDisableBackToBackFlipFix:vlv */
7848 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7849 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7850 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7852 /* WaDisableDopClockGating:vlv */
7853 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7854 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7856 /* This is required by WaCatErrorRejectionIssue:vlv */
7857 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7858 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7859 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7862 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7863 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7865 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7866 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7868 /* WaDisableL3Bank2xClockGate:vlv
7869 * Disabling L3 clock gating- MMIO 940c[25] = 1
7870 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7871 intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7872 intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7875 * WaDisableVLVClockGating_VBIIssue:vlv
7876 * Disable clock gating on th GCFG unit to prevent a delay
7877 * in the reporting of vblank events.
7879 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7882 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7884 /* WaVSRefCountFullforceMissDisable:chv */
7885 /* WaDSRefCountFullforceMissDisable:chv */
7886 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7887 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7888 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7890 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7891 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
7892 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7894 /* WaDisableCSUnitClockGating:chv */
7895 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7896 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7898 /* WaDisableSDEUnitClockGating:chv */
7899 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7900 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7903 * WaProgramL3SqcReg1Default:chv
7904 * See gfxspecs/Related Documents/Performance Guide/
7905 * LSQC Setting Recommendations.
7907 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7910 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7914 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7915 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7916 GS_UNIT_CLOCK_GATE_DISABLE |
7917 CL_UNIT_CLOCK_GATE_DISABLE);
7918 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7919 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7920 OVRUNIT_CLOCK_GATE_DISABLE |
7921 OVCUNIT_CLOCK_GATE_DISABLE;
7922 if (IS_GM45(dev_priv))
7923 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7924 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7926 g4x_disable_trickle_feed(dev_priv);
7929 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7931 struct intel_uncore *uncore = &dev_priv->uncore;
7933 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7934 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7935 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7936 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7937 intel_uncore_write16(uncore, DEUC, 0);
7938 intel_uncore_write(uncore,
7940 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7943 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7945 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7946 I965_RCC_CLOCK_GATE_DISABLE |
7947 I965_RCPB_CLOCK_GATE_DISABLE |
7948 I965_ISC_CLOCK_GATE_DISABLE |
7949 I965_FBC_CLOCK_GATE_DISABLE);
7950 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7951 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7952 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7955 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7957 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7959 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7960 DSTATE_DOT_CLOCK_GATING;
7961 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7963 if (IS_PINEVIEW(dev_priv))
7964 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
7965 _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7967 /* IIR "flip pending" means done if this bit is set */
7968 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
7969 _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7971 /* interrupts should cause a wake up from C3 */
7972 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7974 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7975 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7977 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7978 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7981 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7983 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7985 /* interrupts should cause a wake up from C3 */
7986 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7987 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7989 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7990 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7993 * Have FBC ignore 3D activity since we use software
7994 * render tracking, and otherwise a pure 3D workload
7995 * (even if it just renders a single frame and then does
7996 * abosultely nothing) would not allow FBC to recompress
7997 * until a 2D blit occurs.
7999 intel_uncore_write(&dev_priv->uncore, SCPD0,
8000 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
8003 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
8005 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
8006 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
8007 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
8010 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
8012 dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
8015 void intel_suspend_hw(struct drm_i915_private *dev_priv)
8017 if (HAS_PCH_LPT(dev_priv))
8018 lpt_suspend_hw(dev_priv);
8021 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
8023 drm_dbg_kms(&dev_priv->drm,
8024 "No clock gating settings or workarounds applied.\n");
8027 #define CG_FUNCS(platform) \
8028 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
8029 .init_clock_gating = platform##_init_clock_gating, \
8060 * intel_init_clock_gating_hooks - setup the clock gating hooks
8061 * @dev_priv: device private
8063 * Setup the hooks that configure which clocks of a given platform can be
8064 * gated and also apply various GT and display specific workarounds for these
8065 * platforms. Note that some GT specific workarounds are applied separately
8066 * when GPU contexts or batchbuffers start their execution.
8068 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
8070 if (IS_DG2(dev_priv))
8071 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
8072 else if (IS_XEHPSDV(dev_priv))
8073 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
8074 else if (IS_ALDERLAKE_P(dev_priv))
8075 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
8076 else if (IS_DG1(dev_priv))
8077 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
8078 else if (GRAPHICS_VER(dev_priv) == 12)
8079 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
8080 else if (GRAPHICS_VER(dev_priv) == 11)
8081 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
8082 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
8083 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
8084 else if (IS_SKYLAKE(dev_priv))
8085 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
8086 else if (IS_KABYLAKE(dev_priv))
8087 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
8088 else if (IS_BROXTON(dev_priv))
8089 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
8090 else if (IS_GEMINILAKE(dev_priv))
8091 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
8092 else if (IS_BROADWELL(dev_priv))
8093 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
8094 else if (IS_CHERRYVIEW(dev_priv))
8095 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
8096 else if (IS_HASWELL(dev_priv))
8097 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
8098 else if (IS_IVYBRIDGE(dev_priv))
8099 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
8100 else if (IS_VALLEYVIEW(dev_priv))
8101 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
8102 else if (GRAPHICS_VER(dev_priv) == 6)
8103 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
8104 else if (GRAPHICS_VER(dev_priv) == 5)
8105 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
8106 else if (IS_G4X(dev_priv))
8107 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
8108 else if (IS_I965GM(dev_priv))
8109 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
8110 else if (IS_I965G(dev_priv))
8111 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
8112 else if (GRAPHICS_VER(dev_priv) == 3)
8113 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
8114 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
8115 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
8116 else if (GRAPHICS_VER(dev_priv) == 2)
8117 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
8119 MISSING_CASE(INTEL_DEVID(dev_priv));
8120 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
8124 static const struct drm_i915_wm_disp_funcs skl_wm_funcs = {
8125 .compute_global_watermarks = skl_compute_wm,
8128 static const struct drm_i915_wm_disp_funcs ilk_wm_funcs = {
8129 .compute_pipe_wm = ilk_compute_pipe_wm,
8130 .compute_intermediate_wm = ilk_compute_intermediate_wm,
8131 .initial_watermarks = ilk_initial_watermarks,
8132 .optimize_watermarks = ilk_optimize_watermarks,
8135 static const struct drm_i915_wm_disp_funcs vlv_wm_funcs = {
8136 .compute_pipe_wm = vlv_compute_pipe_wm,
8137 .compute_intermediate_wm = vlv_compute_intermediate_wm,
8138 .initial_watermarks = vlv_initial_watermarks,
8139 .optimize_watermarks = vlv_optimize_watermarks,
8140 .atomic_update_watermarks = vlv_atomic_update_fifo,
8143 static const struct drm_i915_wm_disp_funcs g4x_wm_funcs = {
8144 .compute_pipe_wm = g4x_compute_pipe_wm,
8145 .compute_intermediate_wm = g4x_compute_intermediate_wm,
8146 .initial_watermarks = g4x_initial_watermarks,
8147 .optimize_watermarks = g4x_optimize_watermarks,
8150 static const struct drm_i915_wm_disp_funcs pnv_wm_funcs = {
8151 .update_wm = pnv_update_wm,
8154 static const struct drm_i915_wm_disp_funcs i965_wm_funcs = {
8155 .update_wm = i965_update_wm,
8158 static const struct drm_i915_wm_disp_funcs i9xx_wm_funcs = {
8159 .update_wm = i9xx_update_wm,
8162 static const struct drm_i915_wm_disp_funcs i845_wm_funcs = {
8163 .update_wm = i845_update_wm,
8166 static const struct drm_i915_wm_disp_funcs nop_funcs = {
8169 /* Set up chip specific power management-related functions */
8170 void intel_init_pm(struct drm_i915_private *dev_priv)
8173 if (IS_PINEVIEW(dev_priv))
8174 pnv_get_mem_freq(dev_priv);
8175 else if (GRAPHICS_VER(dev_priv) == 5)
8176 ilk_get_mem_freq(dev_priv);
8178 if (intel_has_sagv(dev_priv))
8179 skl_setup_sagv_block_time(dev_priv);
8181 /* For FIFO watermark updates */
8182 if (DISPLAY_VER(dev_priv) >= 9) {
8183 skl_setup_wm_latency(dev_priv);
8184 dev_priv->wm_disp = &skl_wm_funcs;
8185 } else if (HAS_PCH_SPLIT(dev_priv)) {
8186 ilk_setup_wm_latency(dev_priv);
8188 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] &&
8189 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
8190 (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] &&
8191 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
8192 dev_priv->wm_disp = &ilk_wm_funcs;
8194 drm_dbg_kms(&dev_priv->drm,
8195 "Failed to read display plane latency. "
8197 dev_priv->wm_disp = &nop_funcs;
8199 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
8200 vlv_setup_wm_latency(dev_priv);
8201 dev_priv->wm_disp = &vlv_wm_funcs;
8202 } else if (IS_G4X(dev_priv)) {
8203 g4x_setup_wm_latency(dev_priv);
8204 dev_priv->wm_disp = &g4x_wm_funcs;
8205 } else if (IS_PINEVIEW(dev_priv)) {
8206 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
8209 dev_priv->mem_freq)) {
8210 drm_info(&dev_priv->drm,
8211 "failed to find known CxSR latency "
8212 "(found ddr%s fsb freq %d, mem freq %d), "
8214 (dev_priv->is_ddr3 == 1) ? "3" : "2",
8215 dev_priv->fsb_freq, dev_priv->mem_freq);
8216 /* Disable CxSR and never update its watermark again */
8217 intel_set_memory_cxsr(dev_priv, false);
8218 dev_priv->wm_disp = &nop_funcs;
8220 dev_priv->wm_disp = &pnv_wm_funcs;
8221 } else if (DISPLAY_VER(dev_priv) == 4) {
8222 dev_priv->wm_disp = &i965_wm_funcs;
8223 } else if (DISPLAY_VER(dev_priv) == 3) {
8224 dev_priv->wm_disp = &i9xx_wm_funcs;
8225 } else if (DISPLAY_VER(dev_priv) == 2) {
8226 if (INTEL_NUM_PIPES(dev_priv) == 1)
8227 dev_priv->wm_disp = &i845_wm_funcs;
8229 dev_priv->wm_disp = &i9xx_wm_funcs;
8231 drm_err(&dev_priv->drm,
8232 "unexpected fall-through in %s\n", __func__);
8233 dev_priv->wm_disp = &nop_funcs;
8237 void intel_pm_setup(struct drm_i915_private *dev_priv)
8239 dev_priv->runtime_pm.suspended = false;
8240 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
8243 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
8245 struct intel_dbuf_state *dbuf_state;
8247 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
8251 return &dbuf_state->base;
8254 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
8255 struct intel_global_state *state)
8260 static const struct intel_global_state_funcs intel_dbuf_funcs = {
8261 .atomic_duplicate_state = intel_dbuf_duplicate_state,
8262 .atomic_destroy_state = intel_dbuf_destroy_state,
8265 struct intel_dbuf_state *
8266 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
8268 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8269 struct intel_global_state *dbuf_state;
8271 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
8272 if (IS_ERR(dbuf_state))
8273 return ERR_CAST(dbuf_state);
8275 return to_intel_dbuf_state(dbuf_state);
8278 int intel_dbuf_init(struct drm_i915_private *dev_priv)
8280 struct intel_dbuf_state *dbuf_state;
8282 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
8286 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
8287 &dbuf_state->base, &intel_dbuf_funcs);
8293 * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
8294 * update the request state of all DBUS slices.
8296 static void update_mbus_pre_enable(struct intel_atomic_state *state)
8298 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8299 u32 mbus_ctl, dbuf_min_tracker_val;
8300 enum dbuf_slice slice;
8301 const struct intel_dbuf_state *dbuf_state =
8302 intel_atomic_get_new_dbuf_state(state);
8304 if (!IS_ALDERLAKE_P(dev_priv))
8308 * TODO: Implement vblank synchronized MBUS joining changes.
8309 * Must be properly coordinated with dbuf reprogramming.
8311 if (dbuf_state->joined_mbus) {
8312 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
8313 MBUS_JOIN_PIPE_SELECT_NONE;
8314 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
8316 mbus_ctl = MBUS_HASHING_MODE_2x2 |
8317 MBUS_JOIN_PIPE_SELECT_NONE;
8318 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
8321 intel_de_rmw(dev_priv, MBUS_CTL,
8322 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
8323 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
8325 for_each_dbuf_slice(dev_priv, slice)
8326 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
8327 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
8328 dbuf_min_tracker_val);
8331 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
8333 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8334 const struct intel_dbuf_state *new_dbuf_state =
8335 intel_atomic_get_new_dbuf_state(state);
8336 const struct intel_dbuf_state *old_dbuf_state =
8337 intel_atomic_get_old_dbuf_state(state);
8339 if (!new_dbuf_state ||
8340 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8341 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8344 WARN_ON(!new_dbuf_state->base.changed);
8346 update_mbus_pre_enable(state);
8347 gen9_dbuf_slices_update(dev_priv,
8348 old_dbuf_state->enabled_slices |
8349 new_dbuf_state->enabled_slices);
8352 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
8354 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8355 const struct intel_dbuf_state *new_dbuf_state =
8356 intel_atomic_get_new_dbuf_state(state);
8357 const struct intel_dbuf_state *old_dbuf_state =
8358 intel_atomic_get_old_dbuf_state(state);
8360 if (!new_dbuf_state ||
8361 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8362 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8365 WARN_ON(!new_dbuf_state->base.changed);
8367 gen9_dbuf_slices_update(dev_priv,
8368 new_dbuf_state->enabled_slices);