2 * Copyright © 2012 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eugeni Dodonov <eugeni.dodonov@intel.com>
28 #include <linux/module.h>
29 #include <linux/string_helpers.h>
30 #include <linux/pm_runtime.h>
32 #include <drm/drm_atomic_helper.h>
33 #include <drm/drm_fourcc.h>
34 #include <drm/drm_plane_helper.h>
36 #include "display/intel_atomic.h"
37 #include "display/intel_atomic_plane.h"
38 #include "display/intel_bw.h"
39 #include "display/intel_de.h"
40 #include "display/intel_display_trace.h"
41 #include "display/intel_display_types.h"
42 #include "display/intel_fb.h"
43 #include "display/intel_fbc.h"
44 #include "display/intel_sprite.h"
45 #include "display/skl_universal_plane.h"
47 #include "gt/intel_engine_regs.h"
48 #include "gt/intel_gt_regs.h"
49 #include "gt/intel_llc.h"
52 #include "i915_fixed.h"
54 #include "intel_mchbar_regs.h"
55 #include "intel_pcode.h"
57 #include "vlv_sideband.h"
58 #include "../../../platform/x86/intel_ips.h"
60 static void skl_sagv_disable(struct drm_i915_private *dev_priv);
62 struct drm_i915_clock_gating_funcs {
63 void (*init_clock_gating)(struct drm_i915_private *i915);
66 /* Stores plane specific WM parameters */
67 struct skl_wm_params {
68 bool x_tiled, y_tiled;
75 u32 plane_bytes_per_line;
76 uint_fixed_16_16_t plane_blocks_per_line;
77 uint_fixed_16_16_t y_tile_minimum;
82 /* used in computing the new watermarks state */
83 struct intel_wm_config {
84 unsigned int num_pipes_active;
89 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
91 if (HAS_LLC(dev_priv)) {
93 * WaCompressedResourceDisplayNewHashMode:skl,kbl
94 * Display WA #0390: skl,kbl
96 * Must match Sampler, Pixel Back End, and Media. See
97 * WaCompressedResourceSamplerPbeMediaNewHashMode.
99 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
100 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
101 SKL_DE_COMPRESSED_HASH_MODE);
104 /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
105 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
106 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
108 /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
109 intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
110 intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
113 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
114 * Display WA #0859: skl,bxt,kbl,glk,cfl
116 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
117 DISP_FBC_MEMORY_WAKE);
120 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
122 gen9_init_clock_gating(dev_priv);
124 /* WaDisableSDEUnitClockGating:bxt */
125 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
126 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
130 * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
132 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
133 GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
136 * Wa: Backlight PWM may stop in the asserted state, causing backlight
139 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
140 PWM1_GATING_DIS | PWM2_GATING_DIS);
143 * Lower the display internal timeout.
144 * This is needed to avoid any hard hangs when DSI port PLL
145 * is off and a MMIO access is attempted by any privilege
146 * application, using batch buffers or any other means.
148 intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
151 * WaFbcTurnOffFbcWatermark:bxt
152 * Display WA #0562: bxt
154 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
158 * WaFbcHighMemBwCorruptionAvoidance:bxt
159 * Display WA #0883: bxt
161 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
162 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
163 DPFC_DISABLE_DUMMY0);
166 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
168 gen9_init_clock_gating(dev_priv);
171 * WaDisablePWMClockGating:glk
172 * Backlight PWM may stop in the asserted state, causing backlight
175 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
176 PWM1_GATING_DIS | PWM2_GATING_DIS);
179 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
183 tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
185 switch (tmp & CLKCFG_FSB_MASK) {
187 dev_priv->fsb_freq = 533; /* 133*4 */
190 dev_priv->fsb_freq = 800; /* 200*4 */
193 dev_priv->fsb_freq = 667; /* 167*4 */
196 dev_priv->fsb_freq = 400; /* 100*4 */
200 switch (tmp & CLKCFG_MEM_MASK) {
202 dev_priv->mem_freq = 533;
205 dev_priv->mem_freq = 667;
208 dev_priv->mem_freq = 800;
212 /* detect pineview DDR3 setting */
213 tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
214 dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
217 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
221 ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
222 csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
224 switch (ddrpll & 0xff) {
226 dev_priv->mem_freq = 800;
229 dev_priv->mem_freq = 1066;
232 dev_priv->mem_freq = 1333;
235 dev_priv->mem_freq = 1600;
238 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
240 dev_priv->mem_freq = 0;
244 switch (csipll & 0x3ff) {
246 dev_priv->fsb_freq = 3200;
249 dev_priv->fsb_freq = 3733;
252 dev_priv->fsb_freq = 4266;
255 dev_priv->fsb_freq = 4800;
258 dev_priv->fsb_freq = 5333;
261 dev_priv->fsb_freq = 5866;
264 dev_priv->fsb_freq = 6400;
267 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
269 dev_priv->fsb_freq = 0;
274 static const struct cxsr_latency cxsr_latency_table[] = {
275 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
276 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
277 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
278 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
279 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
281 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
282 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
283 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
284 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
285 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
287 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
288 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
289 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
290 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
291 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
293 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
294 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
295 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
296 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
297 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
299 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
300 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
301 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
302 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
303 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
305 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
306 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
307 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
308 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
309 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
312 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
317 const struct cxsr_latency *latency;
320 if (fsb == 0 || mem == 0)
323 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
324 latency = &cxsr_latency_table[i];
325 if (is_desktop == latency->is_desktop &&
326 is_ddr3 == latency->is_ddr3 &&
327 fsb == latency->fsb_freq && mem == latency->mem_freq)
331 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
336 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
340 vlv_punit_get(dev_priv);
342 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
344 val &= ~FORCE_DDR_HIGH_FREQ;
346 val |= FORCE_DDR_HIGH_FREQ;
347 val &= ~FORCE_DDR_LOW_FREQ;
348 val |= FORCE_DDR_FREQ_REQ_ACK;
349 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
351 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
352 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
353 drm_err(&dev_priv->drm,
354 "timed out waiting for Punit DDR DVFS request\n");
356 vlv_punit_put(dev_priv);
359 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
363 vlv_punit_get(dev_priv);
365 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
367 val |= DSP_MAXFIFO_PM5_ENABLE;
369 val &= ~DSP_MAXFIFO_PM5_ENABLE;
370 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
372 vlv_punit_put(dev_priv);
375 #define FW_WM(value, plane) \
376 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
378 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
383 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
384 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
385 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
386 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
387 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
388 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
389 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
390 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
391 } else if (IS_PINEVIEW(dev_priv)) {
392 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
393 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
395 val |= PINEVIEW_SELF_REFRESH_EN;
397 val &= ~PINEVIEW_SELF_REFRESH_EN;
398 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
399 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
400 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
401 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
402 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
403 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
404 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
405 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
406 } else if (IS_I915GM(dev_priv)) {
408 * FIXME can't find a bit like this for 915G, and
409 * and yet it does have the related watermark in
410 * FW_BLC_SELF. What's going on?
412 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
413 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
414 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
415 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
416 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
421 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
423 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
424 str_enabled_disabled(enable),
425 str_enabled_disabled(was_enabled));
431 * intel_set_memory_cxsr - Configure CxSR state
432 * @dev_priv: i915 device
433 * @enable: Allow vs. disallow CxSR
435 * Allow or disallow the system to enter a special CxSR
436 * (C-state self refresh) state. What typically happens in CxSR mode
437 * is that several display FIFOs may get combined into a single larger
438 * FIFO for a particular plane (so called max FIFO mode) to allow the
439 * system to defer memory fetches longer, and the memory will enter
442 * Note that enabling CxSR does not guarantee that the system enter
443 * this special mode, nor does it guarantee that the system stays
444 * in that mode once entered. So this just allows/disallows the system
445 * to autonomously utilize the CxSR mode. Other factors such as core
446 * C-states will affect when/if the system actually enters/exits the
449 * Note that on VLV/CHV this actually only controls the max FIFO mode,
450 * and the system is free to enter/exit memory self refresh at any time
451 * even when the use of CxSR has been disallowed.
453 * While the system is actually in the CxSR/max FIFO mode, some plane
454 * control registers will not get latched on vblank. Thus in order to
455 * guarantee the system will respond to changes in the plane registers
456 * we must always disallow CxSR prior to making changes to those registers.
457 * Unfortunately the system will re-evaluate the CxSR conditions at
458 * frame start which happens after vblank start (which is when the plane
459 * registers would get latched), so we can't proceed with the plane update
460 * during the same frame where we disallowed CxSR.
462 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
463 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
464 * the hardware w.r.t. HPLL SR when writing to plane registers.
465 * Disallowing just CxSR is sufficient.
467 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
471 mutex_lock(&dev_priv->wm.wm_mutex);
472 ret = _intel_set_memory_cxsr(dev_priv, enable);
473 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
474 dev_priv->wm.vlv.cxsr = enable;
475 else if (IS_G4X(dev_priv))
476 dev_priv->wm.g4x.cxsr = enable;
477 mutex_unlock(&dev_priv->wm.wm_mutex);
483 * Latency for FIFO fetches is dependent on several factors:
484 * - memory configuration (speed, channels)
486 * - current MCH state
487 * It can be fairly high in some situations, so here we assume a fairly
488 * pessimal value. It's a tradeoff between extra memory fetches (if we
489 * set this value too high, the FIFO will fetch frequently to stay full)
490 * and power consumption (set it too low to save power and we might see
491 * FIFO underruns and display "flicker").
493 * A value of 5us seems to be a good balance; safe for very low end
494 * platforms but not overly aggressive on lower latency configs.
496 static const int pessimal_latency_ns = 5000;
498 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
499 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
501 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
503 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
504 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
505 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
506 enum pipe pipe = crtc->pipe;
507 int sprite0_start, sprite1_start;
508 u32 dsparb, dsparb2, dsparb3;
512 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
513 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
514 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
515 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
518 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
519 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
520 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
521 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
524 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
525 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
526 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
527 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
534 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
535 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
536 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
537 fifo_state->plane[PLANE_CURSOR] = 63;
540 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
541 enum i9xx_plane_id i9xx_plane)
543 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
546 size = dsparb & 0x7f;
547 if (i9xx_plane == PLANE_B)
548 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
550 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
551 dsparb, plane_name(i9xx_plane), size);
556 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
557 enum i9xx_plane_id i9xx_plane)
559 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
562 size = dsparb & 0x1ff;
563 if (i9xx_plane == PLANE_B)
564 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
565 size >>= 1; /* Convert to cachelines */
567 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
568 dsparb, plane_name(i9xx_plane), size);
573 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
574 enum i9xx_plane_id i9xx_plane)
576 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
579 size = dsparb & 0x7f;
580 size >>= 2; /* Convert to cachelines */
582 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
583 dsparb, plane_name(i9xx_plane), size);
588 /* Pineview has different values for various configs */
589 static const struct intel_watermark_params pnv_display_wm = {
590 .fifo_size = PINEVIEW_DISPLAY_FIFO,
591 .max_wm = PINEVIEW_MAX_WM,
592 .default_wm = PINEVIEW_DFT_WM,
593 .guard_size = PINEVIEW_GUARD_WM,
594 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
597 static const struct intel_watermark_params pnv_display_hplloff_wm = {
598 .fifo_size = PINEVIEW_DISPLAY_FIFO,
599 .max_wm = PINEVIEW_MAX_WM,
600 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
601 .guard_size = PINEVIEW_GUARD_WM,
602 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
605 static const struct intel_watermark_params pnv_cursor_wm = {
606 .fifo_size = PINEVIEW_CURSOR_FIFO,
607 .max_wm = PINEVIEW_CURSOR_MAX_WM,
608 .default_wm = PINEVIEW_CURSOR_DFT_WM,
609 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
610 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
613 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
614 .fifo_size = PINEVIEW_CURSOR_FIFO,
615 .max_wm = PINEVIEW_CURSOR_MAX_WM,
616 .default_wm = PINEVIEW_CURSOR_DFT_WM,
617 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
618 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
621 static const struct intel_watermark_params i965_cursor_wm_info = {
622 .fifo_size = I965_CURSOR_FIFO,
623 .max_wm = I965_CURSOR_MAX_WM,
624 .default_wm = I965_CURSOR_DFT_WM,
626 .cacheline_size = I915_FIFO_LINE_SIZE,
629 static const struct intel_watermark_params i945_wm_info = {
630 .fifo_size = I945_FIFO_SIZE,
631 .max_wm = I915_MAX_WM,
634 .cacheline_size = I915_FIFO_LINE_SIZE,
637 static const struct intel_watermark_params i915_wm_info = {
638 .fifo_size = I915_FIFO_SIZE,
639 .max_wm = I915_MAX_WM,
642 .cacheline_size = I915_FIFO_LINE_SIZE,
645 static const struct intel_watermark_params i830_a_wm_info = {
646 .fifo_size = I855GM_FIFO_SIZE,
647 .max_wm = I915_MAX_WM,
650 .cacheline_size = I830_FIFO_LINE_SIZE,
653 static const struct intel_watermark_params i830_bc_wm_info = {
654 .fifo_size = I855GM_FIFO_SIZE,
655 .max_wm = I915_MAX_WM/2,
658 .cacheline_size = I830_FIFO_LINE_SIZE,
661 static const struct intel_watermark_params i845_wm_info = {
662 .fifo_size = I830_FIFO_SIZE,
663 .max_wm = I915_MAX_WM,
666 .cacheline_size = I830_FIFO_LINE_SIZE,
670 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
671 * @pixel_rate: Pipe pixel rate in kHz
672 * @cpp: Plane bytes per pixel
673 * @latency: Memory wakeup latency in 0.1us units
675 * Compute the watermark using the method 1 or "small buffer"
676 * formula. The caller may additonally add extra cachelines
677 * to account for TLB misses and clock crossings.
679 * This method is concerned with the short term drain rate
680 * of the FIFO, ie. it does not account for blanking periods
681 * which would effectively reduce the average drain rate across
682 * a longer period. The name "small" refers to the fact the
683 * FIFO is relatively small compared to the amount of data
686 * The FIFO level vs. time graph might look something like:
690 * __---__---__ (- plane active, _ blanking)
693 * or perhaps like this:
696 * __----__----__ (- plane active, _ blanking)
700 * The watermark in bytes
702 static unsigned int intel_wm_method1(unsigned int pixel_rate,
704 unsigned int latency)
708 ret = mul_u32_u32(pixel_rate, cpp * latency);
709 ret = DIV_ROUND_UP_ULL(ret, 10000);
715 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
716 * @pixel_rate: Pipe pixel rate in kHz
717 * @htotal: Pipe horizontal total
718 * @width: Plane width in pixels
719 * @cpp: Plane bytes per pixel
720 * @latency: Memory wakeup latency in 0.1us units
722 * Compute the watermark using the method 2 or "large buffer"
723 * formula. The caller may additonally add extra cachelines
724 * to account for TLB misses and clock crossings.
726 * This method is concerned with the long term drain rate
727 * of the FIFO, ie. it does account for blanking periods
728 * which effectively reduce the average drain rate across
729 * a longer period. The name "large" refers to the fact the
730 * FIFO is relatively large compared to the amount of data
733 * The FIFO level vs. time graph might look something like:
738 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
742 * The watermark in bytes
744 static unsigned int intel_wm_method2(unsigned int pixel_rate,
748 unsigned int latency)
753 * FIXME remove once all users are computing
754 * watermarks in the correct place.
756 if (WARN_ON_ONCE(htotal == 0))
759 ret = (latency * pixel_rate) / (htotal * 10000);
760 ret = (ret + 1) * width * cpp;
766 * intel_calculate_wm - calculate watermark level
767 * @pixel_rate: pixel clock
768 * @wm: chip FIFO params
769 * @fifo_size: size of the FIFO buffer
770 * @cpp: bytes per pixel
771 * @latency_ns: memory latency for the platform
773 * Calculate the watermark level (the level at which the display plane will
774 * start fetching from memory again). Each chip has a different display
775 * FIFO size and allocation, so the caller needs to figure that out and pass
776 * in the correct intel_watermark_params structure.
778 * As the pixel clock runs, the FIFO will be drained at a rate that depends
779 * on the pixel size. When it reaches the watermark level, it'll start
780 * fetching FIFO line sized based chunks from memory until the FIFO fills
781 * past the watermark point. If the FIFO drains completely, a FIFO underrun
782 * will occur, and a display engine hang could result.
784 static unsigned int intel_calculate_wm(int pixel_rate,
785 const struct intel_watermark_params *wm,
786 int fifo_size, int cpp,
787 unsigned int latency_ns)
789 int entries, wm_size;
792 * Note: we need to make sure we don't overflow for various clock &
794 * clocks go from a few thousand to several hundred thousand.
795 * latency is usually a few thousand
797 entries = intel_wm_method1(pixel_rate, cpp,
799 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
801 DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
803 wm_size = fifo_size - entries;
804 DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
806 /* Don't promote wm_size to unsigned... */
807 if (wm_size > wm->max_wm)
808 wm_size = wm->max_wm;
810 wm_size = wm->default_wm;
813 * Bspec seems to indicate that the value shouldn't be lower than
814 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
815 * Lets go for 8 which is the burst size since certain platforms
816 * already use a hardcoded 8 (which is what the spec says should be
825 static bool is_disabling(int old, int new, int threshold)
827 return old >= threshold && new < threshold;
830 static bool is_enabling(int old, int new, int threshold)
832 return old < threshold && new >= threshold;
835 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
837 return dev_priv->wm.max_level + 1;
840 static bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
841 const struct intel_plane_state *plane_state)
843 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
845 /* FIXME check the 'enable' instead */
846 if (!crtc_state->hw.active)
850 * Treat cursor with fb as always visible since cursor updates
851 * can happen faster than the vrefresh rate, and the current
852 * watermark code doesn't handle that correctly. Cursor updates
853 * which set/clear the fb or change the cursor size are going
854 * to get throttled by intel_legacy_cursor_update() to work
855 * around this problem with the watermark code.
857 if (plane->id == PLANE_CURSOR)
858 return plane_state->hw.fb != NULL;
860 return plane_state->uapi.visible;
863 static bool intel_crtc_active(struct intel_crtc *crtc)
865 /* Be paranoid as we can arrive here with only partial
866 * state retrieved from the hardware during setup.
868 * We can ditch the adjusted_mode.crtc_clock check as soon
869 * as Haswell has gained clock readout/fastboot support.
871 * We can ditch the crtc->primary->state->fb check as soon as we can
872 * properly reconstruct framebuffers.
874 * FIXME: The intel_crtc->active here should be switched to
875 * crtc->state->active once we have proper CRTC states wired up
878 return crtc && crtc->active && crtc->base.primary->state->fb &&
879 crtc->config->hw.adjusted_mode.crtc_clock;
882 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
884 struct intel_crtc *crtc, *enabled = NULL;
886 for_each_intel_crtc(&dev_priv->drm, crtc) {
887 if (intel_crtc_active(crtc)) {
897 static void pnv_update_wm(struct drm_i915_private *dev_priv)
899 struct intel_crtc *crtc;
900 const struct cxsr_latency *latency;
904 latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
909 drm_dbg_kms(&dev_priv->drm,
910 "Unknown FSB/MEM found, disable CxSR\n");
911 intel_set_memory_cxsr(dev_priv, false);
915 crtc = single_enabled_crtc(dev_priv);
917 const struct drm_framebuffer *fb =
918 crtc->base.primary->state->fb;
919 int pixel_rate = crtc->config->pixel_rate;
920 int cpp = fb->format->cpp[0];
923 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
924 pnv_display_wm.fifo_size,
925 cpp, latency->display_sr);
926 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
927 reg &= ~DSPFW_SR_MASK;
928 reg |= FW_WM(wm, SR);
929 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
930 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
933 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
934 pnv_display_wm.fifo_size,
935 4, latency->cursor_sr);
936 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
937 reg &= ~DSPFW_CURSOR_SR_MASK;
938 reg |= FW_WM(wm, CURSOR_SR);
939 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
941 /* Display HPLL off SR */
942 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
943 pnv_display_hplloff_wm.fifo_size,
944 cpp, latency->display_hpll_disable);
945 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
946 reg &= ~DSPFW_HPLL_SR_MASK;
947 reg |= FW_WM(wm, HPLL_SR);
948 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
950 /* cursor HPLL off SR */
951 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
952 pnv_display_hplloff_wm.fifo_size,
953 4, latency->cursor_hpll_disable);
954 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
955 reg &= ~DSPFW_HPLL_CURSOR_MASK;
956 reg |= FW_WM(wm, HPLL_CURSOR);
957 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
958 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
960 intel_set_memory_cxsr(dev_priv, true);
962 intel_set_memory_cxsr(dev_priv, false);
967 * Documentation says:
968 * "If the line size is small, the TLB fetches can get in the way of the
969 * data fetches, causing some lag in the pixel data return which is not
970 * accounted for in the above formulas. The following adjustment only
971 * needs to be applied if eight whole lines fit in the buffer at once.
972 * The WM is adjusted upwards by the difference between the FIFO size
973 * and the size of 8 whole lines. This adjustment is always performed
974 * in the actual pixel depth regardless of whether FBC is enabled or not."
976 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
978 int tlb_miss = fifo_size * 64 - width * cpp * 8;
980 return max(0, tlb_miss);
983 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
984 const struct g4x_wm_values *wm)
988 for_each_pipe(dev_priv, pipe)
989 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
991 intel_uncore_write(&dev_priv->uncore, DSPFW1,
992 FW_WM(wm->sr.plane, SR) |
993 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
994 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
995 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
996 intel_uncore_write(&dev_priv->uncore, DSPFW2,
997 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
998 FW_WM(wm->sr.fbc, FBC_SR) |
999 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
1000 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
1001 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1002 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1003 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1004 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
1005 FW_WM(wm->sr.cursor, CURSOR_SR) |
1006 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
1007 FW_WM(wm->hpll.plane, HPLL_SR));
1009 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1012 #define FW_WM_VLV(value, plane) \
1013 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
1015 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
1016 const struct vlv_wm_values *wm)
1020 for_each_pipe(dev_priv, pipe) {
1021 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
1023 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
1024 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
1025 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
1026 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
1027 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
1031 * Zero the (unused) WM1 watermarks, and also clear all the
1032 * high order bits so that there are no out of bounds values
1033 * present in the registers during the reprogramming.
1035 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
1036 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
1037 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
1038 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
1039 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
1041 intel_uncore_write(&dev_priv->uncore, DSPFW1,
1042 FW_WM(wm->sr.plane, SR) |
1043 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1044 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1045 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1046 intel_uncore_write(&dev_priv->uncore, DSPFW2,
1047 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1048 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1049 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1050 intel_uncore_write(&dev_priv->uncore, DSPFW3,
1051 FW_WM(wm->sr.cursor, CURSOR_SR));
1053 if (IS_CHERRYVIEW(dev_priv)) {
1054 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1055 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1056 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1057 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1058 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1059 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1060 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1061 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1062 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1063 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1064 FW_WM(wm->sr.plane >> 9, SR_HI) |
1065 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1066 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1067 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1068 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1069 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1070 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1071 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1072 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1073 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1075 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1076 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1077 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1078 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1079 FW_WM(wm->sr.plane >> 9, SR_HI) |
1080 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1081 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1082 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1083 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1084 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1085 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1088 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1093 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1095 /* all latencies in usec */
1096 dev_priv->wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1097 dev_priv->wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1098 dev_priv->wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1100 dev_priv->wm.max_level = G4X_WM_LEVEL_HPLL;
1103 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1106 * DSPCNTR[13] supposedly controls whether the
1107 * primary plane can use the FIFO space otherwise
1108 * reserved for the sprite plane. It's not 100% clear
1109 * what the actual FIFO size is, but it looks like we
1110 * can happily set both primary and sprite watermarks
1111 * up to 127 cachelines. So that would seem to mean
1112 * that either DSPCNTR[13] doesn't do anything, or that
1113 * the total FIFO is >= 256 cachelines in size. Either
1114 * way, we don't seem to have to worry about this
1115 * repartitioning as the maximum watermark value the
1116 * register can hold for each plane is lower than the
1117 * minimum FIFO size.
1123 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1125 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1127 MISSING_CASE(plane_id);
1132 static int g4x_fbc_fifo_size(int level)
1135 case G4X_WM_LEVEL_SR:
1137 case G4X_WM_LEVEL_HPLL:
1140 MISSING_CASE(level);
1145 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1146 const struct intel_plane_state *plane_state,
1149 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1150 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1151 const struct drm_display_mode *pipe_mode =
1152 &crtc_state->hw.pipe_mode;
1153 unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
1154 unsigned int pixel_rate, htotal, cpp, width, wm;
1159 if (!intel_wm_plane_visible(crtc_state, plane_state))
1162 cpp = plane_state->hw.fb->format->cpp[0];
1165 * WaUse32BppForSRWM:ctg,elk
1167 * The spec fails to list this restriction for the
1168 * HPLL watermark, which seems a little strange.
1169 * Let's use 32bpp for the HPLL watermark as well.
1171 if (plane->id == PLANE_PRIMARY &&
1172 level != G4X_WM_LEVEL_NORMAL)
1175 pixel_rate = crtc_state->pixel_rate;
1176 htotal = pipe_mode->crtc_htotal;
1177 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1179 if (plane->id == PLANE_CURSOR) {
1180 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1181 } else if (plane->id == PLANE_PRIMARY &&
1182 level == G4X_WM_LEVEL_NORMAL) {
1183 wm = intel_wm_method1(pixel_rate, cpp, latency);
1185 unsigned int small, large;
1187 small = intel_wm_method1(pixel_rate, cpp, latency);
1188 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1190 wm = min(small, large);
1193 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1196 wm = DIV_ROUND_UP(wm, 64) + 2;
1198 return min_t(unsigned int, wm, USHRT_MAX);
1201 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1202 int level, enum plane_id plane_id, u16 value)
1204 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1207 for (; level < intel_wm_num_levels(dev_priv); level++) {
1208 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1210 dirty |= raw->plane[plane_id] != value;
1211 raw->plane[plane_id] = value;
1217 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1218 int level, u16 value)
1220 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1223 /* NORMAL level doesn't have an FBC watermark */
1224 level = max(level, G4X_WM_LEVEL_SR);
1226 for (; level < intel_wm_num_levels(dev_priv); level++) {
1227 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1229 dirty |= raw->fbc != value;
1236 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1237 const struct intel_plane_state *plane_state,
1240 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1241 const struct intel_plane_state *plane_state)
1243 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1244 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1245 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1246 enum plane_id plane_id = plane->id;
1250 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1251 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1252 if (plane_id == PLANE_PRIMARY)
1253 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1257 for (level = 0; level < num_levels; level++) {
1258 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1261 wm = g4x_compute_wm(crtc_state, plane_state, level);
1262 max_wm = g4x_plane_fifo_size(plane_id, level);
1267 dirty |= raw->plane[plane_id] != wm;
1268 raw->plane[plane_id] = wm;
1270 if (plane_id != PLANE_PRIMARY ||
1271 level == G4X_WM_LEVEL_NORMAL)
1274 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1275 raw->plane[plane_id]);
1276 max_wm = g4x_fbc_fifo_size(level);
1279 * FBC wm is not mandatory as we
1280 * can always just disable its use.
1285 dirty |= raw->fbc != wm;
1289 /* mark watermarks as invalid */
1290 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1292 if (plane_id == PLANE_PRIMARY)
1293 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1297 drm_dbg_kms(&dev_priv->drm,
1298 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1300 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1301 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1302 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1304 if (plane_id == PLANE_PRIMARY)
1305 drm_dbg_kms(&dev_priv->drm,
1306 "FBC watermarks: SR=%d, HPLL=%d\n",
1307 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1308 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1314 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1315 enum plane_id plane_id, int level)
1317 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1319 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1322 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1325 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1327 if (level > dev_priv->wm.max_level)
1330 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1331 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1332 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1335 /* mark all levels starting from 'level' as invalid */
1336 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1337 struct g4x_wm_state *wm_state, int level)
1339 if (level <= G4X_WM_LEVEL_NORMAL) {
1340 enum plane_id plane_id;
1342 for_each_plane_id_on_crtc(crtc, plane_id)
1343 wm_state->wm.plane[plane_id] = USHRT_MAX;
1346 if (level <= G4X_WM_LEVEL_SR) {
1347 wm_state->cxsr = false;
1348 wm_state->sr.cursor = USHRT_MAX;
1349 wm_state->sr.plane = USHRT_MAX;
1350 wm_state->sr.fbc = USHRT_MAX;
1353 if (level <= G4X_WM_LEVEL_HPLL) {
1354 wm_state->hpll_en = false;
1355 wm_state->hpll.cursor = USHRT_MAX;
1356 wm_state->hpll.plane = USHRT_MAX;
1357 wm_state->hpll.fbc = USHRT_MAX;
1361 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1364 if (level < G4X_WM_LEVEL_SR)
1367 if (level >= G4X_WM_LEVEL_SR &&
1368 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1371 if (level >= G4X_WM_LEVEL_HPLL &&
1372 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1378 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1379 struct intel_crtc *crtc)
1381 struct intel_crtc_state *crtc_state =
1382 intel_atomic_get_new_crtc_state(state, crtc);
1383 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1384 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1385 const struct g4x_pipe_wm *raw;
1386 const struct intel_plane_state *old_plane_state;
1387 const struct intel_plane_state *new_plane_state;
1388 struct intel_plane *plane;
1389 enum plane_id plane_id;
1391 unsigned int dirty = 0;
1393 for_each_oldnew_intel_plane_in_state(state, plane,
1395 new_plane_state, i) {
1396 if (new_plane_state->hw.crtc != &crtc->base &&
1397 old_plane_state->hw.crtc != &crtc->base)
1400 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1401 dirty |= BIT(plane->id);
1407 level = G4X_WM_LEVEL_NORMAL;
1408 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1411 raw = &crtc_state->wm.g4x.raw[level];
1412 for_each_plane_id_on_crtc(crtc, plane_id)
1413 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1415 level = G4X_WM_LEVEL_SR;
1416 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1419 raw = &crtc_state->wm.g4x.raw[level];
1420 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1421 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1422 wm_state->sr.fbc = raw->fbc;
1424 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1426 level = G4X_WM_LEVEL_HPLL;
1427 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1430 raw = &crtc_state->wm.g4x.raw[level];
1431 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1432 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1433 wm_state->hpll.fbc = raw->fbc;
1435 wm_state->hpll_en = wm_state->cxsr;
1440 if (level == G4X_WM_LEVEL_NORMAL)
1443 /* invalidate the higher levels */
1444 g4x_invalidate_wms(crtc, wm_state, level);
1447 * Determine if the FBC watermark(s) can be used. IF
1448 * this isn't the case we prefer to disable the FBC
1449 * watermark(s) rather than disable the SR/HPLL
1450 * level(s) entirely. 'level-1' is the highest valid
1453 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1458 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1459 struct intel_crtc *crtc)
1461 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1462 struct intel_crtc_state *new_crtc_state =
1463 intel_atomic_get_new_crtc_state(state, crtc);
1464 const struct intel_crtc_state *old_crtc_state =
1465 intel_atomic_get_old_crtc_state(state, crtc);
1466 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1467 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1468 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1469 enum plane_id plane_id;
1471 if (!new_crtc_state->hw.active ||
1472 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1473 *intermediate = *optimal;
1475 intermediate->cxsr = false;
1476 intermediate->hpll_en = false;
1480 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1481 !new_crtc_state->disable_cxsr;
1482 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1483 !new_crtc_state->disable_cxsr;
1484 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1486 for_each_plane_id_on_crtc(crtc, plane_id) {
1487 intermediate->wm.plane[plane_id] =
1488 max(optimal->wm.plane[plane_id],
1489 active->wm.plane[plane_id]);
1491 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1492 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1495 intermediate->sr.plane = max(optimal->sr.plane,
1497 intermediate->sr.cursor = max(optimal->sr.cursor,
1499 intermediate->sr.fbc = max(optimal->sr.fbc,
1502 intermediate->hpll.plane = max(optimal->hpll.plane,
1503 active->hpll.plane);
1504 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1505 active->hpll.cursor);
1506 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1509 drm_WARN_ON(&dev_priv->drm,
1510 (intermediate->sr.plane >
1511 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1512 intermediate->sr.cursor >
1513 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1514 intermediate->cxsr);
1515 drm_WARN_ON(&dev_priv->drm,
1516 (intermediate->sr.plane >
1517 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1518 intermediate->sr.cursor >
1519 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1520 intermediate->hpll_en);
1522 drm_WARN_ON(&dev_priv->drm,
1523 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1524 intermediate->fbc_en && intermediate->cxsr);
1525 drm_WARN_ON(&dev_priv->drm,
1526 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1527 intermediate->fbc_en && intermediate->hpll_en);
1531 * If our intermediate WM are identical to the final WM, then we can
1532 * omit the post-vblank programming; only update if it's different.
1534 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1535 new_crtc_state->wm.need_postvbl_update = true;
1540 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1541 struct g4x_wm_values *wm)
1543 struct intel_crtc *crtc;
1544 int num_active_pipes = 0;
1550 for_each_intel_crtc(&dev_priv->drm, crtc) {
1551 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1556 if (!wm_state->cxsr)
1558 if (!wm_state->hpll_en)
1559 wm->hpll_en = false;
1560 if (!wm_state->fbc_en)
1566 if (num_active_pipes != 1) {
1568 wm->hpll_en = false;
1572 for_each_intel_crtc(&dev_priv->drm, crtc) {
1573 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1574 enum pipe pipe = crtc->pipe;
1576 wm->pipe[pipe] = wm_state->wm;
1577 if (crtc->active && wm->cxsr)
1578 wm->sr = wm_state->sr;
1579 if (crtc->active && wm->hpll_en)
1580 wm->hpll = wm_state->hpll;
1584 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1586 struct g4x_wm_values *old_wm = &dev_priv->wm.g4x;
1587 struct g4x_wm_values new_wm = {};
1589 g4x_merge_wm(dev_priv, &new_wm);
1591 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1594 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1595 _intel_set_memory_cxsr(dev_priv, false);
1597 g4x_write_wm_values(dev_priv, &new_wm);
1599 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1600 _intel_set_memory_cxsr(dev_priv, true);
1605 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1606 struct intel_crtc *crtc)
1608 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1609 const struct intel_crtc_state *crtc_state =
1610 intel_atomic_get_new_crtc_state(state, crtc);
1612 mutex_lock(&dev_priv->wm.wm_mutex);
1613 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1614 g4x_program_watermarks(dev_priv);
1615 mutex_unlock(&dev_priv->wm.wm_mutex);
1618 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1619 struct intel_crtc *crtc)
1621 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1622 const struct intel_crtc_state *crtc_state =
1623 intel_atomic_get_new_crtc_state(state, crtc);
1625 if (!crtc_state->wm.need_postvbl_update)
1628 mutex_lock(&dev_priv->wm.wm_mutex);
1629 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1630 g4x_program_watermarks(dev_priv);
1631 mutex_unlock(&dev_priv->wm.wm_mutex);
1634 /* latency must be in 0.1us units. */
1635 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1636 unsigned int htotal,
1639 unsigned int latency)
1643 ret = intel_wm_method2(pixel_rate, htotal,
1644 width, cpp, latency);
1645 ret = DIV_ROUND_UP(ret, 64);
1650 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1652 /* all latencies in usec */
1653 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1655 dev_priv->wm.max_level = VLV_WM_LEVEL_PM2;
1657 if (IS_CHERRYVIEW(dev_priv)) {
1658 dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1659 dev_priv->wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1661 dev_priv->wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1665 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1666 const struct intel_plane_state *plane_state,
1669 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1670 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1671 const struct drm_display_mode *pipe_mode =
1672 &crtc_state->hw.pipe_mode;
1673 unsigned int pixel_rate, htotal, cpp, width, wm;
1675 if (dev_priv->wm.pri_latency[level] == 0)
1678 if (!intel_wm_plane_visible(crtc_state, plane_state))
1681 cpp = plane_state->hw.fb->format->cpp[0];
1682 pixel_rate = crtc_state->pixel_rate;
1683 htotal = pipe_mode->crtc_htotal;
1684 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1686 if (plane->id == PLANE_CURSOR) {
1688 * FIXME the formula gives values that are
1689 * too big for the cursor FIFO, and hence we
1690 * would never be able to use cursors. For
1691 * now just hardcode the watermark.
1695 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1696 dev_priv->wm.pri_latency[level] * 10);
1699 return min_t(unsigned int, wm, USHRT_MAX);
1702 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1704 return (active_planes & (BIT(PLANE_SPRITE0) |
1705 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1708 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1710 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1711 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1712 const struct g4x_pipe_wm *raw =
1713 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1714 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1715 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1716 int num_active_planes = hweight8(active_planes);
1717 const int fifo_size = 511;
1718 int fifo_extra, fifo_left = fifo_size;
1719 int sprite0_fifo_extra = 0;
1720 unsigned int total_rate;
1721 enum plane_id plane_id;
1724 * When enabling sprite0 after sprite1 has already been enabled
1725 * we tend to get an underrun unless sprite0 already has some
1726 * FIFO space allcoated. Hence we always allocate at least one
1727 * cacheline for sprite0 whenever sprite1 is enabled.
1729 * All other plane enable sequences appear immune to this problem.
1731 if (vlv_need_sprite0_fifo_workaround(active_planes))
1732 sprite0_fifo_extra = 1;
1734 total_rate = raw->plane[PLANE_PRIMARY] +
1735 raw->plane[PLANE_SPRITE0] +
1736 raw->plane[PLANE_SPRITE1] +
1739 if (total_rate > fifo_size)
1742 if (total_rate == 0)
1745 for_each_plane_id_on_crtc(crtc, plane_id) {
1748 if ((active_planes & BIT(plane_id)) == 0) {
1749 fifo_state->plane[plane_id] = 0;
1753 rate = raw->plane[plane_id];
1754 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1755 fifo_left -= fifo_state->plane[plane_id];
1758 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1759 fifo_left -= sprite0_fifo_extra;
1761 fifo_state->plane[PLANE_CURSOR] = 63;
1763 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1765 /* spread the remainder evenly */
1766 for_each_plane_id_on_crtc(crtc, plane_id) {
1772 if ((active_planes & BIT(plane_id)) == 0)
1775 plane_extra = min(fifo_extra, fifo_left);
1776 fifo_state->plane[plane_id] += plane_extra;
1777 fifo_left -= plane_extra;
1780 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1782 /* give it all to the first plane if none are active */
1783 if (active_planes == 0) {
1784 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1785 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1791 /* mark all levels starting from 'level' as invalid */
1792 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1793 struct vlv_wm_state *wm_state, int level)
1795 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1797 for (; level < intel_wm_num_levels(dev_priv); level++) {
1798 enum plane_id plane_id;
1800 for_each_plane_id_on_crtc(crtc, plane_id)
1801 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1803 wm_state->sr[level].cursor = USHRT_MAX;
1804 wm_state->sr[level].plane = USHRT_MAX;
1808 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1813 return fifo_size - wm;
1817 * Starting from 'level' set all higher
1818 * levels to 'value' in the "raw" watermarks.
1820 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1821 int level, enum plane_id plane_id, u16 value)
1823 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1824 int num_levels = intel_wm_num_levels(dev_priv);
1827 for (; level < num_levels; level++) {
1828 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1830 dirty |= raw->plane[plane_id] != value;
1831 raw->plane[plane_id] = value;
1837 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1838 const struct intel_plane_state *plane_state)
1840 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1841 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1842 enum plane_id plane_id = plane->id;
1843 int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1847 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1848 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1852 for (level = 0; level < num_levels; level++) {
1853 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1854 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1855 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1860 dirty |= raw->plane[plane_id] != wm;
1861 raw->plane[plane_id] = wm;
1864 /* mark all higher levels as invalid */
1865 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1869 drm_dbg_kms(&dev_priv->drm,
1870 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1872 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1873 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1874 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1879 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1880 enum plane_id plane_id, int level)
1882 const struct g4x_pipe_wm *raw =
1883 &crtc_state->wm.vlv.raw[level];
1884 const struct vlv_fifo_state *fifo_state =
1885 &crtc_state->wm.vlv.fifo_state;
1887 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1890 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1892 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1893 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1894 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1895 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1898 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1899 struct intel_crtc *crtc)
1901 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1902 struct intel_crtc_state *crtc_state =
1903 intel_atomic_get_new_crtc_state(state, crtc);
1904 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1905 const struct vlv_fifo_state *fifo_state =
1906 &crtc_state->wm.vlv.fifo_state;
1907 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1908 int num_active_planes = hweight8(active_planes);
1909 bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1910 const struct intel_plane_state *old_plane_state;
1911 const struct intel_plane_state *new_plane_state;
1912 struct intel_plane *plane;
1913 enum plane_id plane_id;
1915 unsigned int dirty = 0;
1917 for_each_oldnew_intel_plane_in_state(state, plane,
1919 new_plane_state, i) {
1920 if (new_plane_state->hw.crtc != &crtc->base &&
1921 old_plane_state->hw.crtc != &crtc->base)
1924 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1925 dirty |= BIT(plane->id);
1929 * DSPARB registers may have been reset due to the
1930 * power well being turned off. Make sure we restore
1931 * them to a consistent state even if no primary/sprite
1932 * planes are initially active.
1935 crtc_state->fifo_changed = true;
1940 /* cursor changes don't warrant a FIFO recompute */
1941 if (dirty & ~BIT(PLANE_CURSOR)) {
1942 const struct intel_crtc_state *old_crtc_state =
1943 intel_atomic_get_old_crtc_state(state, crtc);
1944 const struct vlv_fifo_state *old_fifo_state =
1945 &old_crtc_state->wm.vlv.fifo_state;
1947 ret = vlv_compute_fifo(crtc_state);
1951 if (needs_modeset ||
1952 memcmp(old_fifo_state, fifo_state,
1953 sizeof(*fifo_state)) != 0)
1954 crtc_state->fifo_changed = true;
1957 /* initially allow all levels */
1958 wm_state->num_levels = intel_wm_num_levels(dev_priv);
1960 * Note that enabling cxsr with no primary/sprite planes
1961 * enabled can wedge the pipe. Hence we only allow cxsr
1962 * with exactly one enabled primary/sprite plane.
1964 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1966 for (level = 0; level < wm_state->num_levels; level++) {
1967 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1968 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1970 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1973 for_each_plane_id_on_crtc(crtc, plane_id) {
1974 wm_state->wm[level].plane[plane_id] =
1975 vlv_invert_wm_value(raw->plane[plane_id],
1976 fifo_state->plane[plane_id]);
1979 wm_state->sr[level].plane =
1980 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1981 raw->plane[PLANE_SPRITE0],
1982 raw->plane[PLANE_SPRITE1]),
1985 wm_state->sr[level].cursor =
1986 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1993 /* limit to only levels we can actually handle */
1994 wm_state->num_levels = level;
1996 /* invalidate the higher levels */
1997 vlv_invalidate_wms(crtc, wm_state, level);
2002 #define VLV_FIFO(plane, value) \
2003 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
2005 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
2006 struct intel_crtc *crtc)
2008 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2009 struct intel_uncore *uncore = &dev_priv->uncore;
2010 const struct intel_crtc_state *crtc_state =
2011 intel_atomic_get_new_crtc_state(state, crtc);
2012 const struct vlv_fifo_state *fifo_state =
2013 &crtc_state->wm.vlv.fifo_state;
2014 int sprite0_start, sprite1_start, fifo_size;
2015 u32 dsparb, dsparb2, dsparb3;
2017 if (!crtc_state->fifo_changed)
2020 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2021 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2022 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2024 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2025 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2027 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2030 * uncore.lock serves a double purpose here. It allows us to
2031 * use the less expensive I915_{READ,WRITE}_FW() functions, and
2032 * it protects the DSPARB registers from getting clobbered by
2033 * parallel updates from multiple pipes.
2035 * intel_pipe_update_start() has already disabled interrupts
2036 * for us, so a plain spin_lock() is sufficient here.
2038 spin_lock(&uncore->lock);
2040 switch (crtc->pipe) {
2042 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2043 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2045 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2046 VLV_FIFO(SPRITEB, 0xff));
2047 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2048 VLV_FIFO(SPRITEB, sprite1_start));
2050 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2051 VLV_FIFO(SPRITEB_HI, 0x1));
2052 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2053 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2055 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2056 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2059 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2060 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2062 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2063 VLV_FIFO(SPRITED, 0xff));
2064 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2065 VLV_FIFO(SPRITED, sprite1_start));
2067 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2068 VLV_FIFO(SPRITED_HI, 0xff));
2069 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2070 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2072 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2073 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2076 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2077 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2079 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2080 VLV_FIFO(SPRITEF, 0xff));
2081 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2082 VLV_FIFO(SPRITEF, sprite1_start));
2084 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2085 VLV_FIFO(SPRITEF_HI, 0xff));
2086 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2087 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2089 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2090 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2096 intel_uncore_posting_read_fw(uncore, DSPARB);
2098 spin_unlock(&uncore->lock);
2103 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2104 struct intel_crtc *crtc)
2106 struct intel_crtc_state *new_crtc_state =
2107 intel_atomic_get_new_crtc_state(state, crtc);
2108 const struct intel_crtc_state *old_crtc_state =
2109 intel_atomic_get_old_crtc_state(state, crtc);
2110 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2111 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2112 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2115 if (!new_crtc_state->hw.active ||
2116 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2117 *intermediate = *optimal;
2119 intermediate->cxsr = false;
2123 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2124 intermediate->cxsr = optimal->cxsr && active->cxsr &&
2125 !new_crtc_state->disable_cxsr;
2127 for (level = 0; level < intermediate->num_levels; level++) {
2128 enum plane_id plane_id;
2130 for_each_plane_id_on_crtc(crtc, plane_id) {
2131 intermediate->wm[level].plane[plane_id] =
2132 min(optimal->wm[level].plane[plane_id],
2133 active->wm[level].plane[plane_id]);
2136 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2137 active->sr[level].plane);
2138 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2139 active->sr[level].cursor);
2142 vlv_invalidate_wms(crtc, intermediate, level);
2146 * If our intermediate WM are identical to the final WM, then we can
2147 * omit the post-vblank programming; only update if it's different.
2149 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2150 new_crtc_state->wm.need_postvbl_update = true;
2155 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2156 struct vlv_wm_values *wm)
2158 struct intel_crtc *crtc;
2159 int num_active_pipes = 0;
2161 wm->level = dev_priv->wm.max_level;
2164 for_each_intel_crtc(&dev_priv->drm, crtc) {
2165 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2170 if (!wm_state->cxsr)
2174 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2177 if (num_active_pipes != 1)
2180 if (num_active_pipes > 1)
2181 wm->level = VLV_WM_LEVEL_PM2;
2183 for_each_intel_crtc(&dev_priv->drm, crtc) {
2184 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2185 enum pipe pipe = crtc->pipe;
2187 wm->pipe[pipe] = wm_state->wm[wm->level];
2188 if (crtc->active && wm->cxsr)
2189 wm->sr = wm_state->sr[wm->level];
2191 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2192 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2193 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2194 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2198 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2200 struct vlv_wm_values *old_wm = &dev_priv->wm.vlv;
2201 struct vlv_wm_values new_wm = {};
2203 vlv_merge_wm(dev_priv, &new_wm);
2205 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2208 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2209 chv_set_memory_dvfs(dev_priv, false);
2211 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2212 chv_set_memory_pm5(dev_priv, false);
2214 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2215 _intel_set_memory_cxsr(dev_priv, false);
2217 vlv_write_wm_values(dev_priv, &new_wm);
2219 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2220 _intel_set_memory_cxsr(dev_priv, true);
2222 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2223 chv_set_memory_pm5(dev_priv, true);
2225 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2226 chv_set_memory_dvfs(dev_priv, true);
2231 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2232 struct intel_crtc *crtc)
2234 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2235 const struct intel_crtc_state *crtc_state =
2236 intel_atomic_get_new_crtc_state(state, crtc);
2238 mutex_lock(&dev_priv->wm.wm_mutex);
2239 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2240 vlv_program_watermarks(dev_priv);
2241 mutex_unlock(&dev_priv->wm.wm_mutex);
2244 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2245 struct intel_crtc *crtc)
2247 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2248 const struct intel_crtc_state *crtc_state =
2249 intel_atomic_get_new_crtc_state(state, crtc);
2251 if (!crtc_state->wm.need_postvbl_update)
2254 mutex_lock(&dev_priv->wm.wm_mutex);
2255 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2256 vlv_program_watermarks(dev_priv);
2257 mutex_unlock(&dev_priv->wm.wm_mutex);
2260 static void i965_update_wm(struct drm_i915_private *dev_priv)
2262 struct intel_crtc *crtc;
2267 /* Calc sr entries for one plane configs */
2268 crtc = single_enabled_crtc(dev_priv);
2270 /* self-refresh has much higher latency */
2271 static const int sr_latency_ns = 12000;
2272 const struct drm_display_mode *pipe_mode =
2273 &crtc->config->hw.pipe_mode;
2274 const struct drm_framebuffer *fb =
2275 crtc->base.primary->state->fb;
2276 int pixel_rate = crtc->config->pixel_rate;
2277 int htotal = pipe_mode->crtc_htotal;
2278 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2279 int cpp = fb->format->cpp[0];
2282 entries = intel_wm_method2(pixel_rate, htotal,
2283 width, cpp, sr_latency_ns / 100);
2284 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2285 srwm = I965_FIFO_SIZE - entries;
2289 drm_dbg_kms(&dev_priv->drm,
2290 "self-refresh entries: %d, wm: %d\n",
2293 entries = intel_wm_method2(pixel_rate, htotal,
2294 crtc->base.cursor->state->crtc_w, 4,
2295 sr_latency_ns / 100);
2296 entries = DIV_ROUND_UP(entries,
2297 i965_cursor_wm_info.cacheline_size) +
2298 i965_cursor_wm_info.guard_size;
2300 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2301 if (cursor_sr > i965_cursor_wm_info.max_wm)
2302 cursor_sr = i965_cursor_wm_info.max_wm;
2304 drm_dbg_kms(&dev_priv->drm,
2305 "self-refresh watermark: display plane %d "
2306 "cursor %d\n", srwm, cursor_sr);
2308 cxsr_enabled = true;
2310 cxsr_enabled = false;
2311 /* Turn off self refresh if both pipes are enabled */
2312 intel_set_memory_cxsr(dev_priv, false);
2315 drm_dbg_kms(&dev_priv->drm,
2316 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2319 /* 965 has limitations... */
2320 intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2324 intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2325 FW_WM(8, PLANEC_OLD));
2326 /* update cursor SR watermark */
2327 intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2330 intel_set_memory_cxsr(dev_priv, true);
2335 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2336 enum i9xx_plane_id i9xx_plane)
2338 struct intel_plane *plane;
2340 for_each_intel_plane(&i915->drm, plane) {
2341 if (plane->id == PLANE_PRIMARY &&
2342 plane->i9xx_plane == i9xx_plane)
2343 return intel_crtc_for_pipe(i915, plane->pipe);
2349 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2351 const struct intel_watermark_params *wm_info;
2356 int planea_wm, planeb_wm;
2357 struct intel_crtc *crtc;
2359 if (IS_I945GM(dev_priv))
2360 wm_info = &i945_wm_info;
2361 else if (DISPLAY_VER(dev_priv) != 2)
2362 wm_info = &i915_wm_info;
2364 wm_info = &i830_a_wm_info;
2366 if (DISPLAY_VER(dev_priv) == 2)
2367 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2369 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2370 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2371 if (intel_crtc_active(crtc)) {
2372 const struct drm_framebuffer *fb =
2373 crtc->base.primary->state->fb;
2376 if (DISPLAY_VER(dev_priv) == 2)
2379 cpp = fb->format->cpp[0];
2381 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2382 wm_info, fifo_size, cpp,
2383 pessimal_latency_ns);
2385 planea_wm = fifo_size - wm_info->guard_size;
2386 if (planea_wm > (long)wm_info->max_wm)
2387 planea_wm = wm_info->max_wm;
2390 if (DISPLAY_VER(dev_priv) == 2)
2391 wm_info = &i830_bc_wm_info;
2393 if (DISPLAY_VER(dev_priv) == 2)
2394 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2396 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2397 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2398 if (intel_crtc_active(crtc)) {
2399 const struct drm_framebuffer *fb =
2400 crtc->base.primary->state->fb;
2403 if (DISPLAY_VER(dev_priv) == 2)
2406 cpp = fb->format->cpp[0];
2408 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2409 wm_info, fifo_size, cpp,
2410 pessimal_latency_ns);
2412 planeb_wm = fifo_size - wm_info->guard_size;
2413 if (planeb_wm > (long)wm_info->max_wm)
2414 planeb_wm = wm_info->max_wm;
2417 drm_dbg_kms(&dev_priv->drm,
2418 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2420 crtc = single_enabled_crtc(dev_priv);
2421 if (IS_I915GM(dev_priv) && crtc) {
2422 struct drm_i915_gem_object *obj;
2424 obj = intel_fb_obj(crtc->base.primary->state->fb);
2426 /* self-refresh seems busted with untiled */
2427 if (!i915_gem_object_is_tiled(obj))
2432 * Overlay gets an aggressive default since video jitter is bad.
2436 /* Play safe and disable self-refresh before adjusting watermarks. */
2437 intel_set_memory_cxsr(dev_priv, false);
2439 /* Calc sr entries for one plane configs */
2440 if (HAS_FW_BLC(dev_priv) && crtc) {
2441 /* self-refresh has much higher latency */
2442 static const int sr_latency_ns = 6000;
2443 const struct drm_display_mode *pipe_mode =
2444 &crtc->config->hw.pipe_mode;
2445 const struct drm_framebuffer *fb =
2446 crtc->base.primary->state->fb;
2447 int pixel_rate = crtc->config->pixel_rate;
2448 int htotal = pipe_mode->crtc_htotal;
2449 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2453 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2456 cpp = fb->format->cpp[0];
2458 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2459 sr_latency_ns / 100);
2460 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2461 drm_dbg_kms(&dev_priv->drm,
2462 "self-refresh entries: %d\n", entries);
2463 srwm = wm_info->fifo_size - entries;
2467 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2468 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2469 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2471 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2474 drm_dbg_kms(&dev_priv->drm,
2475 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2476 planea_wm, planeb_wm, cwm, srwm);
2478 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2479 fwater_hi = (cwm & 0x1f);
2481 /* Set request length to 8 cachelines per fetch */
2482 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2483 fwater_hi = fwater_hi | (1 << 8);
2485 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2486 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2489 intel_set_memory_cxsr(dev_priv, true);
2492 static void i845_update_wm(struct drm_i915_private *dev_priv)
2494 struct intel_crtc *crtc;
2498 crtc = single_enabled_crtc(dev_priv);
2502 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2504 i845_get_fifo_size(dev_priv, PLANE_A),
2505 4, pessimal_latency_ns);
2506 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2507 fwater_lo |= (3<<8) | planea_wm;
2509 drm_dbg_kms(&dev_priv->drm,
2510 "Setting FIFO watermarks - A: %d\n", planea_wm);
2512 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2515 /* latency must be in 0.1us units. */
2516 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2518 unsigned int latency)
2522 ret = intel_wm_method1(pixel_rate, cpp, latency);
2523 ret = DIV_ROUND_UP(ret, 64) + 2;
2528 /* latency must be in 0.1us units. */
2529 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2530 unsigned int htotal,
2533 unsigned int latency)
2537 ret = intel_wm_method2(pixel_rate, htotal,
2538 width, cpp, latency);
2539 ret = DIV_ROUND_UP(ret, 64) + 2;
2544 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2547 * Neither of these should be possible since this function shouldn't be
2548 * called if the CRTC is off or the plane is invisible. But let's be
2549 * extra paranoid to avoid a potential divide-by-zero if we screw up
2550 * elsewhere in the driver.
2554 if (WARN_ON(!horiz_pixels))
2557 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2560 struct ilk_wm_maximums {
2568 * For both WM_PIPE and WM_LP.
2569 * mem_value must be in 0.1us units.
2571 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2572 const struct intel_plane_state *plane_state,
2573 u32 mem_value, bool is_lp)
2575 u32 method1, method2;
2581 if (!intel_wm_plane_visible(crtc_state, plane_state))
2584 cpp = plane_state->hw.fb->format->cpp[0];
2586 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2591 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2592 crtc_state->hw.pipe_mode.crtc_htotal,
2593 drm_rect_width(&plane_state->uapi.src) >> 16,
2596 return min(method1, method2);
2600 * For both WM_PIPE and WM_LP.
2601 * mem_value must be in 0.1us units.
2603 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2604 const struct intel_plane_state *plane_state,
2607 u32 method1, method2;
2613 if (!intel_wm_plane_visible(crtc_state, plane_state))
2616 cpp = plane_state->hw.fb->format->cpp[0];
2618 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2619 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2620 crtc_state->hw.pipe_mode.crtc_htotal,
2621 drm_rect_width(&plane_state->uapi.src) >> 16,
2623 return min(method1, method2);
2627 * For both WM_PIPE and WM_LP.
2628 * mem_value must be in 0.1us units.
2630 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2631 const struct intel_plane_state *plane_state,
2639 if (!intel_wm_plane_visible(crtc_state, plane_state))
2642 cpp = plane_state->hw.fb->format->cpp[0];
2644 return ilk_wm_method2(crtc_state->pixel_rate,
2645 crtc_state->hw.pipe_mode.crtc_htotal,
2646 drm_rect_width(&plane_state->uapi.src) >> 16,
2650 /* Only for WM_LP. */
2651 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2652 const struct intel_plane_state *plane_state,
2657 if (!intel_wm_plane_visible(crtc_state, plane_state))
2660 cpp = plane_state->hw.fb->format->cpp[0];
2662 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2667 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2669 if (DISPLAY_VER(dev_priv) >= 8)
2671 else if (DISPLAY_VER(dev_priv) >= 7)
2678 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2679 int level, bool is_sprite)
2681 if (DISPLAY_VER(dev_priv) >= 8)
2682 /* BDW primary/sprite plane watermarks */
2683 return level == 0 ? 255 : 2047;
2684 else if (DISPLAY_VER(dev_priv) >= 7)
2685 /* IVB/HSW primary/sprite plane watermarks */
2686 return level == 0 ? 127 : 1023;
2687 else if (!is_sprite)
2688 /* ILK/SNB primary plane watermarks */
2689 return level == 0 ? 127 : 511;
2691 /* ILK/SNB sprite plane watermarks */
2692 return level == 0 ? 63 : 255;
2696 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2698 if (DISPLAY_VER(dev_priv) >= 7)
2699 return level == 0 ? 63 : 255;
2701 return level == 0 ? 31 : 63;
2704 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2706 if (DISPLAY_VER(dev_priv) >= 8)
2712 /* Calculate the maximum primary/sprite plane watermark */
2713 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2715 const struct intel_wm_config *config,
2716 enum intel_ddb_partitioning ddb_partitioning,
2719 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2721 /* if sprites aren't enabled, sprites get nothing */
2722 if (is_sprite && !config->sprites_enabled)
2725 /* HSW allows LP1+ watermarks even with multiple pipes */
2726 if (level == 0 || config->num_pipes_active > 1) {
2727 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2730 * For some reason the non self refresh
2731 * FIFO size is only half of the self
2732 * refresh FIFO size on ILK/SNB.
2734 if (DISPLAY_VER(dev_priv) <= 6)
2738 if (config->sprites_enabled) {
2739 /* level 0 is always calculated with 1:1 split */
2740 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2749 /* clamp to max that the registers can hold */
2750 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2753 /* Calculate the maximum cursor plane watermark */
2754 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2756 const struct intel_wm_config *config)
2758 /* HSW LP1+ watermarks w/ multiple pipes */
2759 if (level > 0 && config->num_pipes_active > 1)
2762 /* otherwise just report max that registers can hold */
2763 return ilk_cursor_wm_reg_max(dev_priv, level);
2766 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2768 const struct intel_wm_config *config,
2769 enum intel_ddb_partitioning ddb_partitioning,
2770 struct ilk_wm_maximums *max)
2772 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2773 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2774 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2775 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2778 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2780 struct ilk_wm_maximums *max)
2782 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2783 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2784 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2785 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2788 static bool ilk_validate_wm_level(int level,
2789 const struct ilk_wm_maximums *max,
2790 struct intel_wm_level *result)
2794 /* already determined to be invalid? */
2795 if (!result->enable)
2798 result->enable = result->pri_val <= max->pri &&
2799 result->spr_val <= max->spr &&
2800 result->cur_val <= max->cur;
2802 ret = result->enable;
2805 * HACK until we can pre-compute everything,
2806 * and thus fail gracefully if LP0 watermarks
2809 if (level == 0 && !result->enable) {
2810 if (result->pri_val > max->pri)
2811 DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2812 level, result->pri_val, max->pri);
2813 if (result->spr_val > max->spr)
2814 DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2815 level, result->spr_val, max->spr);
2816 if (result->cur_val > max->cur)
2817 DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2818 level, result->cur_val, max->cur);
2820 result->pri_val = min_t(u32, result->pri_val, max->pri);
2821 result->spr_val = min_t(u32, result->spr_val, max->spr);
2822 result->cur_val = min_t(u32, result->cur_val, max->cur);
2823 result->enable = true;
2829 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2830 const struct intel_crtc *crtc,
2832 struct intel_crtc_state *crtc_state,
2833 const struct intel_plane_state *pristate,
2834 const struct intel_plane_state *sprstate,
2835 const struct intel_plane_state *curstate,
2836 struct intel_wm_level *result)
2838 u16 pri_latency = dev_priv->wm.pri_latency[level];
2839 u16 spr_latency = dev_priv->wm.spr_latency[level];
2840 u16 cur_latency = dev_priv->wm.cur_latency[level];
2842 /* WM1+ latency values stored in 0.5us units */
2850 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2851 pri_latency, level);
2852 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2856 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2859 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2861 result->enable = true;
2864 static void intel_read_wm_latency(struct drm_i915_private *dev_priv,
2867 struct intel_uncore *uncore = &dev_priv->uncore;
2869 if (DISPLAY_VER(dev_priv) >= 9) {
2872 int level, max_level = ilk_wm_max_level(dev_priv);
2873 int mult = IS_DG2(dev_priv) ? 2 : 1;
2875 /* read the first set of memory latencies[0:3] */
2876 val = 0; /* data0 to be programmed to 0 for first set */
2877 ret = snb_pcode_read(dev_priv, GEN9_PCODE_READ_MEM_LATENCY,
2881 drm_err(&dev_priv->drm,
2882 "SKL Mailbox read error = %d\n", ret);
2886 wm[0] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2887 wm[1] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2888 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2889 wm[2] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2890 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2891 wm[3] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2892 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2894 /* read the second set of memory latencies[4:7] */
2895 val = 1; /* data0 to be programmed to 1 for second set */
2896 ret = snb_pcode_read(dev_priv, GEN9_PCODE_READ_MEM_LATENCY,
2899 drm_err(&dev_priv->drm,
2900 "SKL Mailbox read error = %d\n", ret);
2904 wm[4] = (val & GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2905 wm[5] = ((val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
2906 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2907 wm[6] = ((val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
2908 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2909 wm[7] = ((val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
2910 GEN9_MEM_LATENCY_LEVEL_MASK) * mult;
2913 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
2914 * need to be disabled. We make sure to sanitize the values out
2915 * of the punit to satisfy this requirement.
2917 for (level = 1; level <= max_level; level++) {
2918 if (wm[level] == 0) {
2919 for (i = level + 1; i <= max_level; i++)
2922 max_level = level - 1;
2929 * WaWmMemoryReadLatency
2931 * punit doesn't take into account the read latency so we need
2932 * to add proper adjustement to each valid level we retrieve
2933 * from the punit when level 0 response data is 0us.
2936 u8 adjust = DISPLAY_VER(dev_priv) >= 12 ? 3 : 2;
2938 for (level = 0; level <= max_level; level++)
2939 wm[level] += adjust;
2943 * WA Level-0 adjustment for 16GB DIMMs: SKL+
2944 * If we could not get dimm info enable this WA to prevent from
2945 * any underrun. If not able to get Dimm info assume 16GB dimm
2946 * to avoid any underrun.
2948 if (dev_priv->dram_info.wm_lv_0_adjust_needed)
2950 } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
2951 u64 sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
2953 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2955 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2956 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2957 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2958 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2959 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2960 } else if (DISPLAY_VER(dev_priv) >= 6) {
2961 u32 sskpd = intel_uncore_read(uncore, MCH_SSKPD);
2963 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2964 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2965 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2966 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2967 } else if (DISPLAY_VER(dev_priv) >= 5) {
2968 u32 mltr = intel_uncore_read(uncore, MLTR_ILK);
2970 /* ILK primary LP0 latency is 700 ns */
2972 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2973 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2975 MISSING_CASE(INTEL_DEVID(dev_priv));
2979 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2982 /* ILK sprite LP0 latency is 1300 ns */
2983 if (DISPLAY_VER(dev_priv) == 5)
2987 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2990 /* ILK cursor LP0 latency is 1300 ns */
2991 if (DISPLAY_VER(dev_priv) == 5)
2995 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2997 /* how many WM levels are we expecting */
2998 if (HAS_HW_SAGV_WM(dev_priv))
3000 else if (DISPLAY_VER(dev_priv) >= 9)
3002 else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3004 else if (DISPLAY_VER(dev_priv) >= 6)
3010 static void intel_print_wm_latency(struct drm_i915_private *dev_priv,
3014 int level, max_level = ilk_wm_max_level(dev_priv);
3016 for (level = 0; level <= max_level; level++) {
3017 unsigned int latency = wm[level];
3020 drm_dbg_kms(&dev_priv->drm,
3021 "%s WM%d latency not provided\n",
3027 * - latencies are in us on gen9.
3028 * - before then, WM1+ latency values are in 0.5us units
3030 if (DISPLAY_VER(dev_priv) >= 9)
3035 drm_dbg_kms(&dev_priv->drm,
3036 "%s WM%d latency %u (%u.%u usec)\n", name, level,
3037 wm[level], latency / 10, latency % 10);
3041 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
3044 int level, max_level = ilk_wm_max_level(dev_priv);
3049 wm[0] = max(wm[0], min);
3050 for (level = 1; level <= max_level; level++)
3051 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
3056 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
3061 * The BIOS provided WM memory latency values are often
3062 * inadequate for high resolution displays. Adjust them.
3064 changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12);
3065 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12);
3066 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
3071 drm_dbg_kms(&dev_priv->drm,
3072 "WM latency values increased to avoid potential underruns\n");
3073 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3074 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3075 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3078 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
3081 * On some SNB machines (Thinkpad X220 Tablet at least)
3082 * LP3 usage can cause vblank interrupts to be lost.
3083 * The DEIIR bit will go high but it looks like the CPU
3084 * never gets interrupted.
3086 * It's not clear whether other interrupt source could
3087 * be affected or if this is somehow limited to vblank
3088 * interrupts only. To play it safe we disable LP3
3089 * watermarks entirely.
3091 if (dev_priv->wm.pri_latency[3] == 0 &&
3092 dev_priv->wm.spr_latency[3] == 0 &&
3093 dev_priv->wm.cur_latency[3] == 0)
3096 dev_priv->wm.pri_latency[3] = 0;
3097 dev_priv->wm.spr_latency[3] = 0;
3098 dev_priv->wm.cur_latency[3] = 0;
3100 drm_dbg_kms(&dev_priv->drm,
3101 "LP3 watermarks disabled due to potential for lost interrupts\n");
3102 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3103 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3104 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3107 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3109 intel_read_wm_latency(dev_priv, dev_priv->wm.pri_latency);
3111 memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
3112 sizeof(dev_priv->wm.pri_latency));
3113 memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
3114 sizeof(dev_priv->wm.pri_latency));
3116 intel_fixup_spr_wm_latency(dev_priv, dev_priv->wm.spr_latency);
3117 intel_fixup_cur_wm_latency(dev_priv, dev_priv->wm.cur_latency);
3119 intel_print_wm_latency(dev_priv, "Primary", dev_priv->wm.pri_latency);
3120 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->wm.spr_latency);
3121 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->wm.cur_latency);
3123 if (DISPLAY_VER(dev_priv) == 6) {
3124 snb_wm_latency_quirk(dev_priv);
3125 snb_wm_lp3_irq_quirk(dev_priv);
3129 static void skl_setup_wm_latency(struct drm_i915_private *dev_priv)
3131 intel_read_wm_latency(dev_priv, dev_priv->wm.skl_latency);
3132 intel_print_wm_latency(dev_priv, "Gen9 Plane", dev_priv->wm.skl_latency);
3135 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3136 struct intel_pipe_wm *pipe_wm)
3138 /* LP0 watermark maximums depend on this pipe alone */
3139 const struct intel_wm_config config = {
3140 .num_pipes_active = 1,
3141 .sprites_enabled = pipe_wm->sprites_enabled,
3142 .sprites_scaled = pipe_wm->sprites_scaled,
3144 struct ilk_wm_maximums max;
3146 /* LP0 watermarks always use 1/2 DDB partitioning */
3147 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3149 /* At least LP0 must be valid */
3150 if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3151 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3158 /* Compute new watermarks for the pipe */
3159 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3160 struct intel_crtc *crtc)
3162 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3163 struct intel_crtc_state *crtc_state =
3164 intel_atomic_get_new_crtc_state(state, crtc);
3165 struct intel_pipe_wm *pipe_wm;
3166 struct intel_plane *plane;
3167 const struct intel_plane_state *plane_state;
3168 const struct intel_plane_state *pristate = NULL;
3169 const struct intel_plane_state *sprstate = NULL;
3170 const struct intel_plane_state *curstate = NULL;
3171 int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3172 struct ilk_wm_maximums max;
3174 pipe_wm = &crtc_state->wm.ilk.optimal;
3176 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3177 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3178 pristate = plane_state;
3179 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3180 sprstate = plane_state;
3181 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3182 curstate = plane_state;
3185 pipe_wm->pipe_enabled = crtc_state->hw.active;
3186 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3187 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3189 usable_level = max_level;
3191 /* ILK/SNB: LP2+ watermarks only w/o sprites */
3192 if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3195 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3196 if (pipe_wm->sprites_scaled)
3199 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3200 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3201 pristate, sprstate, curstate, &pipe_wm->wm[0]);
3203 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3206 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3208 for (level = 1; level <= usable_level; level++) {
3209 struct intel_wm_level *wm = &pipe_wm->wm[level];
3211 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3212 pristate, sprstate, curstate, wm);
3215 * Disable any watermark level that exceeds the
3216 * register maximums since such watermarks are
3219 if (!ilk_validate_wm_level(level, &max, wm)) {
3220 memset(wm, 0, sizeof(*wm));
3229 * Build a set of 'intermediate' watermark values that satisfy both the old
3230 * state and the new state. These can be programmed to the hardware
3233 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3234 struct intel_crtc *crtc)
3236 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3237 struct intel_crtc_state *new_crtc_state =
3238 intel_atomic_get_new_crtc_state(state, crtc);
3239 const struct intel_crtc_state *old_crtc_state =
3240 intel_atomic_get_old_crtc_state(state, crtc);
3241 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3242 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3243 int level, max_level = ilk_wm_max_level(dev_priv);
3246 * Start with the final, target watermarks, then combine with the
3247 * currently active watermarks to get values that are safe both before
3248 * and after the vblank.
3250 *a = new_crtc_state->wm.ilk.optimal;
3251 if (!new_crtc_state->hw.active ||
3252 drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) ||
3253 state->skip_intermediate_wm)
3256 a->pipe_enabled |= b->pipe_enabled;
3257 a->sprites_enabled |= b->sprites_enabled;
3258 a->sprites_scaled |= b->sprites_scaled;
3260 for (level = 0; level <= max_level; level++) {
3261 struct intel_wm_level *a_wm = &a->wm[level];
3262 const struct intel_wm_level *b_wm = &b->wm[level];
3264 a_wm->enable &= b_wm->enable;
3265 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3266 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3267 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3268 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3272 * We need to make sure that these merged watermark values are
3273 * actually a valid configuration themselves. If they're not,
3274 * there's no safe way to transition from the old state to
3275 * the new state, so we need to fail the atomic transaction.
3277 if (!ilk_validate_pipe_wm(dev_priv, a))
3281 * If our intermediate WM are identical to the final WM, then we can
3282 * omit the post-vblank programming; only update if it's different.
3284 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3285 new_crtc_state->wm.need_postvbl_update = true;
3291 * Merge the watermarks from all active pipes for a specific level.
3293 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3295 struct intel_wm_level *ret_wm)
3297 const struct intel_crtc *crtc;
3299 ret_wm->enable = true;
3301 for_each_intel_crtc(&dev_priv->drm, crtc) {
3302 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3303 const struct intel_wm_level *wm = &active->wm[level];
3305 if (!active->pipe_enabled)
3309 * The watermark values may have been used in the past,
3310 * so we must maintain them in the registers for some
3311 * time even if the level is now disabled.
3314 ret_wm->enable = false;
3316 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3317 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3318 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3319 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3324 * Merge all low power watermarks for all active pipes.
3326 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3327 const struct intel_wm_config *config,
3328 const struct ilk_wm_maximums *max,
3329 struct intel_pipe_wm *merged)
3331 int level, max_level = ilk_wm_max_level(dev_priv);
3332 int last_enabled_level = max_level;
3334 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3335 if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3336 config->num_pipes_active > 1)
3337 last_enabled_level = 0;
3339 /* ILK: FBC WM must be disabled always */
3340 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3342 /* merge each WM1+ level */
3343 for (level = 1; level <= max_level; level++) {
3344 struct intel_wm_level *wm = &merged->wm[level];
3346 ilk_merge_wm_level(dev_priv, level, wm);
3348 if (level > last_enabled_level)
3350 else if (!ilk_validate_wm_level(level, max, wm))
3351 /* make sure all following levels get disabled */
3352 last_enabled_level = level - 1;
3355 * The spec says it is preferred to disable
3356 * FBC WMs instead of disabling a WM level.
3358 if (wm->fbc_val > max->fbc) {
3360 merged->fbc_wm_enabled = false;
3365 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3366 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3367 dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3368 for (level = 2; level <= max_level; level++) {
3369 struct intel_wm_level *wm = &merged->wm[level];
3376 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3378 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3379 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3382 /* The value we need to program into the WM_LPx latency field */
3383 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3386 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3389 return dev_priv->wm.pri_latency[level];
3392 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3393 const struct intel_pipe_wm *merged,
3394 enum intel_ddb_partitioning partitioning,
3395 struct ilk_wm_values *results)
3397 struct intel_crtc *crtc;
3400 results->enable_fbc_wm = merged->fbc_wm_enabled;
3401 results->partitioning = partitioning;
3403 /* LP1+ register values */
3404 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3405 const struct intel_wm_level *r;
3407 level = ilk_wm_lp_to_level(wm_lp, merged);
3409 r = &merged->wm[level];
3412 * Maintain the watermark values even if the level is
3413 * disabled. Doing otherwise could cause underruns.
3415 results->wm_lp[wm_lp - 1] =
3416 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3417 WM_LP_PRIMARY(r->pri_val) |
3418 WM_LP_CURSOR(r->cur_val);
3421 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3423 if (DISPLAY_VER(dev_priv) >= 8)
3424 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3426 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3428 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3431 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3432 * level is disabled. Doing otherwise could cause underruns.
3434 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3435 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3436 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3440 /* LP0 register values */
3441 for_each_intel_crtc(&dev_priv->drm, crtc) {
3442 enum pipe pipe = crtc->pipe;
3443 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3444 const struct intel_wm_level *r = &pipe_wm->wm[0];
3446 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3449 results->wm_pipe[pipe] =
3450 WM0_PIPE_PRIMARY(r->pri_val) |
3451 WM0_PIPE_SPRITE(r->spr_val) |
3452 WM0_PIPE_CURSOR(r->cur_val);
3456 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3457 * case both are at the same level. Prefer r1 in case they're the same. */
3458 static struct intel_pipe_wm *
3459 ilk_find_best_result(struct drm_i915_private *dev_priv,
3460 struct intel_pipe_wm *r1,
3461 struct intel_pipe_wm *r2)
3463 int level, max_level = ilk_wm_max_level(dev_priv);
3464 int level1 = 0, level2 = 0;
3466 for (level = 1; level <= max_level; level++) {
3467 if (r1->wm[level].enable)
3469 if (r2->wm[level].enable)
3473 if (level1 == level2) {
3474 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3478 } else if (level1 > level2) {
3485 /* dirty bits used to track which watermarks need changes */
3486 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3487 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3488 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3489 #define WM_DIRTY_FBC (1 << 24)
3490 #define WM_DIRTY_DDB (1 << 25)
3492 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3493 const struct ilk_wm_values *old,
3494 const struct ilk_wm_values *new)
3496 unsigned int dirty = 0;
3500 for_each_pipe(dev_priv, pipe) {
3501 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3502 dirty |= WM_DIRTY_PIPE(pipe);
3503 /* Must disable LP1+ watermarks too */
3504 dirty |= WM_DIRTY_LP_ALL;
3508 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3509 dirty |= WM_DIRTY_FBC;
3510 /* Must disable LP1+ watermarks too */
3511 dirty |= WM_DIRTY_LP_ALL;
3514 if (old->partitioning != new->partitioning) {
3515 dirty |= WM_DIRTY_DDB;
3516 /* Must disable LP1+ watermarks too */
3517 dirty |= WM_DIRTY_LP_ALL;
3520 /* LP1+ watermarks already deemed dirty, no need to continue */
3521 if (dirty & WM_DIRTY_LP_ALL)
3524 /* Find the lowest numbered LP1+ watermark in need of an update... */
3525 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3526 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3527 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3531 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3532 for (; wm_lp <= 3; wm_lp++)
3533 dirty |= WM_DIRTY_LP(wm_lp);
3538 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3541 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3542 bool changed = false;
3544 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3545 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3546 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3549 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3550 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3551 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3554 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3555 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3556 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3561 * Don't touch WM_LP_SPRITE_ENABLE here.
3562 * Doing so could cause underruns.
3569 * The spec says we shouldn't write when we don't need, because every write
3570 * causes WMs to be re-evaluated, expending some power.
3572 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3573 struct ilk_wm_values *results)
3575 struct ilk_wm_values *previous = &dev_priv->wm.hw;
3579 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3583 _ilk_disable_lp_wm(dev_priv, dirty);
3585 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3586 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3587 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3588 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3589 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3590 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3592 if (dirty & WM_DIRTY_DDB) {
3593 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
3594 val = intel_uncore_read(&dev_priv->uncore, WM_MISC);
3595 if (results->partitioning == INTEL_DDB_PART_1_2)
3596 val &= ~WM_MISC_DATA_PARTITION_5_6;
3598 val |= WM_MISC_DATA_PARTITION_5_6;
3599 intel_uncore_write(&dev_priv->uncore, WM_MISC, val);
3601 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
3602 if (results->partitioning == INTEL_DDB_PART_1_2)
3603 val &= ~DISP_DATA_PARTITION_5_6;
3605 val |= DISP_DATA_PARTITION_5_6;
3606 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
3610 if (dirty & WM_DIRTY_FBC) {
3611 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL);
3612 if (results->enable_fbc_wm)
3613 val &= ~DISP_FBC_WM_DIS;
3615 val |= DISP_FBC_WM_DIS;
3616 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, val);
3619 if (dirty & WM_DIRTY_LP(1) &&
3620 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3621 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3623 if (DISPLAY_VER(dev_priv) >= 7) {
3624 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3625 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3626 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3627 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3630 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3631 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3632 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3633 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3634 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3635 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3637 dev_priv->wm.hw = *results;
3640 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3642 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3645 u8 intel_enabled_dbuf_slices_mask(struct drm_i915_private *dev_priv)
3647 u8 enabled_slices = 0;
3648 enum dbuf_slice slice;
3650 for_each_dbuf_slice(dev_priv, slice) {
3651 if (intel_uncore_read(&dev_priv->uncore,
3652 DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
3653 enabled_slices |= BIT(slice);
3656 return enabled_slices;
3660 * FIXME: We still don't have the proper code detect if we need to apply the WA,
3661 * so assume we'll always need it in order to avoid underruns.
3663 static bool skl_needs_memory_bw_wa(struct drm_i915_private *dev_priv)
3665 return DISPLAY_VER(dev_priv) == 9;
3669 intel_has_sagv(struct drm_i915_private *dev_priv)
3671 return DISPLAY_VER(dev_priv) >= 9 && !IS_LP(dev_priv) &&
3672 dev_priv->sagv_status != I915_SAGV_NOT_CONTROLLED;
3676 intel_sagv_block_time(struct drm_i915_private *dev_priv)
3678 if (DISPLAY_VER(dev_priv) >= 12) {
3682 ret = snb_pcode_read(dev_priv,
3683 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
3686 drm_dbg_kms(&dev_priv->drm, "Couldn't read SAGV block time!\n");
3691 } else if (DISPLAY_VER(dev_priv) == 11) {
3693 } else if (DISPLAY_VER(dev_priv) == 9 && !IS_LP(dev_priv)) {
3700 static void intel_sagv_init(struct drm_i915_private *i915)
3702 if (!intel_has_sagv(i915))
3703 i915->sagv_status = I915_SAGV_NOT_CONTROLLED;
3706 * Probe to see if we have working SAGV control.
3707 * For icl+ this was already determined by intel_bw_init_hw().
3709 if (DISPLAY_VER(i915) < 11)
3710 skl_sagv_disable(i915);
3712 drm_WARN_ON(&i915->drm, i915->sagv_status == I915_SAGV_UNKNOWN);
3714 i915->sagv_block_time_us = intel_sagv_block_time(i915);
3716 drm_dbg_kms(&i915->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
3717 str_yes_no(intel_has_sagv(i915)), i915->sagv_block_time_us);
3719 /* avoid overflow when adding with wm0 latency/etc. */
3720 if (drm_WARN(&i915->drm, i915->sagv_block_time_us > U16_MAX,
3721 "Excessive SAGV block time %u, ignoring\n",
3722 i915->sagv_block_time_us))
3723 i915->sagv_block_time_us = 0;
3725 if (!intel_has_sagv(i915))
3726 i915->sagv_block_time_us = 0;
3730 * SAGV dynamically adjusts the system agent voltage and clock frequencies
3731 * depending on power and performance requirements. The display engine access
3732 * to system memory is blocked during the adjustment time. Because of the
3733 * blocking time, having this enabled can cause full system hangs and/or pipe
3734 * underruns if we don't meet all of the following requirements:
3736 * - <= 1 pipe enabled
3737 * - All planes can enable watermarks for latencies >= SAGV engine block time
3738 * - We're not using an interlaced display configuration
3740 static void skl_sagv_enable(struct drm_i915_private *dev_priv)
3744 if (!intel_has_sagv(dev_priv))
3747 if (dev_priv->sagv_status == I915_SAGV_ENABLED)
3750 drm_dbg_kms(&dev_priv->drm, "Enabling SAGV\n");
3751 ret = snb_pcode_write(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3754 /* We don't need to wait for SAGV when enabling */
3757 * Some skl systems, pre-release machines in particular,
3758 * don't actually have SAGV.
3760 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3761 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3762 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3764 } else if (ret < 0) {
3765 drm_err(&dev_priv->drm, "Failed to enable SAGV\n");
3769 dev_priv->sagv_status = I915_SAGV_ENABLED;
3772 static void skl_sagv_disable(struct drm_i915_private *dev_priv)
3776 if (!intel_has_sagv(dev_priv))
3779 if (dev_priv->sagv_status == I915_SAGV_DISABLED)
3782 drm_dbg_kms(&dev_priv->drm, "Disabling SAGV\n");
3783 /* bspec says to keep retrying for at least 1 ms */
3784 ret = skl_pcode_request(dev_priv, GEN9_PCODE_SAGV_CONTROL,
3786 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
3789 * Some skl systems, pre-release machines in particular,
3790 * don't actually have SAGV.
3792 if (IS_SKYLAKE(dev_priv) && ret == -ENXIO) {
3793 drm_dbg(&dev_priv->drm, "No SAGV found on system, ignoring\n");
3794 dev_priv->sagv_status = I915_SAGV_NOT_CONTROLLED;
3796 } else if (ret < 0) {
3797 drm_err(&dev_priv->drm, "Failed to disable SAGV (%d)\n", ret);
3801 dev_priv->sagv_status = I915_SAGV_DISABLED;
3804 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
3806 struct drm_i915_private *i915 = to_i915(state->base.dev);
3807 const struct intel_bw_state *new_bw_state =
3808 intel_atomic_get_new_bw_state(state);
3813 if (!intel_can_enable_sagv(i915, new_bw_state))
3814 skl_sagv_disable(i915);
3817 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
3819 struct drm_i915_private *i915 = to_i915(state->base.dev);
3820 const struct intel_bw_state *new_bw_state =
3821 intel_atomic_get_new_bw_state(state);
3826 if (intel_can_enable_sagv(i915, new_bw_state))
3827 skl_sagv_enable(i915);
3830 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
3832 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3833 const struct intel_bw_state *old_bw_state =
3834 intel_atomic_get_old_bw_state(state);
3835 const struct intel_bw_state *new_bw_state =
3836 intel_atomic_get_new_bw_state(state);
3837 u16 old_mask, new_mask;
3842 old_mask = old_bw_state->qgv_points_mask;
3843 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3845 if (old_mask == new_mask)
3848 WARN_ON(!new_bw_state->base.changed);
3850 drm_dbg_kms(&dev_priv->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
3851 old_mask, new_mask);
3854 * Restrict required qgv points before updating the configuration.
3855 * According to BSpec we can't mask and unmask qgv points at the same
3856 * time. Also masking should be done before updating the configuration
3857 * and unmasking afterwards.
3859 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3862 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
3864 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3865 const struct intel_bw_state *old_bw_state =
3866 intel_atomic_get_old_bw_state(state);
3867 const struct intel_bw_state *new_bw_state =
3868 intel_atomic_get_new_bw_state(state);
3869 u16 old_mask, new_mask;
3874 old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
3875 new_mask = new_bw_state->qgv_points_mask;
3877 if (old_mask == new_mask)
3880 WARN_ON(!new_bw_state->base.changed);
3882 drm_dbg_kms(&dev_priv->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
3883 old_mask, new_mask);
3886 * Allow required qgv points after updating the configuration.
3887 * According to BSpec we can't mask and unmask qgv points at the same
3888 * time. Also masking should be done before updating the configuration
3889 * and unmasking afterwards.
3891 icl_pcode_restrict_qgv_points(dev_priv, new_mask);
3894 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
3896 struct drm_i915_private *i915 = to_i915(state->base.dev);
3899 * Just return if we can't control SAGV or don't have it.
3900 * This is different from situation when we have SAGV but just can't
3901 * afford it due to DBuf limitation - in case if SAGV is completely
3902 * disabled in a BIOS, we are not even allowed to send a PCode request,
3903 * as it will throw an error. So have to check it here.
3905 if (!intel_has_sagv(i915))
3908 if (DISPLAY_VER(i915) >= 11)
3909 icl_sagv_pre_plane_update(state);
3911 skl_sagv_pre_plane_update(state);
3914 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
3916 struct drm_i915_private *i915 = to_i915(state->base.dev);
3919 * Just return if we can't control SAGV or don't have it.
3920 * This is different from situation when we have SAGV but just can't
3921 * afford it due to DBuf limitation - in case if SAGV is completely
3922 * disabled in a BIOS, we are not even allowed to send a PCode request,
3923 * as it will throw an error. So have to check it here.
3925 if (!intel_has_sagv(i915))
3928 if (DISPLAY_VER(i915) >= 11)
3929 icl_sagv_post_plane_update(state);
3931 skl_sagv_post_plane_update(state);
3934 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3936 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3937 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3938 enum plane_id plane_id;
3939 int max_level = INT_MAX;
3941 if (!intel_has_sagv(dev_priv))
3944 if (!crtc_state->hw.active)
3947 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
3950 for_each_plane_id_on_crtc(crtc, plane_id) {
3951 const struct skl_plane_wm *wm =
3952 &crtc_state->wm.skl.optimal.planes[plane_id];
3955 /* Skip this plane if it's not enabled */
3956 if (!wm->wm[0].enable)
3959 /* Find the highest enabled wm level for this plane */
3960 for (level = ilk_wm_max_level(dev_priv);
3961 !wm->wm[level].enable; --level)
3964 /* Highest common enabled wm level for all planes */
3965 max_level = min(level, max_level);
3968 /* No enabled planes? */
3969 if (max_level == INT_MAX)
3972 for_each_plane_id_on_crtc(crtc, plane_id) {
3973 const struct skl_plane_wm *wm =
3974 &crtc_state->wm.skl.optimal.planes[plane_id];
3977 * All enabled planes must have enabled a common wm level that
3978 * can tolerate memory latencies higher than sagv_block_time_us
3980 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
3987 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
3989 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
3990 enum plane_id plane_id;
3992 if (!crtc_state->hw.active)
3995 for_each_plane_id_on_crtc(crtc, plane_id) {
3996 const struct skl_plane_wm *wm =
3997 &crtc_state->wm.skl.optimal.planes[plane_id];
3999 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
4006 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
4008 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4009 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4011 if (DISPLAY_VER(dev_priv) >= 12)
4012 return tgl_crtc_can_enable_sagv(crtc_state);
4014 return skl_crtc_can_enable_sagv(crtc_state);
4017 bool intel_can_enable_sagv(struct drm_i915_private *dev_priv,
4018 const struct intel_bw_state *bw_state)
4020 if (DISPLAY_VER(dev_priv) < 11 &&
4021 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
4024 return bw_state->pipe_sagv_reject == 0;
4027 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
4029 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
4031 struct intel_crtc *crtc;
4032 struct intel_crtc_state *new_crtc_state;
4033 struct intel_bw_state *new_bw_state = NULL;
4034 const struct intel_bw_state *old_bw_state = NULL;
4037 for_each_new_intel_crtc_in_state(state, crtc,
4038 new_crtc_state, i) {
4039 new_bw_state = intel_atomic_get_bw_state(state);
4040 if (IS_ERR(new_bw_state))
4041 return PTR_ERR(new_bw_state);
4043 old_bw_state = intel_atomic_get_old_bw_state(state);
4045 if (intel_crtc_can_enable_sagv(new_crtc_state))
4046 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
4048 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
4054 new_bw_state->active_pipes =
4055 intel_calc_active_pipes(state, old_bw_state->active_pipes);
4057 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
4058 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4063 if (intel_can_enable_sagv(dev_priv, new_bw_state) !=
4064 intel_can_enable_sagv(dev_priv, old_bw_state)) {
4065 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
4068 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
4069 ret = intel_atomic_lock_global_state(&new_bw_state->base);
4074 for_each_new_intel_crtc_in_state(state, crtc,
4075 new_crtc_state, i) {
4076 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
4079 * We store use_sagv_wm in the crtc state rather than relying on
4080 * that bw state since we have no convenient way to get at the
4081 * latter from the plane commit hooks (especially in the legacy
4084 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(dev_priv) &&
4085 DISPLAY_VER(dev_priv) >= 12 &&
4086 intel_can_enable_sagv(dev_priv, new_bw_state);
4092 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
4095 entry->start = start;
4101 static int intel_dbuf_slice_size(struct drm_i915_private *dev_priv)
4103 return INTEL_INFO(dev_priv)->dbuf.size /
4104 hweight8(INTEL_INFO(dev_priv)->dbuf.slice_mask);
4108 skl_ddb_entry_for_slices(struct drm_i915_private *dev_priv, u8 slice_mask,
4109 struct skl_ddb_entry *ddb)
4111 int slice_size = intel_dbuf_slice_size(dev_priv);
4119 ddb->start = (ffs(slice_mask) - 1) * slice_size;
4120 ddb->end = fls(slice_mask) * slice_size;
4122 WARN_ON(ddb->start >= ddb->end);
4123 WARN_ON(ddb->end > INTEL_INFO(dev_priv)->dbuf.size);
4126 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
4128 struct skl_ddb_entry ddb;
4130 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
4131 slice_mask = BIT(DBUF_S1);
4132 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
4133 slice_mask = BIT(DBUF_S3);
4135 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
4140 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *dev_priv,
4141 const struct skl_ddb_entry *entry)
4143 int slice_size = intel_dbuf_slice_size(dev_priv);
4144 enum dbuf_slice start_slice, end_slice;
4147 if (!skl_ddb_entry_size(entry))
4150 start_slice = entry->start / slice_size;
4151 end_slice = (entry->end - 1) / slice_size;
4154 * Per plane DDB entry can in a really worst case be on multiple slices
4155 * but single entry is anyway contigious.
4157 while (start_slice <= end_slice) {
4158 slice_mask |= BIT(start_slice);
4165 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
4167 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
4168 int hdisplay, vdisplay;
4170 if (!crtc_state->hw.active)
4174 * Watermark/ddb requirement highly depends upon width of the
4175 * framebuffer, So instead of allocating DDB equally among pipes
4176 * distribute DDB based on resolution/width of the display.
4178 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
4183 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
4185 unsigned int *weight_start,
4186 unsigned int *weight_end,
4187 unsigned int *weight_total)
4189 struct drm_i915_private *dev_priv =
4190 to_i915(dbuf_state->base.state->base.dev);
4197 for_each_pipe(dev_priv, pipe) {
4198 int weight = dbuf_state->weight[pipe];
4201 * Do not account pipes using other slice sets
4202 * luckily as of current BSpec slice sets do not partially
4203 * intersect(pipes share either same one slice or same slice set
4204 * i.e no partial intersection), so it is enough to check for
4207 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
4210 *weight_total += weight;
4211 if (pipe < for_pipe) {
4212 *weight_start += weight;
4213 *weight_end += weight;
4214 } else if (pipe == for_pipe) {
4215 *weight_end += weight;
4221 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
4223 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4224 unsigned int weight_total, weight_start, weight_end;
4225 const struct intel_dbuf_state *old_dbuf_state =
4226 intel_atomic_get_old_dbuf_state(state);
4227 struct intel_dbuf_state *new_dbuf_state =
4228 intel_atomic_get_new_dbuf_state(state);
4229 struct intel_crtc_state *crtc_state;
4230 struct skl_ddb_entry ddb_slices;
4231 enum pipe pipe = crtc->pipe;
4232 unsigned int mbus_offset = 0;
4234 u32 dbuf_slice_mask;
4238 if (new_dbuf_state->weight[pipe] == 0) {
4239 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
4243 dbuf_slice_mask = new_dbuf_state->slices[pipe];
4245 skl_ddb_entry_for_slices(dev_priv, dbuf_slice_mask, &ddb_slices);
4246 mbus_offset = mbus_ddb_offset(dev_priv, dbuf_slice_mask);
4247 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
4249 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
4250 &weight_start, &weight_end, &weight_total);
4252 start = ddb_range_size * weight_start / weight_total;
4253 end = ddb_range_size * weight_end / weight_total;
4255 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
4256 ddb_slices.start - mbus_offset + start,
4257 ddb_slices.start - mbus_offset + end);
4260 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
4261 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
4262 &new_dbuf_state->ddb[pipe]))
4265 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
4269 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
4270 if (IS_ERR(crtc_state))
4271 return PTR_ERR(crtc_state);
4274 * Used for checking overlaps, so we need absolute
4275 * offsets instead of MBUS relative offsets.
4277 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
4278 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
4280 drm_dbg_kms(&dev_priv->drm,
4281 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
4282 crtc->base.base.id, crtc->base.name,
4283 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
4284 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
4285 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
4286 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
4291 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
4292 int width, const struct drm_format_info *format,
4293 u64 modifier, unsigned int rotation,
4294 u32 plane_pixel_rate, struct skl_wm_params *wp,
4297 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
4298 struct intel_plane *plane,
4300 unsigned int latency,
4301 const struct skl_wm_params *wp,
4302 const struct skl_wm_level *result_prev,
4303 struct skl_wm_level *result /* out */);
4306 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
4309 struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
4310 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
4311 int level, max_level = ilk_wm_max_level(dev_priv);
4312 struct skl_wm_level wm = {};
4313 int ret, min_ddb_alloc = 0;
4314 struct skl_wm_params wp;
4316 ret = skl_compute_wm_params(crtc_state, 256,
4317 drm_format_info(DRM_FORMAT_ARGB8888),
4318 DRM_FORMAT_MOD_LINEAR,
4320 crtc_state->pixel_rate, &wp, 0);
4321 drm_WARN_ON(&dev_priv->drm, ret);
4323 for (level = 0; level <= max_level; level++) {
4324 unsigned int latency = dev_priv->wm.skl_latency[level];
4326 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
4327 if (wm.min_ddb_alloc == U16_MAX)
4330 min_ddb_alloc = wm.min_ddb_alloc;
4333 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
4336 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
4338 skl_ddb_entry_init(entry,
4339 REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
4340 REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
4346 skl_ddb_get_hw_plane_state(struct drm_i915_private *dev_priv,
4347 const enum pipe pipe,
4348 const enum plane_id plane_id,
4349 struct skl_ddb_entry *ddb,
4350 struct skl_ddb_entry *ddb_y)
4354 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
4355 if (plane_id == PLANE_CURSOR) {
4356 val = intel_uncore_read(&dev_priv->uncore, CUR_BUF_CFG(pipe));
4357 skl_ddb_entry_init_from_hw(ddb, val);
4361 val = intel_uncore_read(&dev_priv->uncore, PLANE_BUF_CFG(pipe, plane_id));
4362 skl_ddb_entry_init_from_hw(ddb, val);
4364 if (DISPLAY_VER(dev_priv) >= 11)
4367 val = intel_uncore_read(&dev_priv->uncore, PLANE_NV12_BUF_CFG(pipe, plane_id));
4368 skl_ddb_entry_init_from_hw(ddb_y, val);
4371 void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
4372 struct skl_ddb_entry *ddb,
4373 struct skl_ddb_entry *ddb_y)
4375 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4376 enum intel_display_power_domain power_domain;
4377 enum pipe pipe = crtc->pipe;
4378 intel_wakeref_t wakeref;
4379 enum plane_id plane_id;
4381 power_domain = POWER_DOMAIN_PIPE(pipe);
4382 wakeref = intel_display_power_get_if_enabled(dev_priv, power_domain);
4386 for_each_plane_id_on_crtc(crtc, plane_id)
4387 skl_ddb_get_hw_plane_state(dev_priv, pipe,
4392 intel_display_power_put(dev_priv, power_domain, wakeref);
4395 struct dbuf_slice_conf_entry {
4397 u8 dbuf_mask[I915_MAX_PIPES];
4402 * Table taken from Bspec 12716
4403 * Pipes do have some preferred DBuf slice affinity,
4404 * plus there are some hardcoded requirements on how
4405 * those should be distributed for multipipe scenarios.
4406 * For more DBuf slices algorithm can get even more messy
4407 * and less readable, so decided to use a table almost
4408 * as is from BSpec itself - that way it is at least easier
4409 * to compare, change and check.
4411 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
4412 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4415 .active_pipes = BIT(PIPE_A),
4417 [PIPE_A] = BIT(DBUF_S1),
4421 .active_pipes = BIT(PIPE_B),
4423 [PIPE_B] = BIT(DBUF_S1),
4427 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4429 [PIPE_A] = BIT(DBUF_S1),
4430 [PIPE_B] = BIT(DBUF_S2),
4434 .active_pipes = BIT(PIPE_C),
4436 [PIPE_C] = BIT(DBUF_S2),
4440 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4442 [PIPE_A] = BIT(DBUF_S1),
4443 [PIPE_C] = BIT(DBUF_S2),
4447 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4449 [PIPE_B] = BIT(DBUF_S1),
4450 [PIPE_C] = BIT(DBUF_S2),
4454 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4456 [PIPE_A] = BIT(DBUF_S1),
4457 [PIPE_B] = BIT(DBUF_S1),
4458 [PIPE_C] = BIT(DBUF_S2),
4465 * Table taken from Bspec 49255
4466 * Pipes do have some preferred DBuf slice affinity,
4467 * plus there are some hardcoded requirements on how
4468 * those should be distributed for multipipe scenarios.
4469 * For more DBuf slices algorithm can get even more messy
4470 * and less readable, so decided to use a table almost
4471 * as is from BSpec itself - that way it is at least easier
4472 * to compare, change and check.
4474 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
4475 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
4478 .active_pipes = BIT(PIPE_A),
4480 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4484 .active_pipes = BIT(PIPE_B),
4486 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4490 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4492 [PIPE_A] = BIT(DBUF_S2),
4493 [PIPE_B] = BIT(DBUF_S1),
4497 .active_pipes = BIT(PIPE_C),
4499 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
4503 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4505 [PIPE_A] = BIT(DBUF_S1),
4506 [PIPE_C] = BIT(DBUF_S2),
4510 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4512 [PIPE_B] = BIT(DBUF_S1),
4513 [PIPE_C] = BIT(DBUF_S2),
4517 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4519 [PIPE_A] = BIT(DBUF_S1),
4520 [PIPE_B] = BIT(DBUF_S1),
4521 [PIPE_C] = BIT(DBUF_S2),
4525 .active_pipes = BIT(PIPE_D),
4527 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
4531 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4533 [PIPE_A] = BIT(DBUF_S1),
4534 [PIPE_D] = BIT(DBUF_S2),
4538 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4540 [PIPE_B] = BIT(DBUF_S1),
4541 [PIPE_D] = BIT(DBUF_S2),
4545 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4547 [PIPE_A] = BIT(DBUF_S1),
4548 [PIPE_B] = BIT(DBUF_S1),
4549 [PIPE_D] = BIT(DBUF_S2),
4553 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4555 [PIPE_C] = BIT(DBUF_S1),
4556 [PIPE_D] = BIT(DBUF_S2),
4560 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4562 [PIPE_A] = BIT(DBUF_S1),
4563 [PIPE_C] = BIT(DBUF_S2),
4564 [PIPE_D] = BIT(DBUF_S2),
4568 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4570 [PIPE_B] = BIT(DBUF_S1),
4571 [PIPE_C] = BIT(DBUF_S2),
4572 [PIPE_D] = BIT(DBUF_S2),
4576 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4578 [PIPE_A] = BIT(DBUF_S1),
4579 [PIPE_B] = BIT(DBUF_S1),
4580 [PIPE_C] = BIT(DBUF_S2),
4581 [PIPE_D] = BIT(DBUF_S2),
4587 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
4589 .active_pipes = BIT(PIPE_A),
4591 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4595 .active_pipes = BIT(PIPE_B),
4597 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4601 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4603 [PIPE_A] = BIT(DBUF_S1),
4604 [PIPE_B] = BIT(DBUF_S2),
4608 .active_pipes = BIT(PIPE_C),
4610 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4614 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4616 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4617 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4621 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4623 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4624 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4628 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4630 [PIPE_A] = BIT(DBUF_S1),
4631 [PIPE_B] = BIT(DBUF_S2),
4632 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4636 .active_pipes = BIT(PIPE_D),
4638 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4642 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4644 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4645 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4649 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4651 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4652 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4656 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4658 [PIPE_A] = BIT(DBUF_S1),
4659 [PIPE_B] = BIT(DBUF_S2),
4660 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
4664 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4666 [PIPE_C] = BIT(DBUF_S3),
4667 [PIPE_D] = BIT(DBUF_S4),
4671 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4673 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4674 [PIPE_C] = BIT(DBUF_S3),
4675 [PIPE_D] = BIT(DBUF_S4),
4679 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4681 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
4682 [PIPE_C] = BIT(DBUF_S3),
4683 [PIPE_D] = BIT(DBUF_S4),
4687 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4689 [PIPE_A] = BIT(DBUF_S1),
4690 [PIPE_B] = BIT(DBUF_S2),
4691 [PIPE_C] = BIT(DBUF_S3),
4692 [PIPE_D] = BIT(DBUF_S4),
4698 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
4700 * Keep the join_mbus cases first so check_mbus_joined()
4701 * will prefer them over the !join_mbus cases.
4704 .active_pipes = BIT(PIPE_A),
4706 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4711 .active_pipes = BIT(PIPE_B),
4713 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
4718 .active_pipes = BIT(PIPE_A),
4720 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4725 .active_pipes = BIT(PIPE_B),
4727 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4732 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
4734 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4735 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4739 .active_pipes = BIT(PIPE_C),
4741 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4745 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
4747 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4748 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4752 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
4754 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4755 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4759 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
4761 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4762 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4763 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4767 .active_pipes = BIT(PIPE_D),
4769 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4773 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
4775 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4776 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4780 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
4782 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4783 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4787 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
4789 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4790 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4791 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4795 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
4797 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4798 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4802 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
4804 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4805 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4806 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4810 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4812 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4813 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4814 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4818 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
4820 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
4821 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
4822 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
4823 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
4830 static bool check_mbus_joined(u8 active_pipes,
4831 const struct dbuf_slice_conf_entry *dbuf_slices)
4835 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
4836 if (dbuf_slices[i].active_pipes == active_pipes)
4837 return dbuf_slices[i].join_mbus;
4842 static bool adlp_check_mbus_joined(u8 active_pipes)
4844 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
4847 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
4848 const struct dbuf_slice_conf_entry *dbuf_slices)
4852 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
4853 if (dbuf_slices[i].active_pipes == active_pipes &&
4854 dbuf_slices[i].join_mbus == join_mbus)
4855 return dbuf_slices[i].dbuf_mask[pipe];
4861 * This function finds an entry with same enabled pipe configuration and
4862 * returns correspondent DBuf slice mask as stated in BSpec for particular
4865 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4868 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
4869 * required calculating "pipe ratio" in order to determine
4870 * if one or two slices can be used for single pipe configurations
4871 * as additional constraint to the existing table.
4872 * However based on recent info, it should be not "pipe ratio"
4873 * but rather ratio between pixel_rate and cdclk with additional
4874 * constants, so for now we are using only table until this is
4875 * clarified. Also this is the reason why crtc_state param is
4876 * still here - we will need it once those additional constraints
4879 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4883 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4885 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4889 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4891 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4892 adlp_allowed_dbufs);
4895 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
4897 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
4901 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
4903 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
4904 enum pipe pipe = crtc->pipe;
4906 if (IS_DG2(dev_priv))
4907 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4908 else if (IS_ALDERLAKE_P(dev_priv))
4909 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4910 else if (DISPLAY_VER(dev_priv) == 12)
4911 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4912 else if (DISPLAY_VER(dev_priv) == 11)
4913 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
4915 * For anything else just return one slice yet.
4916 * Should be extended for other platforms.
4918 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
4922 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
4923 struct intel_plane *plane)
4925 struct drm_i915_private *i915 = to_i915(plane->base.dev);
4927 return DISPLAY_VER(i915) >= 13 &&
4928 crtc_state->uapi.async_flip &&
4933 skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
4935 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
4936 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
4937 enum plane_id plane_id;
4940 for_each_plane_id_on_crtc(crtc, plane_id) {
4941 if (plane_id == PLANE_CURSOR)
4944 data_rate += crtc_state->rel_data_rate[plane_id];
4946 if (DISPLAY_VER(i915) < 11)
4947 data_rate += crtc_state->rel_data_rate_y[plane_id];
4953 const struct skl_wm_level *
4954 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
4955 enum plane_id plane_id,
4958 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4960 if (level == 0 && pipe_wm->use_sagv_wm)
4961 return &wm->sagv.wm0;
4963 return &wm->wm[level];
4966 const struct skl_wm_level *
4967 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
4968 enum plane_id plane_id)
4970 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
4972 if (pipe_wm->use_sagv_wm)
4973 return &wm->sagv.trans_wm;
4975 return &wm->trans_wm;
4979 * We only disable the watermarks for each plane if
4980 * they exceed the ddb allocation of said plane. This
4981 * is done so that we don't end up touching cursor
4982 * watermarks needlessly when some other plane reduces
4983 * our max possible watermark level.
4985 * Bspec has this to say about the PLANE_WM enable bit:
4986 * "All the watermarks at this level for all enabled
4987 * planes must be enabled before the level will be used."
4988 * So this is actually safe to do.
4991 skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
4993 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
4994 memset(wm, 0, sizeof(*wm));
4998 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
4999 const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
5001 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
5002 uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
5003 memset(wm, 0, sizeof(*wm));
5004 memset(uv_wm, 0, sizeof(*uv_wm));
5008 static bool icl_need_wm1_wa(struct drm_i915_private *i915,
5009 enum plane_id plane_id)
5012 * Wa_1408961008:icl, ehl
5013 * Wa_14012656716:tgl, adl
5014 * Underruns with WM1+ disabled
5016 return DISPLAY_VER(i915) == 11 ||
5017 (IS_DISPLAY_VER(i915, 12, 13) && plane_id == PLANE_CURSOR);
5020 struct skl_plane_ddb_iter {
5026 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
5027 struct skl_ddb_entry *ddb,
5028 const struct skl_wm_level *wm,
5031 u16 size, extra = 0;
5034 extra = min_t(u16, iter->size,
5035 DIV64_U64_ROUND_UP(iter->size * data_rate,
5037 iter->size -= extra;
5038 iter->data_rate -= data_rate;
5042 * Keep ddb entry of all disabled planes explicitly zeroed
5043 * to avoid skl_ddb_add_affected_planes() adding them to
5044 * the state when other planes change their allocations.
5046 size = wm->min_ddb_alloc + extra;
5048 iter->start = skl_ddb_entry_init(ddb, iter->start,
5049 iter->start + size);
5053 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
5054 struct intel_crtc *crtc)
5056 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5057 struct intel_crtc_state *crtc_state =
5058 intel_atomic_get_new_crtc_state(state, crtc);
5059 const struct intel_dbuf_state *dbuf_state =
5060 intel_atomic_get_new_dbuf_state(state);
5061 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
5062 int num_active = hweight8(dbuf_state->active_pipes);
5063 struct skl_plane_ddb_iter iter;
5064 enum plane_id plane_id;
5069 /* Clear the partitioning for disabled planes. */
5070 memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
5071 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
5073 if (!crtc_state->hw.active)
5076 iter.start = alloc->start;
5077 iter.size = skl_ddb_entry_size(alloc);
5081 /* Allocate fixed number of blocks for cursor. */
5082 cursor_size = skl_cursor_allocation(crtc_state, num_active);
5083 iter.size -= cursor_size;
5084 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
5085 alloc->end - cursor_size, alloc->end);
5087 iter.data_rate = skl_total_relative_data_rate(crtc_state);
5090 * Find the highest watermark level for which we can satisfy the block
5091 * requirement of active planes.
5093 for (level = ilk_wm_max_level(dev_priv); level >= 0; level--) {
5095 for_each_plane_id_on_crtc(crtc, plane_id) {
5096 const struct skl_plane_wm *wm =
5097 &crtc_state->wm.skl.optimal.planes[plane_id];
5099 if (plane_id == PLANE_CURSOR) {
5100 const struct skl_ddb_entry *ddb =
5101 &crtc_state->wm.skl.plane_ddb[plane_id];
5103 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
5104 drm_WARN_ON(&dev_priv->drm,
5105 wm->wm[level].min_ddb_alloc != U16_MAX);
5112 blocks += wm->wm[level].min_ddb_alloc;
5113 blocks += wm->uv_wm[level].min_ddb_alloc;
5116 if (blocks <= iter.size) {
5117 iter.size -= blocks;
5123 drm_dbg_kms(&dev_priv->drm,
5124 "Requested display configuration exceeds system DDB limitations");
5125 drm_dbg_kms(&dev_priv->drm, "minimum required %d/%d\n",
5130 /* avoid the WARN later when we don't allocate any extra DDB */
5131 if (iter.data_rate == 0)
5135 * Grant each plane the blocks it requires at the highest achievable
5136 * watermark level, plus an extra share of the leftover blocks
5137 * proportional to its relative data rate.
5139 for_each_plane_id_on_crtc(crtc, plane_id) {
5140 struct skl_ddb_entry *ddb =
5141 &crtc_state->wm.skl.plane_ddb[plane_id];
5142 struct skl_ddb_entry *ddb_y =
5143 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5144 const struct skl_plane_wm *wm =
5145 &crtc_state->wm.skl.optimal.planes[plane_id];
5147 if (plane_id == PLANE_CURSOR)
5150 if (DISPLAY_VER(dev_priv) < 11 &&
5151 crtc_state->nv12_planes & BIT(plane_id)) {
5152 skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
5153 crtc_state->rel_data_rate_y[plane_id]);
5154 skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
5155 crtc_state->rel_data_rate[plane_id]);
5157 skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
5158 crtc_state->rel_data_rate[plane_id]);
5161 drm_WARN_ON(&dev_priv->drm, iter.size != 0 || iter.data_rate != 0);
5164 * When we calculated watermark values we didn't know how high
5165 * of a level we'd actually be able to hit, so we just marked
5166 * all levels as "enabled." Go back now and disable the ones
5167 * that aren't actually possible.
5169 for (level++; level <= ilk_wm_max_level(dev_priv); level++) {
5170 for_each_plane_id_on_crtc(crtc, plane_id) {
5171 const struct skl_ddb_entry *ddb =
5172 &crtc_state->wm.skl.plane_ddb[plane_id];
5173 const struct skl_ddb_entry *ddb_y =
5174 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5175 struct skl_plane_wm *wm =
5176 &crtc_state->wm.skl.optimal.planes[plane_id];
5178 if (DISPLAY_VER(dev_priv) < 11 &&
5179 crtc_state->nv12_planes & BIT(plane_id))
5180 skl_check_nv12_wm_level(&wm->wm[level],
5184 skl_check_wm_level(&wm->wm[level], ddb);
5186 if (icl_need_wm1_wa(dev_priv, plane_id) &&
5187 level == 1 && wm->wm[0].enable) {
5188 wm->wm[level].blocks = wm->wm[0].blocks;
5189 wm->wm[level].lines = wm->wm[0].lines;
5190 wm->wm[level].ignore_lines = wm->wm[0].ignore_lines;
5196 * Go back and disable the transition and SAGV watermarks
5197 * if it turns out we don't have enough DDB blocks for them.
5199 for_each_plane_id_on_crtc(crtc, plane_id) {
5200 const struct skl_ddb_entry *ddb =
5201 &crtc_state->wm.skl.plane_ddb[plane_id];
5202 const struct skl_ddb_entry *ddb_y =
5203 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5204 struct skl_plane_wm *wm =
5205 &crtc_state->wm.skl.optimal.planes[plane_id];
5207 if (DISPLAY_VER(dev_priv) < 11 &&
5208 crtc_state->nv12_planes & BIT(plane_id)) {
5209 skl_check_wm_level(&wm->trans_wm, ddb_y);
5211 WARN_ON(skl_ddb_entry_size(ddb_y));
5213 skl_check_wm_level(&wm->trans_wm, ddb);
5216 skl_check_wm_level(&wm->sagv.wm0, ddb);
5217 skl_check_wm_level(&wm->sagv.trans_wm, ddb);
5224 * The max latency should be 257 (max the punit can code is 255 and we add 2us
5225 * for the read latency) and cpp should always be <= 8, so that
5226 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
5227 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
5229 static uint_fixed_16_16_t
5230 skl_wm_method1(const struct drm_i915_private *dev_priv, u32 pixel_rate,
5231 u8 cpp, u32 latency, u32 dbuf_block_size)
5233 u32 wm_intermediate_val;
5234 uint_fixed_16_16_t ret;
5237 return FP_16_16_MAX;
5239 wm_intermediate_val = latency * pixel_rate * cpp;
5240 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
5242 if (DISPLAY_VER(dev_priv) >= 10)
5243 ret = add_fixed16_u32(ret, 1);
5248 static uint_fixed_16_16_t
5249 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
5250 uint_fixed_16_16_t plane_blocks_per_line)
5252 u32 wm_intermediate_val;
5253 uint_fixed_16_16_t ret;
5256 return FP_16_16_MAX;
5258 wm_intermediate_val = latency * pixel_rate;
5259 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
5260 pipe_htotal * 1000);
5261 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
5265 static uint_fixed_16_16_t
5266 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
5268 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5271 uint_fixed_16_16_t linetime_us;
5273 if (!crtc_state->hw.active)
5274 return u32_to_fixed16(0);
5276 pixel_rate = crtc_state->pixel_rate;
5278 if (drm_WARN_ON(&dev_priv->drm, pixel_rate == 0))
5279 return u32_to_fixed16(0);
5281 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
5282 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
5288 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
5289 int width, const struct drm_format_info *format,
5290 u64 modifier, unsigned int rotation,
5291 u32 plane_pixel_rate, struct skl_wm_params *wp,
5294 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5295 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5298 /* only planar format has two planes */
5299 if (color_plane == 1 &&
5300 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
5301 drm_dbg_kms(&dev_priv->drm,
5302 "Non planar format have single plane\n");
5306 wp->y_tiled = modifier == I915_FORMAT_MOD_Y_TILED ||
5307 modifier == I915_FORMAT_MOD_4_TILED ||
5308 modifier == I915_FORMAT_MOD_Yf_TILED ||
5309 modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5310 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5311 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
5312 wp->rc_surface = modifier == I915_FORMAT_MOD_Y_TILED_CCS ||
5313 modifier == I915_FORMAT_MOD_Yf_TILED_CCS;
5314 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
5317 if (color_plane == 1 && wp->is_planar)
5320 wp->cpp = format->cpp[color_plane];
5321 wp->plane_pixel_rate = plane_pixel_rate;
5323 if (DISPLAY_VER(dev_priv) >= 11 &&
5324 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
5325 wp->dbuf_block_size = 256;
5327 wp->dbuf_block_size = 512;
5329 if (drm_rotation_90_or_270(rotation)) {
5332 wp->y_min_scanlines = 16;
5335 wp->y_min_scanlines = 8;
5338 wp->y_min_scanlines = 4;
5341 MISSING_CASE(wp->cpp);
5345 wp->y_min_scanlines = 4;
5348 if (skl_needs_memory_bw_wa(dev_priv))
5349 wp->y_min_scanlines *= 2;
5351 wp->plane_bytes_per_line = wp->width * wp->cpp;
5353 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
5354 wp->y_min_scanlines,
5355 wp->dbuf_block_size);
5357 if (DISPLAY_VER(dev_priv) >= 10)
5360 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
5361 wp->y_min_scanlines);
5363 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
5364 wp->dbuf_block_size);
5366 if (!wp->x_tiled || DISPLAY_VER(dev_priv) >= 10)
5369 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
5372 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
5373 wp->plane_blocks_per_line);
5375 wp->linetime_us = fixed16_to_u32_round_up(
5376 intel_get_linetime_us(crtc_state));
5382 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
5383 const struct intel_plane_state *plane_state,
5384 struct skl_wm_params *wp, int color_plane)
5386 const struct drm_framebuffer *fb = plane_state->hw.fb;
5390 * Src coordinates are already rotated by 270 degrees for
5391 * the 90/270 degree plane rotation cases (to match the
5392 * GTT mapping), hence no need to account for rotation here.
5394 width = drm_rect_width(&plane_state->uapi.src) >> 16;
5396 return skl_compute_wm_params(crtc_state, width,
5397 fb->format, fb->modifier,
5398 plane_state->hw.rotation,
5399 intel_plane_pixel_rate(crtc_state, plane_state),
5403 static bool skl_wm_has_lines(struct drm_i915_private *dev_priv, int level)
5405 if (DISPLAY_VER(dev_priv) >= 10)
5408 /* The number of lines are ignored for the level 0 watermark. */
5412 static int skl_wm_max_lines(struct drm_i915_private *dev_priv)
5414 if (DISPLAY_VER(dev_priv) >= 13)
5420 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
5421 struct intel_plane *plane,
5423 unsigned int latency,
5424 const struct skl_wm_params *wp,
5425 const struct skl_wm_level *result_prev,
5426 struct skl_wm_level *result /* out */)
5428 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5429 uint_fixed_16_16_t method1, method2;
5430 uint_fixed_16_16_t selected_result;
5431 u32 blocks, lines, min_ddb_alloc = 0;
5434 (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
5436 result->min_ddb_alloc = U16_MAX;
5441 * WaIncreaseLatencyIPCEnabled: kbl,cfl
5442 * Display WA #1141: kbl,cfl
5444 if ((IS_KABYLAKE(dev_priv) ||
5445 IS_COFFEELAKE(dev_priv) ||
5446 IS_COMETLAKE(dev_priv)) &&
5447 dev_priv->ipc_enabled)
5450 if (skl_needs_memory_bw_wa(dev_priv) && wp->x_tiled)
5453 method1 = skl_wm_method1(dev_priv, wp->plane_pixel_rate,
5454 wp->cpp, latency, wp->dbuf_block_size);
5455 method2 = skl_wm_method2(wp->plane_pixel_rate,
5456 crtc_state->hw.pipe_mode.crtc_htotal,
5458 wp->plane_blocks_per_line);
5461 selected_result = max_fixed16(method2, wp->y_tile_minimum);
5463 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
5464 wp->dbuf_block_size < 1) &&
5465 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
5466 selected_result = method2;
5467 } else if (latency >= wp->linetime_us) {
5468 if (DISPLAY_VER(dev_priv) == 9)
5469 selected_result = min_fixed16(method1, method2);
5471 selected_result = method2;
5473 selected_result = method1;
5477 blocks = fixed16_to_u32_round_up(selected_result) + 1;
5479 * Lets have blocks at minimum equivalent to plane_blocks_per_line
5480 * as there will be at minimum one line for lines configuration. This
5481 * is a work around for FIFO underruns observed with resolutions like
5482 * 4k 60 Hz in single channel DRAM configurations.
5484 * As per the Bspec 49325, if the ddb allocation can hold at least
5485 * one plane_blocks_per_line, we should have selected method2 in
5486 * the above logic. Assuming that modern versions have enough dbuf
5487 * and method2 guarantees blocks equivalent to at least 1 line,
5488 * select the blocks as plane_blocks_per_line.
5490 * TODO: Revisit the logic when we have better understanding on DRAM
5491 * channels' impact on the level 0 memory latency and the relevant
5494 if (skl_wm_has_lines(dev_priv, level))
5495 blocks = max(blocks,
5496 fixed16_to_u32_round_up(wp->plane_blocks_per_line));
5497 lines = div_round_up_fixed16(selected_result,
5498 wp->plane_blocks_per_line);
5500 if (DISPLAY_VER(dev_priv) == 9) {
5501 /* Display WA #1125: skl,bxt,kbl */
5502 if (level == 0 && wp->rc_surface)
5503 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5505 /* Display WA #1126: skl,bxt,kbl */
5506 if (level >= 1 && level <= 7) {
5508 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
5509 lines += wp->y_min_scanlines;
5515 * Make sure result blocks for higher latency levels are
5516 * atleast as high as level below the current level.
5517 * Assumption in DDB algorithm optimization for special
5518 * cases. Also covers Display WA #1125 for RC.
5520 if (result_prev->blocks > blocks)
5521 blocks = result_prev->blocks;
5525 if (DISPLAY_VER(dev_priv) >= 11) {
5529 if (lines % wp->y_min_scanlines == 0)
5530 extra_lines = wp->y_min_scanlines;
5532 extra_lines = wp->y_min_scanlines * 2 -
5533 lines % wp->y_min_scanlines;
5535 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
5536 wp->plane_blocks_per_line);
5538 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
5542 if (!skl_wm_has_lines(dev_priv, level))
5545 if (lines > skl_wm_max_lines(dev_priv)) {
5547 result->min_ddb_alloc = U16_MAX;
5552 * If lines is valid, assume we can use this watermark level
5553 * for now. We'll come back and disable it after we calculate the
5554 * DDB allocation if it turns out we don't actually have enough
5555 * blocks to satisfy it.
5557 result->blocks = blocks;
5558 result->lines = lines;
5559 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
5560 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
5561 result->enable = true;
5563 if (DISPLAY_VER(dev_priv) < 12 && dev_priv->sagv_block_time_us)
5564 result->can_sagv = latency >= dev_priv->sagv_block_time_us;
5568 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
5569 struct intel_plane *plane,
5570 const struct skl_wm_params *wm_params,
5571 struct skl_wm_level *levels)
5573 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5574 int level, max_level = ilk_wm_max_level(dev_priv);
5575 struct skl_wm_level *result_prev = &levels[0];
5577 for (level = 0; level <= max_level; level++) {
5578 struct skl_wm_level *result = &levels[level];
5579 unsigned int latency = dev_priv->wm.skl_latency[level];
5581 skl_compute_plane_wm(crtc_state, plane, level, latency,
5582 wm_params, result_prev, result);
5584 result_prev = result;
5588 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
5589 struct intel_plane *plane,
5590 const struct skl_wm_params *wm_params,
5591 struct skl_plane_wm *plane_wm)
5593 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
5594 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
5595 struct skl_wm_level *levels = plane_wm->wm;
5596 unsigned int latency = 0;
5598 if (dev_priv->sagv_block_time_us)
5599 latency = dev_priv->sagv_block_time_us + dev_priv->wm.skl_latency[0];
5601 skl_compute_plane_wm(crtc_state, plane, 0, latency,
5602 wm_params, &levels[0],
5606 static void skl_compute_transition_wm(struct drm_i915_private *dev_priv,
5607 struct skl_wm_level *trans_wm,
5608 const struct skl_wm_level *wm0,
5609 const struct skl_wm_params *wp)
5611 u16 trans_min, trans_amount, trans_y_tile_min;
5612 u16 wm0_blocks, trans_offset, blocks;
5614 /* Transition WM don't make any sense if ipc is disabled */
5615 if (!dev_priv->ipc_enabled)
5619 * WaDisableTWM:skl,kbl,cfl,bxt
5620 * Transition WM are not recommended by HW team for GEN9
5622 if (DISPLAY_VER(dev_priv) == 9)
5625 if (DISPLAY_VER(dev_priv) >= 11)
5630 /* Display WA #1140: glk,cnl */
5631 if (DISPLAY_VER(dev_priv) == 10)
5634 trans_amount = 10; /* This is configurable amount */
5636 trans_offset = trans_min + trans_amount;
5639 * The spec asks for Selected Result Blocks for wm0 (the real value),
5640 * not Result Blocks (the integer value). Pay attention to the capital
5641 * letters. The value wm_l0->blocks is actually Result Blocks, but
5642 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
5643 * and since we later will have to get the ceiling of the sum in the
5644 * transition watermarks calculation, we can just pretend Selected
5645 * Result Blocks is Result Blocks minus 1 and it should work for the
5646 * current platforms.
5648 wm0_blocks = wm0->blocks - 1;
5652 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
5653 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
5655 blocks = wm0_blocks + trans_offset;
5660 * Just assume we can enable the transition watermark. After
5661 * computing the DDB we'll come back and disable it if that
5662 * assumption turns out to be false.
5664 trans_wm->blocks = blocks;
5665 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
5666 trans_wm->enable = true;
5669 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
5670 const struct intel_plane_state *plane_state,
5671 struct intel_plane *plane, int color_plane)
5673 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
5674 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5675 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
5676 struct skl_wm_params wm_params;
5679 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5680 &wm_params, color_plane);
5684 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
5686 skl_compute_transition_wm(dev_priv, &wm->trans_wm,
5687 &wm->wm[0], &wm_params);
5689 if (DISPLAY_VER(dev_priv) >= 12) {
5690 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
5692 skl_compute_transition_wm(dev_priv, &wm->sagv.trans_wm,
5693 &wm->sagv.wm0, &wm_params);
5699 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
5700 const struct intel_plane_state *plane_state,
5701 struct intel_plane *plane)
5703 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
5704 struct skl_wm_params wm_params;
5707 wm->is_planar = true;
5709 /* uv plane watermarks must also be validated for NV12/Planar */
5710 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
5715 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
5720 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
5721 const struct intel_plane_state *plane_state)
5723 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5724 enum plane_id plane_id = plane->id;
5725 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5726 const struct drm_framebuffer *fb = plane_state->hw.fb;
5729 memset(wm, 0, sizeof(*wm));
5731 if (!intel_wm_plane_visible(crtc_state, plane_state))
5734 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5739 if (fb->format->is_yuv && fb->format->num_planes > 1) {
5740 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
5749 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
5750 const struct intel_plane_state *plane_state)
5752 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
5753 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5754 enum plane_id plane_id = plane->id;
5755 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
5758 /* Watermarks calculated in master */
5759 if (plane_state->planar_slave)
5762 memset(wm, 0, sizeof(*wm));
5764 if (plane_state->planar_linked_plane) {
5765 const struct drm_framebuffer *fb = plane_state->hw.fb;
5767 drm_WARN_ON(&dev_priv->drm,
5768 !intel_wm_plane_visible(crtc_state, plane_state));
5769 drm_WARN_ON(&dev_priv->drm, !fb->format->is_yuv ||
5770 fb->format->num_planes == 1);
5772 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5773 plane_state->planar_linked_plane, 0);
5777 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5781 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
5782 ret = skl_build_plane_wm_single(crtc_state, plane_state,
5791 static int skl_build_pipe_wm(struct intel_atomic_state *state,
5792 struct intel_crtc *crtc)
5794 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5795 struct intel_crtc_state *crtc_state =
5796 intel_atomic_get_new_crtc_state(state, crtc);
5797 const struct intel_plane_state *plane_state;
5798 struct intel_plane *plane;
5801 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
5803 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
5804 * instead but we don't populate that correctly for NV12 Y
5805 * planes so for now hack this.
5807 if (plane->pipe != crtc->pipe)
5810 if (DISPLAY_VER(dev_priv) >= 11)
5811 ret = icl_build_plane_wm(crtc_state, plane_state);
5813 ret = skl_build_plane_wm(crtc_state, plane_state);
5818 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
5823 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
5825 const struct skl_ddb_entry *entry)
5828 intel_de_write_fw(dev_priv, reg,
5829 PLANE_BUF_END(entry->end - 1) |
5830 PLANE_BUF_START(entry->start));
5832 intel_de_write_fw(dev_priv, reg, 0);
5835 static void skl_write_wm_level(struct drm_i915_private *dev_priv,
5837 const struct skl_wm_level *level)
5843 if (level->ignore_lines)
5844 val |= PLANE_WM_IGNORE_LINES;
5845 val |= REG_FIELD_PREP(PLANE_WM_BLOCKS_MASK, level->blocks);
5846 val |= REG_FIELD_PREP(PLANE_WM_LINES_MASK, level->lines);
5848 intel_de_write_fw(dev_priv, reg, val);
5851 void skl_write_plane_wm(struct intel_plane *plane,
5852 const struct intel_crtc_state *crtc_state)
5854 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5855 int level, max_level = ilk_wm_max_level(dev_priv);
5856 enum plane_id plane_id = plane->id;
5857 enum pipe pipe = plane->pipe;
5858 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5859 const struct skl_ddb_entry *ddb =
5860 &crtc_state->wm.skl.plane_ddb[plane_id];
5861 const struct skl_ddb_entry *ddb_y =
5862 &crtc_state->wm.skl.plane_ddb_y[plane_id];
5864 for (level = 0; level <= max_level; level++)
5865 skl_write_wm_level(dev_priv, PLANE_WM(pipe, plane_id, level),
5866 skl_plane_wm_level(pipe_wm, plane_id, level));
5868 skl_write_wm_level(dev_priv, PLANE_WM_TRANS(pipe, plane_id),
5869 skl_plane_trans_wm(pipe_wm, plane_id));
5871 if (HAS_HW_SAGV_WM(dev_priv)) {
5872 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5874 skl_write_wm_level(dev_priv, PLANE_WM_SAGV(pipe, plane_id),
5876 skl_write_wm_level(dev_priv, PLANE_WM_SAGV_TRANS(pipe, plane_id),
5877 &wm->sagv.trans_wm);
5880 skl_ddb_entry_write(dev_priv,
5881 PLANE_BUF_CFG(pipe, plane_id), ddb);
5883 if (DISPLAY_VER(dev_priv) < 11)
5884 skl_ddb_entry_write(dev_priv,
5885 PLANE_NV12_BUF_CFG(pipe, plane_id), ddb_y);
5888 void skl_write_cursor_wm(struct intel_plane *plane,
5889 const struct intel_crtc_state *crtc_state)
5891 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
5892 int level, max_level = ilk_wm_max_level(dev_priv);
5893 enum plane_id plane_id = plane->id;
5894 enum pipe pipe = plane->pipe;
5895 const struct skl_pipe_wm *pipe_wm = &crtc_state->wm.skl.optimal;
5896 const struct skl_ddb_entry *ddb =
5897 &crtc_state->wm.skl.plane_ddb[plane_id];
5899 for (level = 0; level <= max_level; level++)
5900 skl_write_wm_level(dev_priv, CUR_WM(pipe, level),
5901 skl_plane_wm_level(pipe_wm, plane_id, level));
5903 skl_write_wm_level(dev_priv, CUR_WM_TRANS(pipe),
5904 skl_plane_trans_wm(pipe_wm, plane_id));
5906 if (HAS_HW_SAGV_WM(dev_priv)) {
5907 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
5909 skl_write_wm_level(dev_priv, CUR_WM_SAGV(pipe),
5911 skl_write_wm_level(dev_priv, CUR_WM_SAGV_TRANS(pipe),
5912 &wm->sagv.trans_wm);
5915 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe), ddb);
5918 bool skl_wm_level_equals(const struct skl_wm_level *l1,
5919 const struct skl_wm_level *l2)
5921 return l1->enable == l2->enable &&
5922 l1->ignore_lines == l2->ignore_lines &&
5923 l1->lines == l2->lines &&
5924 l1->blocks == l2->blocks;
5927 static bool skl_plane_wm_equals(struct drm_i915_private *dev_priv,
5928 const struct skl_plane_wm *wm1,
5929 const struct skl_plane_wm *wm2)
5931 int level, max_level = ilk_wm_max_level(dev_priv);
5933 for (level = 0; level <= max_level; level++) {
5935 * We don't check uv_wm as the hardware doesn't actually
5936 * use it. It only gets used for calculating the required
5939 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
5943 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
5944 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
5945 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
5948 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
5949 const struct skl_ddb_entry *b)
5951 return a->start < b->end && b->start < a->end;
5954 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
5955 const struct skl_ddb_entry *b)
5957 if (a->end && b->end) {
5958 a->start = min(a->start, b->start);
5959 a->end = max(a->end, b->end);
5960 } else if (b->end) {
5961 a->start = b->start;
5966 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
5967 const struct skl_ddb_entry *entries,
5968 int num_entries, int ignore_idx)
5972 for (i = 0; i < num_entries; i++) {
5973 if (i != ignore_idx &&
5974 skl_ddb_entries_overlap(ddb, &entries[i]))
5982 skl_ddb_add_affected_planes(const struct intel_crtc_state *old_crtc_state,
5983 struct intel_crtc_state *new_crtc_state)
5985 struct intel_atomic_state *state = to_intel_atomic_state(new_crtc_state->uapi.state);
5986 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
5987 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
5988 struct intel_plane *plane;
5990 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
5991 struct intel_plane_state *plane_state;
5992 enum plane_id plane_id = plane->id;
5994 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
5995 &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
5996 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
5997 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
6000 plane_state = intel_atomic_get_plane_state(state, plane);
6001 if (IS_ERR(plane_state))
6002 return PTR_ERR(plane_state);
6004 new_crtc_state->update_planes |= BIT(plane_id);
6010 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
6012 struct drm_i915_private *dev_priv = to_i915(dbuf_state->base.state->base.dev);
6017 * FIXME: For now we always enable slice S1 as per
6018 * the Bspec display initialization sequence.
6020 enabled_slices = BIT(DBUF_S1);
6022 for_each_pipe(dev_priv, pipe)
6023 enabled_slices |= dbuf_state->slices[pipe];
6025 return enabled_slices;
6029 skl_compute_ddb(struct intel_atomic_state *state)
6031 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6032 const struct intel_dbuf_state *old_dbuf_state;
6033 struct intel_dbuf_state *new_dbuf_state = NULL;
6034 const struct intel_crtc_state *old_crtc_state;
6035 struct intel_crtc_state *new_crtc_state;
6036 struct intel_crtc *crtc;
6039 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6040 new_dbuf_state = intel_atomic_get_dbuf_state(state);
6041 if (IS_ERR(new_dbuf_state))
6042 return PTR_ERR(new_dbuf_state);
6044 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
6048 if (!new_dbuf_state)
6051 new_dbuf_state->active_pipes =
6052 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
6054 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
6055 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6060 if (HAS_MBUS_JOINING(dev_priv))
6061 new_dbuf_state->joined_mbus =
6062 adlp_check_mbus_joined(new_dbuf_state->active_pipes);
6064 for_each_intel_crtc(&dev_priv->drm, crtc) {
6065 enum pipe pipe = crtc->pipe;
6067 new_dbuf_state->slices[pipe] =
6068 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
6069 new_dbuf_state->joined_mbus);
6071 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
6074 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6079 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
6081 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
6082 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6083 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
6087 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
6088 /* TODO: Implement vblank synchronized MBUS joining changes */
6089 ret = intel_modeset_all_pipes(state);
6094 drm_dbg_kms(&dev_priv->drm,
6095 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
6096 old_dbuf_state->enabled_slices,
6097 new_dbuf_state->enabled_slices,
6098 INTEL_INFO(dev_priv)->dbuf.slice_mask,
6099 str_yes_no(old_dbuf_state->joined_mbus),
6100 str_yes_no(new_dbuf_state->joined_mbus));
6103 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6104 enum pipe pipe = crtc->pipe;
6106 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
6108 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
6111 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
6116 for_each_intel_crtc(&dev_priv->drm, crtc) {
6117 ret = skl_crtc_allocate_ddb(state, crtc);
6122 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6123 new_crtc_state, i) {
6124 ret = skl_crtc_allocate_plane_ddb(state, crtc);
6128 ret = skl_ddb_add_affected_planes(old_crtc_state,
6137 static char enast(bool enable)
6139 return enable ? '*' : ' ';
6143 skl_print_wm_changes(struct intel_atomic_state *state)
6145 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
6146 const struct intel_crtc_state *old_crtc_state;
6147 const struct intel_crtc_state *new_crtc_state;
6148 struct intel_plane *plane;
6149 struct intel_crtc *crtc;
6152 if (!drm_debug_enabled(DRM_UT_KMS))
6155 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
6156 new_crtc_state, i) {
6157 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
6159 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
6160 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
6162 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6163 enum plane_id plane_id = plane->id;
6164 const struct skl_ddb_entry *old, *new;
6166 old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
6167 new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
6169 if (skl_ddb_entry_equal(old, new))
6172 drm_dbg_kms(&dev_priv->drm,
6173 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
6174 plane->base.base.id, plane->base.name,
6175 old->start, old->end, new->start, new->end,
6176 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
6179 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6180 enum plane_id plane_id = plane->id;
6181 const struct skl_plane_wm *old_wm, *new_wm;
6183 old_wm = &old_pipe_wm->planes[plane_id];
6184 new_wm = &new_pipe_wm->planes[plane_id];
6186 if (skl_plane_wm_equals(dev_priv, old_wm, new_wm))
6189 drm_dbg_kms(&dev_priv->drm,
6190 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
6191 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
6192 plane->base.base.id, plane->base.name,
6193 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
6194 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
6195 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
6196 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
6197 enast(old_wm->trans_wm.enable),
6198 enast(old_wm->sagv.wm0.enable),
6199 enast(old_wm->sagv.trans_wm.enable),
6200 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
6201 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
6202 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
6203 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
6204 enast(new_wm->trans_wm.enable),
6205 enast(new_wm->sagv.wm0.enable),
6206 enast(new_wm->sagv.trans_wm.enable));
6208 drm_dbg_kms(&dev_priv->drm,
6209 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
6210 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
6211 plane->base.base.id, plane->base.name,
6212 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
6213 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
6214 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
6215 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
6216 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
6217 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
6218 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
6219 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
6220 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
6221 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
6222 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
6223 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
6224 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
6225 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
6226 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
6227 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
6228 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
6229 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
6230 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
6231 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
6232 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
6233 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
6235 drm_dbg_kms(&dev_priv->drm,
6236 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6237 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6238 plane->base.base.id, plane->base.name,
6239 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
6240 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
6241 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
6242 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
6243 old_wm->trans_wm.blocks,
6244 old_wm->sagv.wm0.blocks,
6245 old_wm->sagv.trans_wm.blocks,
6246 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
6247 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
6248 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
6249 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
6250 new_wm->trans_wm.blocks,
6251 new_wm->sagv.wm0.blocks,
6252 new_wm->sagv.trans_wm.blocks);
6254 drm_dbg_kms(&dev_priv->drm,
6255 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
6256 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
6257 plane->base.base.id, plane->base.name,
6258 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
6259 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
6260 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
6261 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
6262 old_wm->trans_wm.min_ddb_alloc,
6263 old_wm->sagv.wm0.min_ddb_alloc,
6264 old_wm->sagv.trans_wm.min_ddb_alloc,
6265 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
6266 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
6267 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
6268 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
6269 new_wm->trans_wm.min_ddb_alloc,
6270 new_wm->sagv.wm0.min_ddb_alloc,
6271 new_wm->sagv.trans_wm.min_ddb_alloc);
6276 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
6277 const struct skl_pipe_wm *old_pipe_wm,
6278 const struct skl_pipe_wm *new_pipe_wm)
6280 struct drm_i915_private *i915 = to_i915(plane->base.dev);
6281 int level, max_level = ilk_wm_max_level(i915);
6283 for (level = 0; level <= max_level; level++) {
6285 * We don't check uv_wm as the hardware doesn't actually
6286 * use it. It only gets used for calculating the required
6289 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
6290 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
6294 if (HAS_HW_SAGV_WM(i915)) {
6295 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
6296 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
6298 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
6299 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
6303 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
6304 skl_plane_trans_wm(new_pipe_wm, plane->id));
6308 * To make sure the cursor watermark registers are always consistent
6309 * with our computed state the following scenario needs special
6313 * 2. move cursor entirely offscreen
6316 * Step 2. does call .disable_plane() but does not zero the watermarks
6317 * (since we consider an offscreen cursor still active for the purposes
6318 * of watermarks). Step 3. would not normally call .disable_plane()
6319 * because the actual plane visibility isn't changing, and we don't
6320 * deallocate the cursor ddb until the pipe gets disabled. So we must
6321 * force step 3. to call .disable_plane() to update the watermark
6322 * registers properly.
6324 * Other planes do not suffer from this issues as their watermarks are
6325 * calculated based on the actual plane visibility. The only time this
6326 * can trigger for the other planes is during the initial readout as the
6327 * default value of the watermarks registers is not zero.
6329 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
6330 struct intel_crtc *crtc)
6332 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6333 const struct intel_crtc_state *old_crtc_state =
6334 intel_atomic_get_old_crtc_state(state, crtc);
6335 struct intel_crtc_state *new_crtc_state =
6336 intel_atomic_get_new_crtc_state(state, crtc);
6337 struct intel_plane *plane;
6339 for_each_intel_plane_on_crtc(&dev_priv->drm, crtc, plane) {
6340 struct intel_plane_state *plane_state;
6341 enum plane_id plane_id = plane->id;
6344 * Force a full wm update for every plane on modeset.
6345 * Required because the reset value of the wm registers
6346 * is non-zero, whereas we want all disabled planes to
6347 * have zero watermarks. So if we turn off the relevant
6348 * power well the hardware state will go out of sync
6349 * with the software state.
6351 if (!drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) &&
6352 skl_plane_selected_wm_equals(plane,
6353 &old_crtc_state->wm.skl.optimal,
6354 &new_crtc_state->wm.skl.optimal))
6357 plane_state = intel_atomic_get_plane_state(state, plane);
6358 if (IS_ERR(plane_state))
6359 return PTR_ERR(plane_state);
6361 new_crtc_state->update_planes |= BIT(plane_id);
6368 skl_compute_wm(struct intel_atomic_state *state)
6370 struct intel_crtc *crtc;
6371 struct intel_crtc_state *new_crtc_state;
6374 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6375 ret = skl_build_pipe_wm(state, crtc);
6380 ret = skl_compute_ddb(state);
6384 ret = intel_compute_sagv_mask(state);
6389 * skl_compute_ddb() will have adjusted the final watermarks
6390 * based on how much ddb is available. Now we can actually
6391 * check if the final watermarks changed.
6393 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
6394 ret = skl_wm_add_affected_planes(state, crtc);
6399 skl_print_wm_changes(state);
6404 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
6405 struct intel_wm_config *config)
6407 struct intel_crtc *crtc;
6409 /* Compute the currently _active_ config */
6410 for_each_intel_crtc(&dev_priv->drm, crtc) {
6411 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
6413 if (!wm->pipe_enabled)
6416 config->sprites_enabled |= wm->sprites_enabled;
6417 config->sprites_scaled |= wm->sprites_scaled;
6418 config->num_pipes_active++;
6422 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
6424 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
6425 struct ilk_wm_maximums max;
6426 struct intel_wm_config config = {};
6427 struct ilk_wm_values results = {};
6428 enum intel_ddb_partitioning partitioning;
6430 ilk_compute_wm_config(dev_priv, &config);
6432 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
6433 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
6435 /* 5/6 split only in single pipe config on IVB+ */
6436 if (DISPLAY_VER(dev_priv) >= 7 &&
6437 config.num_pipes_active == 1 && config.sprites_enabled) {
6438 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
6439 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
6441 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
6443 best_lp_wm = &lp_wm_1_2;
6446 partitioning = (best_lp_wm == &lp_wm_1_2) ?
6447 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
6449 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
6451 ilk_write_wm_values(dev_priv, &results);
6454 static void ilk_initial_watermarks(struct intel_atomic_state *state,
6455 struct intel_crtc *crtc)
6457 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6458 const struct intel_crtc_state *crtc_state =
6459 intel_atomic_get_new_crtc_state(state, crtc);
6461 mutex_lock(&dev_priv->wm.wm_mutex);
6462 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
6463 ilk_program_watermarks(dev_priv);
6464 mutex_unlock(&dev_priv->wm.wm_mutex);
6467 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
6468 struct intel_crtc *crtc)
6470 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6471 const struct intel_crtc_state *crtc_state =
6472 intel_atomic_get_new_crtc_state(state, crtc);
6474 if (!crtc_state->wm.need_postvbl_update)
6477 mutex_lock(&dev_priv->wm.wm_mutex);
6478 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
6479 ilk_program_watermarks(dev_priv);
6480 mutex_unlock(&dev_priv->wm.wm_mutex);
6483 static void skl_wm_level_from_reg_val(u32 val, struct skl_wm_level *level)
6485 level->enable = val & PLANE_WM_EN;
6486 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
6487 level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
6488 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
6491 void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
6492 struct skl_pipe_wm *out)
6494 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
6495 enum pipe pipe = crtc->pipe;
6496 int level, max_level;
6497 enum plane_id plane_id;
6500 max_level = ilk_wm_max_level(dev_priv);
6502 for_each_plane_id_on_crtc(crtc, plane_id) {
6503 struct skl_plane_wm *wm = &out->planes[plane_id];
6505 for (level = 0; level <= max_level; level++) {
6506 if (plane_id != PLANE_CURSOR)
6507 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM(pipe, plane_id, level));
6509 val = intel_uncore_read(&dev_priv->uncore, CUR_WM(pipe, level));
6511 skl_wm_level_from_reg_val(val, &wm->wm[level]);
6514 if (plane_id != PLANE_CURSOR)
6515 val = intel_uncore_read(&dev_priv->uncore, PLANE_WM_TRANS(pipe, plane_id));
6517 val = intel_uncore_read(&dev_priv->uncore, CUR_WM_TRANS(pipe));
6519 skl_wm_level_from_reg_val(val, &wm->trans_wm);
6521 if (HAS_HW_SAGV_WM(dev_priv)) {
6522 if (plane_id != PLANE_CURSOR)
6523 val = intel_uncore_read(&dev_priv->uncore,
6524 PLANE_WM_SAGV(pipe, plane_id));
6526 val = intel_uncore_read(&dev_priv->uncore,
6529 skl_wm_level_from_reg_val(val, &wm->sagv.wm0);
6531 if (plane_id != PLANE_CURSOR)
6532 val = intel_uncore_read(&dev_priv->uncore,
6533 PLANE_WM_SAGV_TRANS(pipe, plane_id));
6535 val = intel_uncore_read(&dev_priv->uncore,
6536 CUR_WM_SAGV_TRANS(pipe));
6538 skl_wm_level_from_reg_val(val, &wm->sagv.trans_wm);
6539 } else if (DISPLAY_VER(dev_priv) >= 12) {
6540 wm->sagv.wm0 = wm->wm[0];
6541 wm->sagv.trans_wm = wm->trans_wm;
6546 void skl_wm_get_hw_state(struct drm_i915_private *dev_priv)
6548 struct intel_dbuf_state *dbuf_state =
6549 to_intel_dbuf_state(dev_priv->dbuf.obj.state);
6550 struct intel_crtc *crtc;
6552 if (HAS_MBUS_JOINING(dev_priv))
6553 dbuf_state->joined_mbus = intel_de_read(dev_priv, MBUS_CTL) & MBUS_JOIN;
6555 for_each_intel_crtc(&dev_priv->drm, crtc) {
6556 struct intel_crtc_state *crtc_state =
6557 to_intel_crtc_state(crtc->base.state);
6558 enum pipe pipe = crtc->pipe;
6559 unsigned int mbus_offset;
6560 enum plane_id plane_id;
6563 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
6564 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
6566 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
6568 for_each_plane_id_on_crtc(crtc, plane_id) {
6569 struct skl_ddb_entry *ddb =
6570 &crtc_state->wm.skl.plane_ddb[plane_id];
6571 struct skl_ddb_entry *ddb_y =
6572 &crtc_state->wm.skl.plane_ddb_y[plane_id];
6574 skl_ddb_get_hw_plane_state(dev_priv, crtc->pipe,
6575 plane_id, ddb, ddb_y);
6577 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
6578 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
6581 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
6584 * Used for checking overlaps, so we need absolute
6585 * offsets instead of MBUS relative offsets.
6587 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
6588 dbuf_state->joined_mbus);
6589 mbus_offset = mbus_ddb_offset(dev_priv, slices);
6590 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
6591 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
6593 /* The slices actually used by the planes on the pipe */
6594 dbuf_state->slices[pipe] =
6595 skl_ddb_dbuf_slice_mask(dev_priv, &crtc_state->wm.skl.ddb);
6597 drm_dbg_kms(&dev_priv->drm,
6598 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
6599 crtc->base.base.id, crtc->base.name,
6600 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
6601 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
6602 str_yes_no(dbuf_state->joined_mbus));
6605 dbuf_state->enabled_slices = dev_priv->dbuf.enabled_slices;
6608 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
6610 const struct intel_dbuf_state *dbuf_state =
6611 to_intel_dbuf_state(i915->dbuf.obj.state);
6612 struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
6613 struct intel_crtc *crtc;
6615 for_each_intel_crtc(&i915->drm, crtc) {
6616 const struct intel_crtc_state *crtc_state =
6617 to_intel_crtc_state(crtc->base.state);
6619 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
6622 for_each_intel_crtc(&i915->drm, crtc) {
6623 const struct intel_crtc_state *crtc_state =
6624 to_intel_crtc_state(crtc->base.state);
6627 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
6628 dbuf_state->joined_mbus);
6629 if (dbuf_state->slices[crtc->pipe] & ~slices)
6632 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
6633 I915_MAX_PIPES, crtc->pipe))
6640 void skl_wm_sanitize(struct drm_i915_private *i915)
6642 struct intel_crtc *crtc;
6645 * On TGL/RKL (at least) the BIOS likes to assign the planes
6646 * to the wrong DBUF slices. This will cause an infinite loop
6647 * in skl_commit_modeset_enables() as it can't find a way to
6648 * transition between the old bogus DBUF layout to the new
6649 * proper DBUF layout without DBUF allocation overlaps between
6650 * the planes (which cannot be allowed or else the hardware
6651 * may hang). If we detect a bogus DBUF layout just turn off
6652 * all the planes so that skl_commit_modeset_enables() can
6653 * simply ignore them.
6655 if (!skl_dbuf_is_misconfigured(i915))
6658 drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
6660 for_each_intel_crtc(&i915->drm, crtc) {
6661 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
6662 const struct intel_plane_state *plane_state =
6663 to_intel_plane_state(plane->base.state);
6664 struct intel_crtc_state *crtc_state =
6665 to_intel_crtc_state(crtc->base.state);
6667 if (plane_state->uapi.visible)
6668 intel_plane_disable_noatomic(crtc, plane);
6670 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
6672 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
6676 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
6678 struct drm_device *dev = crtc->base.dev;
6679 struct drm_i915_private *dev_priv = to_i915(dev);
6680 struct ilk_wm_values *hw = &dev_priv->wm.hw;
6681 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
6682 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
6683 enum pipe pipe = crtc->pipe;
6685 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
6687 memset(active, 0, sizeof(*active));
6689 active->pipe_enabled = crtc->active;
6691 if (active->pipe_enabled) {
6692 u32 tmp = hw->wm_pipe[pipe];
6695 * For active pipes LP0 watermark is marked as
6696 * enabled, and LP1+ watermaks as disabled since
6697 * we can't really reverse compute them in case
6698 * multiple pipes are active.
6700 active->wm[0].enable = true;
6701 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
6702 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
6703 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
6705 int level, max_level = ilk_wm_max_level(dev_priv);
6708 * For inactive pipes, all watermark levels
6709 * should be marked as enabled but zeroed,
6710 * which is what we'd compute them to.
6712 for (level = 0; level <= max_level; level++)
6713 active->wm[level].enable = true;
6716 crtc->wm.active.ilk = *active;
6719 #define _FW_WM(value, plane) \
6720 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
6721 #define _FW_WM_VLV(value, plane) \
6722 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
6724 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
6725 struct g4x_wm_values *wm)
6729 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6730 wm->sr.plane = _FW_WM(tmp, SR);
6731 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6732 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
6733 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
6735 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6736 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
6737 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
6738 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
6739 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
6740 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6741 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
6743 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6744 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
6745 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6746 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
6747 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
6750 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
6751 struct vlv_wm_values *wm)
6756 for_each_pipe(dev_priv, pipe) {
6757 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
6759 wm->ddl[pipe].plane[PLANE_PRIMARY] =
6760 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6761 wm->ddl[pipe].plane[PLANE_CURSOR] =
6762 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6763 wm->ddl[pipe].plane[PLANE_SPRITE0] =
6764 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6765 wm->ddl[pipe].plane[PLANE_SPRITE1] =
6766 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
6769 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
6770 wm->sr.plane = _FW_WM(tmp, SR);
6771 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
6772 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
6773 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
6775 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
6776 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
6777 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
6778 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
6780 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
6781 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
6783 if (IS_CHERRYVIEW(dev_priv)) {
6784 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
6785 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6786 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6788 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
6789 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
6790 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
6792 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
6793 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
6794 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
6796 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6797 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6798 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
6799 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
6800 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
6801 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6802 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6803 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6804 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6805 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6806 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6808 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
6809 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
6810 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
6812 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
6813 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
6814 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
6815 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
6816 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
6817 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
6818 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
6819 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
6826 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
6828 struct g4x_wm_values *wm = &dev_priv->wm.g4x;
6829 struct intel_crtc *crtc;
6831 g4x_read_wm_values(dev_priv, wm);
6833 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
6835 for_each_intel_crtc(&dev_priv->drm, crtc) {
6836 struct intel_crtc_state *crtc_state =
6837 to_intel_crtc_state(crtc->base.state);
6838 struct g4x_wm_state *active = &crtc->wm.active.g4x;
6839 struct g4x_pipe_wm *raw;
6840 enum pipe pipe = crtc->pipe;
6841 enum plane_id plane_id;
6842 int level, max_level;
6844 active->cxsr = wm->cxsr;
6845 active->hpll_en = wm->hpll_en;
6846 active->fbc_en = wm->fbc_en;
6848 active->sr = wm->sr;
6849 active->hpll = wm->hpll;
6851 for_each_plane_id_on_crtc(crtc, plane_id) {
6852 active->wm.plane[plane_id] =
6853 wm->pipe[pipe].plane[plane_id];
6856 if (wm->cxsr && wm->hpll_en)
6857 max_level = G4X_WM_LEVEL_HPLL;
6859 max_level = G4X_WM_LEVEL_SR;
6861 max_level = G4X_WM_LEVEL_NORMAL;
6863 level = G4X_WM_LEVEL_NORMAL;
6864 raw = &crtc_state->wm.g4x.raw[level];
6865 for_each_plane_id_on_crtc(crtc, plane_id)
6866 raw->plane[plane_id] = active->wm.plane[plane_id];
6868 level = G4X_WM_LEVEL_SR;
6869 if (level > max_level)
6872 raw = &crtc_state->wm.g4x.raw[level];
6873 raw->plane[PLANE_PRIMARY] = active->sr.plane;
6874 raw->plane[PLANE_CURSOR] = active->sr.cursor;
6875 raw->plane[PLANE_SPRITE0] = 0;
6876 raw->fbc = active->sr.fbc;
6878 level = G4X_WM_LEVEL_HPLL;
6879 if (level > max_level)
6882 raw = &crtc_state->wm.g4x.raw[level];
6883 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
6884 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
6885 raw->plane[PLANE_SPRITE0] = 0;
6886 raw->fbc = active->hpll.fbc;
6890 for_each_plane_id_on_crtc(crtc, plane_id)
6891 g4x_raw_plane_wm_set(crtc_state, level,
6892 plane_id, USHRT_MAX);
6893 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
6895 crtc_state->wm.g4x.optimal = *active;
6896 crtc_state->wm.g4x.intermediate = *active;
6898 drm_dbg_kms(&dev_priv->drm,
6899 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
6901 wm->pipe[pipe].plane[PLANE_PRIMARY],
6902 wm->pipe[pipe].plane[PLANE_CURSOR],
6903 wm->pipe[pipe].plane[PLANE_SPRITE0]);
6906 drm_dbg_kms(&dev_priv->drm,
6907 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
6908 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
6909 drm_dbg_kms(&dev_priv->drm,
6910 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
6911 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
6912 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
6913 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
6914 str_yes_no(wm->fbc_en));
6917 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
6919 struct intel_plane *plane;
6920 struct intel_crtc *crtc;
6922 mutex_lock(&dev_priv->wm.wm_mutex);
6924 for_each_intel_plane(&dev_priv->drm, plane) {
6925 struct intel_crtc *crtc =
6926 intel_crtc_for_pipe(dev_priv, plane->pipe);
6927 struct intel_crtc_state *crtc_state =
6928 to_intel_crtc_state(crtc->base.state);
6929 struct intel_plane_state *plane_state =
6930 to_intel_plane_state(plane->base.state);
6931 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
6932 enum plane_id plane_id = plane->id;
6935 if (plane_state->uapi.visible)
6938 for (level = 0; level < 3; level++) {
6939 struct g4x_pipe_wm *raw =
6940 &crtc_state->wm.g4x.raw[level];
6942 raw->plane[plane_id] = 0;
6943 wm_state->wm.plane[plane_id] = 0;
6946 if (plane_id == PLANE_PRIMARY) {
6947 for (level = 0; level < 3; level++) {
6948 struct g4x_pipe_wm *raw =
6949 &crtc_state->wm.g4x.raw[level];
6953 wm_state->sr.fbc = 0;
6954 wm_state->hpll.fbc = 0;
6955 wm_state->fbc_en = false;
6959 for_each_intel_crtc(&dev_priv->drm, crtc) {
6960 struct intel_crtc_state *crtc_state =
6961 to_intel_crtc_state(crtc->base.state);
6963 crtc_state->wm.g4x.intermediate =
6964 crtc_state->wm.g4x.optimal;
6965 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
6968 g4x_program_watermarks(dev_priv);
6970 mutex_unlock(&dev_priv->wm.wm_mutex);
6973 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
6975 struct vlv_wm_values *wm = &dev_priv->wm.vlv;
6976 struct intel_crtc *crtc;
6979 vlv_read_wm_values(dev_priv, wm);
6981 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
6982 wm->level = VLV_WM_LEVEL_PM2;
6984 if (IS_CHERRYVIEW(dev_priv)) {
6985 vlv_punit_get(dev_priv);
6987 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
6988 if (val & DSP_MAXFIFO_PM5_ENABLE)
6989 wm->level = VLV_WM_LEVEL_PM5;
6992 * If DDR DVFS is disabled in the BIOS, Punit
6993 * will never ack the request. So if that happens
6994 * assume we don't have to enable/disable DDR DVFS
6995 * dynamically. To test that just set the REQ_ACK
6996 * bit to poke the Punit, but don't change the
6997 * HIGH/LOW bits so that we don't actually change
6998 * the current state.
7000 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
7001 val |= FORCE_DDR_FREQ_REQ_ACK;
7002 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
7004 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
7005 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
7006 drm_dbg_kms(&dev_priv->drm,
7007 "Punit not acking DDR DVFS request, "
7008 "assuming DDR DVFS is disabled\n");
7009 dev_priv->wm.max_level = VLV_WM_LEVEL_PM5;
7011 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
7012 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
7013 wm->level = VLV_WM_LEVEL_DDR_DVFS;
7016 vlv_punit_put(dev_priv);
7019 for_each_intel_crtc(&dev_priv->drm, crtc) {
7020 struct intel_crtc_state *crtc_state =
7021 to_intel_crtc_state(crtc->base.state);
7022 struct vlv_wm_state *active = &crtc->wm.active.vlv;
7023 const struct vlv_fifo_state *fifo_state =
7024 &crtc_state->wm.vlv.fifo_state;
7025 enum pipe pipe = crtc->pipe;
7026 enum plane_id plane_id;
7029 vlv_get_fifo_size(crtc_state);
7031 active->num_levels = wm->level + 1;
7032 active->cxsr = wm->cxsr;
7034 for (level = 0; level < active->num_levels; level++) {
7035 struct g4x_pipe_wm *raw =
7036 &crtc_state->wm.vlv.raw[level];
7038 active->sr[level].plane = wm->sr.plane;
7039 active->sr[level].cursor = wm->sr.cursor;
7041 for_each_plane_id_on_crtc(crtc, plane_id) {
7042 active->wm[level].plane[plane_id] =
7043 wm->pipe[pipe].plane[plane_id];
7045 raw->plane[plane_id] =
7046 vlv_invert_wm_value(active->wm[level].plane[plane_id],
7047 fifo_state->plane[plane_id]);
7051 for_each_plane_id_on_crtc(crtc, plane_id)
7052 vlv_raw_plane_wm_set(crtc_state, level,
7053 plane_id, USHRT_MAX);
7054 vlv_invalidate_wms(crtc, active, level);
7056 crtc_state->wm.vlv.optimal = *active;
7057 crtc_state->wm.vlv.intermediate = *active;
7059 drm_dbg_kms(&dev_priv->drm,
7060 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
7062 wm->pipe[pipe].plane[PLANE_PRIMARY],
7063 wm->pipe[pipe].plane[PLANE_CURSOR],
7064 wm->pipe[pipe].plane[PLANE_SPRITE0],
7065 wm->pipe[pipe].plane[PLANE_SPRITE1]);
7068 drm_dbg_kms(&dev_priv->drm,
7069 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
7070 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
7073 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
7075 struct intel_plane *plane;
7076 struct intel_crtc *crtc;
7078 mutex_lock(&dev_priv->wm.wm_mutex);
7080 for_each_intel_plane(&dev_priv->drm, plane) {
7081 struct intel_crtc *crtc =
7082 intel_crtc_for_pipe(dev_priv, plane->pipe);
7083 struct intel_crtc_state *crtc_state =
7084 to_intel_crtc_state(crtc->base.state);
7085 struct intel_plane_state *plane_state =
7086 to_intel_plane_state(plane->base.state);
7087 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
7088 const struct vlv_fifo_state *fifo_state =
7089 &crtc_state->wm.vlv.fifo_state;
7090 enum plane_id plane_id = plane->id;
7093 if (plane_state->uapi.visible)
7096 for (level = 0; level < wm_state->num_levels; level++) {
7097 struct g4x_pipe_wm *raw =
7098 &crtc_state->wm.vlv.raw[level];
7100 raw->plane[plane_id] = 0;
7102 wm_state->wm[level].plane[plane_id] =
7103 vlv_invert_wm_value(raw->plane[plane_id],
7104 fifo_state->plane[plane_id]);
7108 for_each_intel_crtc(&dev_priv->drm, crtc) {
7109 struct intel_crtc_state *crtc_state =
7110 to_intel_crtc_state(crtc->base.state);
7112 crtc_state->wm.vlv.intermediate =
7113 crtc_state->wm.vlv.optimal;
7114 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
7117 vlv_program_watermarks(dev_priv);
7119 mutex_unlock(&dev_priv->wm.wm_mutex);
7123 * FIXME should probably kill this and improve
7124 * the real watermark readout/sanitation instead
7126 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
7128 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM_LP_ENABLE);
7129 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM_LP_ENABLE);
7130 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM_LP_ENABLE);
7133 * Don't touch WM_LP_SPRITE_ENABLE here.
7134 * Doing so could cause underruns.
7138 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
7140 struct ilk_wm_values *hw = &dev_priv->wm.hw;
7141 struct intel_crtc *crtc;
7143 ilk_init_lp_watermarks(dev_priv);
7145 for_each_intel_crtc(&dev_priv->drm, crtc)
7146 ilk_pipe_wm_get_hw_state(crtc);
7148 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
7149 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
7150 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
7152 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
7153 if (DISPLAY_VER(dev_priv) >= 7) {
7154 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
7155 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
7158 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
7159 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
7160 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7161 else if (IS_IVYBRIDGE(dev_priv))
7162 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
7163 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
7166 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
7169 void intel_enable_ipc(struct drm_i915_private *dev_priv)
7173 if (!HAS_IPC(dev_priv))
7176 val = intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2);
7178 if (dev_priv->ipc_enabled)
7179 val |= DISP_IPC_ENABLE;
7181 val &= ~DISP_IPC_ENABLE;
7183 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL2, val);
7186 static bool intel_can_enable_ipc(struct drm_i915_private *dev_priv)
7188 /* Display WA #0477 WaDisableIPC: skl */
7189 if (IS_SKYLAKE(dev_priv))
7192 /* Display WA #1141: SKL:all KBL:all CFL */
7193 if (IS_KABYLAKE(dev_priv) ||
7194 IS_COFFEELAKE(dev_priv) ||
7195 IS_COMETLAKE(dev_priv))
7196 return dev_priv->dram_info.symmetric_memory;
7201 void intel_init_ipc(struct drm_i915_private *dev_priv)
7203 if (!HAS_IPC(dev_priv))
7206 dev_priv->ipc_enabled = intel_can_enable_ipc(dev_priv);
7208 intel_enable_ipc(dev_priv);
7211 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
7214 * On Ibex Peak and Cougar Point, we need to disable clock
7215 * gating for the panel power sequencer or it will fail to
7216 * start up when no ports are active.
7218 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
7221 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
7225 for_each_pipe(dev_priv, pipe) {
7226 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
7227 intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
7228 DISP_TRICKLE_FEED_DISABLE);
7230 intel_uncore_write(&dev_priv->uncore, DSPSURF(pipe), intel_uncore_read(&dev_priv->uncore, DSPSURF(pipe)));
7231 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
7235 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
7237 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7241 * WaFbcDisableDpfcClockGating:ilk
7243 dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
7244 ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
7245 ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
7247 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
7248 MARIUNIT_CLOCK_GATE_DISABLE |
7249 SVSMUNIT_CLOCK_GATE_DISABLE);
7250 intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
7251 VFMUNIT_CLOCK_GATE_DISABLE);
7254 * According to the spec the following bits should be set in
7255 * order to enable memory self-refresh
7256 * The bit 22/21 of 0x42004
7257 * The bit 5 of 0x42020
7258 * The bit 15 of 0x45000
7260 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7261 (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7262 ILK_DPARB_GATE | ILK_VSDPFD_FULL));
7263 dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
7264 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
7265 (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7269 * Based on the document from hardware guys the following bits
7270 * should be set unconditionally in order to enable FBC.
7271 * The bit 22 of 0x42000
7272 * The bit 22 of 0x42004
7273 * The bit 7,8,9 of 0x42020.
7275 if (IS_IRONLAKE_M(dev_priv)) {
7276 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
7277 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7278 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7280 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7281 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7285 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7287 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7288 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7289 ILK_ELPIN_409_SELECT);
7291 g4x_disable_trickle_feed(dev_priv);
7293 ibx_init_clock_gating(dev_priv);
7296 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
7302 * On Ibex Peak and Cougar Point, we need to disable clock
7303 * gating for the panel power sequencer or it will fail to
7304 * start up when no ports are active.
7306 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
7307 PCH_DPLUNIT_CLOCK_GATE_DISABLE |
7308 PCH_CPUNIT_CLOCK_GATE_DISABLE);
7309 intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
7310 DPLS_EDP_PPS_FIX_DIS);
7311 /* The below fixes the weird display corruption, a few pixels shifted
7312 * downward, on (only) LVDS of some HP laptops with IVY.
7314 for_each_pipe(dev_priv, pipe) {
7315 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
7316 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
7317 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7318 if (dev_priv->vbt.fdi_rx_polarity_inverted)
7319 val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
7320 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
7321 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
7322 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
7324 /* WADP0ClockGatingDisable */
7325 for_each_pipe(dev_priv, pipe) {
7326 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
7327 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7331 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
7335 tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
7336 if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
7337 drm_dbg_kms(&dev_priv->drm,
7338 "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
7342 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
7344 u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
7346 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
7348 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7349 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7350 ILK_ELPIN_409_SELECT);
7352 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7353 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7354 GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
7355 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7357 /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
7358 * gating disable must be set. Failure to set it results in
7359 * flickering pixels due to Z write ordering failures after
7360 * some amount of runtime in the Mesa "fire" demo, and Unigine
7361 * Sanctuary and Tropics, and apparently anything else with
7362 * alpha test or pixel discard.
7364 * According to the spec, bit 11 (RCCUNIT) must also be set,
7365 * but we didn't debug actual testcases to find it out.
7367 * WaDisableRCCUnitClockGating:snb
7368 * WaDisableRCPBUnitClockGating:snb
7370 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7371 GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
7372 GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
7375 * According to the spec the following bits should be
7376 * set in order to enable memory self-refresh and fbc:
7377 * The bit21 and bit22 of 0x42000
7378 * The bit21 and bit22 of 0x42004
7379 * The bit5 and bit7 of 0x42020
7380 * The bit14 of 0x70180
7381 * The bit14 of 0x71180
7383 * WaFbcAsynchFlipDisableFbcQueue:snb
7385 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7386 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7387 ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
7388 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
7389 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
7390 ILK_DPARB_GATE | ILK_VSDPFD_FULL);
7391 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
7392 intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
7393 ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
7394 ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
7396 g4x_disable_trickle_feed(dev_priv);
7398 cpt_init_clock_gating(dev_priv);
7400 gen6_check_mch_setup(dev_priv);
7403 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
7406 * TODO: this bit should only be enabled when really needed, then
7407 * disabled when not needed anymore in order to save power.
7409 if (HAS_PCH_LPT_LP(dev_priv))
7410 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
7411 intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7412 PCH_LP_PARTITION_LEVEL_DISABLE);
7414 /* WADPOClockGatingDisable:hsw */
7415 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
7416 intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
7417 TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
7420 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
7422 if (HAS_PCH_LPT_LP(dev_priv)) {
7423 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
7425 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
7426 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
7430 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
7431 int general_prio_credits,
7432 int high_prio_credits)
7437 /* WaTempDisableDOPClkGating:bdw */
7438 misccpctl = intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL);
7439 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
7441 val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7442 val &= ~L3_PRIO_CREDITS_MASK;
7443 val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
7444 val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
7445 intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
7448 * Wait at least 100 clocks before re-enabling clock gating.
7449 * See the definition of L3SQCREG1 in BSpec.
7451 intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
7453 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
7456 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
7458 /* Wa_1409120013:icl,ehl */
7459 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7460 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7462 /*Wa_14010594013:icl, ehl */
7463 intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
7464 0, ICL_DELAY_PMRSP);
7467 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
7469 /* Wa_1409120013:tgl,rkl,adl-s,dg1,dg2 */
7470 if (IS_TIGERLAKE(dev_priv) || IS_ROCKETLAKE(dev_priv) ||
7471 IS_ALDERLAKE_S(dev_priv) || IS_DG1(dev_priv) || IS_DG2(dev_priv))
7472 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7473 DPFC_CHICKEN_COMP_DUMMY_PIXEL);
7475 /* Wa_1409825376:tgl (pre-prod)*/
7476 if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
7477 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7478 TGL_VRH_GATING_DIS);
7480 /* Wa_14013723622:tgl,rkl,dg1,adl-s */
7481 if (DISPLAY_VER(dev_priv) == 12)
7482 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
7483 CLKREQ_POLICY_MEM_UP_OVRD, 0);
7486 static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
7488 gen12lp_init_clock_gating(dev_priv);
7490 /* Wa_22011091694:adlp */
7491 intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
7493 /* Bspec/49189 Initialize Sequence */
7494 intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
7497 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
7499 gen12lp_init_clock_gating(dev_priv);
7501 /* Wa_1409836686:dg1[a0] */
7502 if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7503 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
7507 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
7509 /* Wa_22010146351:xehpsdv */
7510 if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
7511 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
7514 static void dg2_init_clock_gating(struct drm_i915_private *i915)
7516 /* Wa_22010954014:dg2_g10 */
7517 if (IS_DG2_G10(i915))
7518 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7522 * Wa_14010733611:dg2_g10
7523 * Wa_22010146351:dg2_g10
7525 if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
7526 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
7527 SGR_DIS | SGGI_DIS);
7530 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
7532 if (!HAS_PCH_CNP(dev_priv))
7535 /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
7536 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
7537 CNP_PWM_CGE_GATING_DISABLE);
7540 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
7542 cnp_init_clock_gating(dev_priv);
7543 gen9_init_clock_gating(dev_priv);
7545 /* WAC6entrylatency:cfl */
7546 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7547 FBC_LLC_FULLY_OPEN);
7550 * WaFbcTurnOffFbcWatermark:cfl
7551 * Display WA #0562: cfl
7553 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7557 * WaFbcNukeOnHostModify:cfl
7558 * Display WA #0873: cfl
7560 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7561 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7562 DPFC_NUKE_ON_ANY_MODIFICATION);
7565 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
7567 gen9_init_clock_gating(dev_priv);
7569 /* WAC6entrylatency:kbl */
7570 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7571 FBC_LLC_FULLY_OPEN);
7573 /* WaDisableSDEUnitClockGating:kbl */
7574 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7575 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7576 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7578 /* WaDisableGamClockGating:kbl */
7579 if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
7580 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7581 GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
7584 * WaFbcTurnOffFbcWatermark:kbl
7585 * Display WA #0562: kbl
7587 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7591 * WaFbcNukeOnHostModify:kbl
7592 * Display WA #0873: kbl
7594 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7595 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7596 DPFC_NUKE_ON_ANY_MODIFICATION);
7599 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
7601 gen9_init_clock_gating(dev_priv);
7603 /* WaDisableDopClockGating:skl */
7604 intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
7605 ~GEN7_DOP_CLOCK_GATE_ENABLE);
7607 /* WAC6entrylatency:skl */
7608 intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
7609 FBC_LLC_FULLY_OPEN);
7612 * WaFbcTurnOffFbcWatermark:skl
7613 * Display WA #0562: skl
7615 intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
7619 * WaFbcNukeOnHostModify:skl
7620 * Display WA #0873: skl
7622 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7623 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7624 DPFC_NUKE_ON_ANY_MODIFICATION);
7627 * WaFbcHighMemBwCorruptionAvoidance:skl
7628 * Display WA #0883: skl
7630 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
7631 intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
7632 DPFC_DISABLE_DUMMY0);
7635 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
7639 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7640 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7641 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7644 /* WaSwitchSolVfFArbitrationPriority:bdw */
7645 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7647 /* WaPsrDPAMaskVBlankInSRD:bdw */
7648 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
7649 intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
7651 for_each_pipe(dev_priv, pipe) {
7652 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
7653 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
7654 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
7655 BDW_DPRS_MASK_VBLANK_SRD);
7658 /* WaVSRefCountFullforceMissDisable:bdw */
7659 /* WaDSRefCountFullforceMissDisable:bdw */
7660 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7661 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7662 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7664 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
7665 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7667 /* WaDisableSDEUnitClockGating:bdw */
7668 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7669 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7671 /* WaProgramL3SqcReg1Default:bdw */
7672 gen8_set_l3sqc_credits(dev_priv, 30, 2);
7674 /* WaKVMNotificationOnConfigChange:bdw */
7675 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
7676 | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
7678 lpt_init_clock_gating(dev_priv);
7680 /* WaDisableDopClockGating:bdw
7682 * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
7685 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
7686 intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
7689 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
7691 /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
7692 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
7693 intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
7696 /* This is required by WaCatErrorRejectionIssue:hsw */
7697 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7698 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7699 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7701 /* WaSwitchSolVfFArbitrationPriority:hsw */
7702 intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
7704 lpt_init_clock_gating(dev_priv);
7707 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
7711 intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
7713 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
7714 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
7715 intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
7718 /* WaDisableBackToBackFlipFix:ivb */
7719 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7720 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7721 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7723 if (IS_IVB_GT1(dev_priv))
7724 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7725 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7727 /* must write both registers */
7728 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7729 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7730 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
7731 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7735 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7736 * This implements the WaDisableRCZUnitClockGating:ivb workaround.
7738 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7739 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7741 /* This is required by WaCatErrorRejectionIssue:ivb */
7742 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7743 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7744 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7746 g4x_disable_trickle_feed(dev_priv);
7748 snpcr = intel_uncore_read(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR);
7749 snpcr &= ~GEN6_MBC_SNPCR_MASK;
7750 snpcr |= GEN6_MBC_SNPCR_MED;
7751 intel_uncore_write(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, snpcr);
7753 if (!HAS_PCH_NOP(dev_priv))
7754 cpt_init_clock_gating(dev_priv);
7756 gen6_check_mch_setup(dev_priv);
7759 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
7761 /* WaDisableBackToBackFlipFix:vlv */
7762 intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
7763 CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
7764 CHICKEN3_DGMG_DONE_FIX_DISABLE);
7766 /* WaDisableDopClockGating:vlv */
7767 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
7768 _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
7770 /* This is required by WaCatErrorRejectionIssue:vlv */
7771 intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
7772 intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
7773 GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
7776 * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
7777 * This implements the WaDisableRCZUnitClockGating:vlv workaround.
7779 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
7780 GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
7782 /* WaDisableL3Bank2xClockGate:vlv
7783 * Disabling L3 clock gating- MMIO 940c[25] = 1
7784 * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
7785 intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
7786 intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
7789 * WaDisableVLVClockGating_VBIIssue:vlv
7790 * Disable clock gating on th GCFG unit to prevent a delay
7791 * in the reporting of vblank events.
7793 intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
7796 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
7798 /* WaVSRefCountFullforceMissDisable:chv */
7799 /* WaDSRefCountFullforceMissDisable:chv */
7800 intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
7801 intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
7802 ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
7804 /* WaDisableSemaphoreAndSyncFlipWait:chv */
7805 intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
7806 _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
7808 /* WaDisableCSUnitClockGating:chv */
7809 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
7810 GEN6_CSUNIT_CLOCK_GATE_DISABLE);
7812 /* WaDisableSDEUnitClockGating:chv */
7813 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
7814 GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
7817 * WaProgramL3SqcReg1Default:chv
7818 * See gfxspecs/Related Documents/Performance Guide/
7819 * LSQC Setting Recommendations.
7821 gen8_set_l3sqc_credits(dev_priv, 38, 2);
7824 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
7828 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
7829 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
7830 GS_UNIT_CLOCK_GATE_DISABLE |
7831 CL_UNIT_CLOCK_GATE_DISABLE);
7832 intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
7833 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
7834 OVRUNIT_CLOCK_GATE_DISABLE |
7835 OVCUNIT_CLOCK_GATE_DISABLE;
7836 if (IS_GM45(dev_priv))
7837 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
7838 intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D, dspclk_gate);
7840 g4x_disable_trickle_feed(dev_priv);
7843 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
7845 struct intel_uncore *uncore = &dev_priv->uncore;
7847 intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
7848 intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
7849 intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
7850 intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
7851 intel_uncore_write16(uncore, DEUC, 0);
7852 intel_uncore_write(uncore,
7854 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7857 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
7859 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
7860 I965_RCC_CLOCK_GATE_DISABLE |
7861 I965_RCPB_CLOCK_GATE_DISABLE |
7862 I965_ISC_CLOCK_GATE_DISABLE |
7863 I965_FBC_CLOCK_GATE_DISABLE);
7864 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
7865 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7866 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7869 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
7871 u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
7873 dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
7874 DSTATE_DOT_CLOCK_GATING;
7875 intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
7877 if (IS_PINEVIEW(dev_priv))
7878 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
7879 _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
7881 /* IIR "flip pending" means done if this bit is set */
7882 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
7883 _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
7885 /* interrupts should cause a wake up from C3 */
7886 intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
7888 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
7889 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
7891 intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
7892 _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
7895 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
7897 intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
7899 /* interrupts should cause a wake up from C3 */
7900 intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
7901 _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
7903 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7904 _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
7907 * Have FBC ignore 3D activity since we use software
7908 * render tracking, and otherwise a pure 3D workload
7909 * (even if it just renders a single frame and then does
7910 * abosultely nothing) would not allow FBC to recompress
7911 * until a 2D blit occurs.
7913 intel_uncore_write(&dev_priv->uncore, SCPD0,
7914 _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
7917 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
7919 intel_uncore_write(&dev_priv->uncore, MEM_MODE,
7920 _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
7921 _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
7924 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
7926 dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
7929 void intel_suspend_hw(struct drm_i915_private *dev_priv)
7931 if (HAS_PCH_LPT(dev_priv))
7932 lpt_suspend_hw(dev_priv);
7935 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
7937 drm_dbg_kms(&dev_priv->drm,
7938 "No clock gating settings or workarounds applied.\n");
7941 #define CG_FUNCS(platform) \
7942 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
7943 .init_clock_gating = platform##_init_clock_gating, \
7974 * intel_init_clock_gating_hooks - setup the clock gating hooks
7975 * @dev_priv: device private
7977 * Setup the hooks that configure which clocks of a given platform can be
7978 * gated and also apply various GT and display specific workarounds for these
7979 * platforms. Note that some GT specific workarounds are applied separately
7980 * when GPU contexts or batchbuffers start their execution.
7982 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
7984 if (IS_DG2(dev_priv))
7985 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
7986 else if (IS_XEHPSDV(dev_priv))
7987 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
7988 else if (IS_ALDERLAKE_P(dev_priv))
7989 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
7990 else if (IS_DG1(dev_priv))
7991 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
7992 else if (GRAPHICS_VER(dev_priv) == 12)
7993 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
7994 else if (GRAPHICS_VER(dev_priv) == 11)
7995 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
7996 else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
7997 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
7998 else if (IS_SKYLAKE(dev_priv))
7999 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
8000 else if (IS_KABYLAKE(dev_priv))
8001 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
8002 else if (IS_BROXTON(dev_priv))
8003 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
8004 else if (IS_GEMINILAKE(dev_priv))
8005 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
8006 else if (IS_BROADWELL(dev_priv))
8007 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
8008 else if (IS_CHERRYVIEW(dev_priv))
8009 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
8010 else if (IS_HASWELL(dev_priv))
8011 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
8012 else if (IS_IVYBRIDGE(dev_priv))
8013 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
8014 else if (IS_VALLEYVIEW(dev_priv))
8015 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
8016 else if (GRAPHICS_VER(dev_priv) == 6)
8017 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
8018 else if (GRAPHICS_VER(dev_priv) == 5)
8019 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
8020 else if (IS_G4X(dev_priv))
8021 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
8022 else if (IS_I965GM(dev_priv))
8023 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
8024 else if (IS_I965G(dev_priv))
8025 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
8026 else if (GRAPHICS_VER(dev_priv) == 3)
8027 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
8028 else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
8029 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
8030 else if (GRAPHICS_VER(dev_priv) == 2)
8031 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
8033 MISSING_CASE(INTEL_DEVID(dev_priv));
8034 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
8038 static const struct drm_i915_wm_disp_funcs skl_wm_funcs = {
8039 .compute_global_watermarks = skl_compute_wm,
8042 static const struct drm_i915_wm_disp_funcs ilk_wm_funcs = {
8043 .compute_pipe_wm = ilk_compute_pipe_wm,
8044 .compute_intermediate_wm = ilk_compute_intermediate_wm,
8045 .initial_watermarks = ilk_initial_watermarks,
8046 .optimize_watermarks = ilk_optimize_watermarks,
8049 static const struct drm_i915_wm_disp_funcs vlv_wm_funcs = {
8050 .compute_pipe_wm = vlv_compute_pipe_wm,
8051 .compute_intermediate_wm = vlv_compute_intermediate_wm,
8052 .initial_watermarks = vlv_initial_watermarks,
8053 .optimize_watermarks = vlv_optimize_watermarks,
8054 .atomic_update_watermarks = vlv_atomic_update_fifo,
8057 static const struct drm_i915_wm_disp_funcs g4x_wm_funcs = {
8058 .compute_pipe_wm = g4x_compute_pipe_wm,
8059 .compute_intermediate_wm = g4x_compute_intermediate_wm,
8060 .initial_watermarks = g4x_initial_watermarks,
8061 .optimize_watermarks = g4x_optimize_watermarks,
8064 static const struct drm_i915_wm_disp_funcs pnv_wm_funcs = {
8065 .update_wm = pnv_update_wm,
8068 static const struct drm_i915_wm_disp_funcs i965_wm_funcs = {
8069 .update_wm = i965_update_wm,
8072 static const struct drm_i915_wm_disp_funcs i9xx_wm_funcs = {
8073 .update_wm = i9xx_update_wm,
8076 static const struct drm_i915_wm_disp_funcs i845_wm_funcs = {
8077 .update_wm = i845_update_wm,
8080 static const struct drm_i915_wm_disp_funcs nop_funcs = {
8083 /* Set up chip specific power management-related functions */
8084 void intel_init_pm(struct drm_i915_private *dev_priv)
8087 if (IS_PINEVIEW(dev_priv))
8088 pnv_get_mem_freq(dev_priv);
8089 else if (GRAPHICS_VER(dev_priv) == 5)
8090 ilk_get_mem_freq(dev_priv);
8092 intel_sagv_init(dev_priv);
8094 /* For FIFO watermark updates */
8095 if (DISPLAY_VER(dev_priv) >= 9) {
8096 skl_setup_wm_latency(dev_priv);
8097 dev_priv->wm_disp = &skl_wm_funcs;
8098 } else if (HAS_PCH_SPLIT(dev_priv)) {
8099 ilk_setup_wm_latency(dev_priv);
8101 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->wm.pri_latency[1] &&
8102 dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
8103 (DISPLAY_VER(dev_priv) != 5 && dev_priv->wm.pri_latency[0] &&
8104 dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
8105 dev_priv->wm_disp = &ilk_wm_funcs;
8107 drm_dbg_kms(&dev_priv->drm,
8108 "Failed to read display plane latency. "
8110 dev_priv->wm_disp = &nop_funcs;
8112 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
8113 vlv_setup_wm_latency(dev_priv);
8114 dev_priv->wm_disp = &vlv_wm_funcs;
8115 } else if (IS_G4X(dev_priv)) {
8116 g4x_setup_wm_latency(dev_priv);
8117 dev_priv->wm_disp = &g4x_wm_funcs;
8118 } else if (IS_PINEVIEW(dev_priv)) {
8119 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
8122 dev_priv->mem_freq)) {
8123 drm_info(&dev_priv->drm,
8124 "failed to find known CxSR latency "
8125 "(found ddr%s fsb freq %d, mem freq %d), "
8127 (dev_priv->is_ddr3 == 1) ? "3" : "2",
8128 dev_priv->fsb_freq, dev_priv->mem_freq);
8129 /* Disable CxSR and never update its watermark again */
8130 intel_set_memory_cxsr(dev_priv, false);
8131 dev_priv->wm_disp = &nop_funcs;
8133 dev_priv->wm_disp = &pnv_wm_funcs;
8134 } else if (DISPLAY_VER(dev_priv) == 4) {
8135 dev_priv->wm_disp = &i965_wm_funcs;
8136 } else if (DISPLAY_VER(dev_priv) == 3) {
8137 dev_priv->wm_disp = &i9xx_wm_funcs;
8138 } else if (DISPLAY_VER(dev_priv) == 2) {
8139 if (INTEL_NUM_PIPES(dev_priv) == 1)
8140 dev_priv->wm_disp = &i845_wm_funcs;
8142 dev_priv->wm_disp = &i9xx_wm_funcs;
8144 drm_err(&dev_priv->drm,
8145 "unexpected fall-through in %s\n", __func__);
8146 dev_priv->wm_disp = &nop_funcs;
8150 void intel_pm_setup(struct drm_i915_private *dev_priv)
8152 dev_priv->runtime_pm.suspended = false;
8153 atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
8156 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
8158 struct intel_dbuf_state *dbuf_state;
8160 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
8164 return &dbuf_state->base;
8167 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
8168 struct intel_global_state *state)
8173 static const struct intel_global_state_funcs intel_dbuf_funcs = {
8174 .atomic_duplicate_state = intel_dbuf_duplicate_state,
8175 .atomic_destroy_state = intel_dbuf_destroy_state,
8178 struct intel_dbuf_state *
8179 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
8181 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8182 struct intel_global_state *dbuf_state;
8184 dbuf_state = intel_atomic_get_global_obj_state(state, &dev_priv->dbuf.obj);
8185 if (IS_ERR(dbuf_state))
8186 return ERR_CAST(dbuf_state);
8188 return to_intel_dbuf_state(dbuf_state);
8191 int intel_dbuf_init(struct drm_i915_private *dev_priv)
8193 struct intel_dbuf_state *dbuf_state;
8195 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
8199 intel_atomic_global_obj_init(dev_priv, &dev_priv->dbuf.obj,
8200 &dbuf_state->base, &intel_dbuf_funcs);
8206 * Configure MBUS_CTL and all DBUF_CTL_S of each slice to join_mbus state before
8207 * update the request state of all DBUS slices.
8209 static void update_mbus_pre_enable(struct intel_atomic_state *state)
8211 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8212 u32 mbus_ctl, dbuf_min_tracker_val;
8213 enum dbuf_slice slice;
8214 const struct intel_dbuf_state *dbuf_state =
8215 intel_atomic_get_new_dbuf_state(state);
8217 if (!HAS_MBUS_JOINING(dev_priv))
8221 * TODO: Implement vblank synchronized MBUS joining changes.
8222 * Must be properly coordinated with dbuf reprogramming.
8224 if (dbuf_state->joined_mbus) {
8225 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN |
8226 MBUS_JOIN_PIPE_SELECT_NONE;
8227 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(3);
8229 mbus_ctl = MBUS_HASHING_MODE_2x2 |
8230 MBUS_JOIN_PIPE_SELECT_NONE;
8231 dbuf_min_tracker_val = DBUF_MIN_TRACKER_STATE_SERVICE(1);
8234 intel_de_rmw(dev_priv, MBUS_CTL,
8235 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
8236 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
8238 for_each_dbuf_slice(dev_priv, slice)
8239 intel_de_rmw(dev_priv, DBUF_CTL_S(slice),
8240 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
8241 dbuf_min_tracker_val);
8244 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
8246 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8247 const struct intel_dbuf_state *new_dbuf_state =
8248 intel_atomic_get_new_dbuf_state(state);
8249 const struct intel_dbuf_state *old_dbuf_state =
8250 intel_atomic_get_old_dbuf_state(state);
8252 if (!new_dbuf_state ||
8253 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8254 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8257 WARN_ON(!new_dbuf_state->base.changed);
8259 update_mbus_pre_enable(state);
8260 gen9_dbuf_slices_update(dev_priv,
8261 old_dbuf_state->enabled_slices |
8262 new_dbuf_state->enabled_slices);
8265 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
8267 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
8268 const struct intel_dbuf_state *new_dbuf_state =
8269 intel_atomic_get_new_dbuf_state(state);
8270 const struct intel_dbuf_state *old_dbuf_state =
8271 intel_atomic_get_old_dbuf_state(state);
8273 if (!new_dbuf_state ||
8274 ((new_dbuf_state->enabled_slices == old_dbuf_state->enabled_slices)
8275 && (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus)))
8278 WARN_ON(!new_dbuf_state->base.changed);
8280 gen9_dbuf_slices_update(dev_priv,
8281 new_dbuf_state->enabled_slices);
8284 void intel_mbus_dbox_update(struct intel_atomic_state *state)
8286 struct drm_i915_private *i915 = to_i915(state->base.dev);
8287 const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
8288 const struct intel_crtc_state *new_crtc_state;
8289 const struct intel_crtc *crtc;
8293 if (DISPLAY_VER(i915) < 11)
8296 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
8297 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
8298 if (!new_dbuf_state ||
8299 (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
8300 new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
8303 if (DISPLAY_VER(i915) >= 12) {
8304 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
8305 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
8306 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
8309 /* Wa_22010947358:adl-p */
8310 if (IS_ALDERLAKE_P(i915))
8311 val |= new_dbuf_state->joined_mbus ? MBUS_DBOX_A_CREDIT(6) :
8312 MBUS_DBOX_A_CREDIT(4);
8314 val |= MBUS_DBOX_A_CREDIT(2);
8316 if (IS_ALDERLAKE_P(i915)) {
8317 val |= MBUS_DBOX_BW_CREDIT(2);
8318 val |= MBUS_DBOX_B_CREDIT(8);
8319 } else if (DISPLAY_VER(i915) >= 12) {
8320 val |= MBUS_DBOX_BW_CREDIT(2);
8321 val |= MBUS_DBOX_B_CREDIT(12);
8323 val |= MBUS_DBOX_BW_CREDIT(1);
8324 val |= MBUS_DBOX_B_CREDIT(8);
8327 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
8328 if (!new_crtc_state->hw.active ||
8329 !intel_crtc_needs_modeset(new_crtc_state))
8332 intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe), val);