drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include "display/intel_de.h"
  29 #include "display/intel_display_trace.h"
  30 #include "display/skl_watermark.h"
  31
  32 #include "gt/intel_engine_regs.h"
  33 #include "gt/intel_gt_regs.h"
  34
  35 #include "i915_drv.h"
  36 #include "intel_mchbar_regs.h"
  37 #include "intel_pm.h"
  38 #include "vlv_sideband.h"
  39
  40 struct drm_i915_clock_gating_funcs {
  41         void (*init_clock_gating)(struct drm_i915_private *i915);
  42 };
  43
  44 /* used in computing the new watermarks state */
  45 struct intel_wm_config {
  46         unsigned int num_pipes_active;
  47         bool sprites_enabled;
  48         bool sprites_scaled;
  49 };
  50
  51 static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
  52 {
  53         if (HAS_LLC(dev_priv)) {
  54                 /*
  55                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  56                  * Display WA #0390: skl,kbl
  57                  *
  58                  * Must match Sampler, Pixel Back End, and Media. See
  59                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  60                  */
  61                 intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
  62                            intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
  63                            SKL_DE_COMPRESSED_HASH_MODE);
  64         }
  65
  66         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  67         intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
  68                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
  69
  70         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  71         intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
  72                    intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
  73
  74         /*
  75          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
  76          * Display WA #0859: skl,bxt,kbl,glk,cfl
  77          */
  78         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
  79                    DISP_FBC_MEMORY_WAKE);
  80 }
  81
  82 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
  83 {
  84         gen9_init_clock_gating(dev_priv);
  85
  86         /* WaDisableSDEUnitClockGating:bxt */
  87         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
  88                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  89
  90         /*
  91          * FIXME:
  92          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
  93          */
  94         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
  95                    GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
  96
  97         /*
  98          * Wa: Backlight PWM may stop in the asserted state, causing backlight
  99          * to stay fully on.
 100          */
 101         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
 102                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 103
 104         /*
 105          * Lower the display internal timeout.
 106          * This is needed to avoid any hard hangs when DSI port PLL
 107          * is off and a MMIO access is attempted by any privilege
 108          * application, using batch buffers or any other means.
 109          */
 110         intel_uncore_write(&dev_priv->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 111
 112         /*
 113          * WaFbcTurnOffFbcWatermark:bxt
 114          * Display WA #0562: bxt
 115          */
 116         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
 117                    DISP_FBC_WM_DIS);
 118
 119         /*
 120          * WaFbcHighMemBwCorruptionAvoidance:bxt
 121          * Display WA #0883: bxt
 122          */
 123         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
 124                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
 125                            DPFC_DISABLE_DUMMY0);
 126 }
 127
 128 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 129 {
 130         gen9_init_clock_gating(dev_priv);
 131
 132         /*
 133          * WaDisablePWMClockGating:glk
 134          * Backlight PWM may stop in the asserted state, causing backlight
 135          * to stay fully on.
 136          */
 137         intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_0, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_0) |
 138                    PWM1_GATING_DIS | PWM2_GATING_DIS);
 139 }
 140
 141 static void pnv_get_mem_freq(struct drm_i915_private *dev_priv)
 142 {
 143         u32 tmp;
 144
 145         tmp = intel_uncore_read(&dev_priv->uncore, CLKCFG);
 146
 147         switch (tmp & CLKCFG_FSB_MASK) {
 148         case CLKCFG_FSB_533:
 149                 dev_priv->fsb_freq = 533; /* 133*4 */
 150                 break;
 151         case CLKCFG_FSB_800:
 152                 dev_priv->fsb_freq = 800; /* 200*4 */
 153                 break;
 154         case CLKCFG_FSB_667:
 155                 dev_priv->fsb_freq =  667; /* 167*4 */
 156                 break;
 157         case CLKCFG_FSB_400:
 158                 dev_priv->fsb_freq = 400; /* 100*4 */
 159                 break;
 160         }
 161
 162         switch (tmp & CLKCFG_MEM_MASK) {
 163         case CLKCFG_MEM_533:
 164                 dev_priv->mem_freq = 533;
 165                 break;
 166         case CLKCFG_MEM_667:
 167                 dev_priv->mem_freq = 667;
 168                 break;
 169         case CLKCFG_MEM_800:
 170                 dev_priv->mem_freq = 800;
 171                 break;
 172         }
 173
 174         /* detect pineview DDR3 setting */
 175         tmp = intel_uncore_read(&dev_priv->uncore, CSHRDDR3CTL);
 176         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 177 }
 178
 179 static void ilk_get_mem_freq(struct drm_i915_private *dev_priv)
 180 {
 181         u16 ddrpll, csipll;
 182
 183         ddrpll = intel_uncore_read16(&dev_priv->uncore, DDRMPLL1);
 184         csipll = intel_uncore_read16(&dev_priv->uncore, CSIPLL0);
 185
 186         switch (ddrpll & 0xff) {
 187         case 0xc:
 188                 dev_priv->mem_freq = 800;
 189                 break;
 190         case 0x10:
 191                 dev_priv->mem_freq = 1066;
 192                 break;
 193         case 0x14:
 194                 dev_priv->mem_freq = 1333;
 195                 break;
 196         case 0x18:
 197                 dev_priv->mem_freq = 1600;
 198                 break;
 199         default:
 200                 drm_dbg(&dev_priv->drm, "unknown memory frequency 0x%02x\n",
 201                         ddrpll & 0xff);
 202                 dev_priv->mem_freq = 0;
 203                 break;
 204         }
 205
 206         switch (csipll & 0x3ff) {
 207         case 0x00c:
 208                 dev_priv->fsb_freq = 3200;
 209                 break;
 210         case 0x00e:
 211                 dev_priv->fsb_freq = 3733;
 212                 break;
 213         case 0x010:
 214                 dev_priv->fsb_freq = 4266;
 215                 break;
 216         case 0x012:
 217                 dev_priv->fsb_freq = 4800;
 218                 break;
 219         case 0x014:
 220                 dev_priv->fsb_freq = 5333;
 221                 break;
 222         case 0x016:
 223                 dev_priv->fsb_freq = 5866;
 224                 break;
 225         case 0x018:
 226                 dev_priv->fsb_freq = 6400;
 227                 break;
 228         default:
 229                 drm_dbg(&dev_priv->drm, "unknown fsb frequency 0x%04x\n",
 230                         csipll & 0x3ff);
 231                 dev_priv->fsb_freq = 0;
 232                 break;
 233         }
 234 }
 235
 236 static const struct cxsr_latency cxsr_latency_table[] = {
 237         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 238         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 239         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 240         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 241         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 242
 243         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 244         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 245         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 246         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 247         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 248
 249         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 250         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 251         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 252         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 253         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 254
 255         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 256         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 257         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 258         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 259         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 260
 261         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 262         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 263         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 264         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 265         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 266
 267         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 268         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 269         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 270         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 271         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 272 };
 273
 274 static const struct cxsr_latency *intel_get_cxsr_latency(bool is_desktop,
 275                                                          bool is_ddr3,
 276                                                          int fsb,
 277                                                          int mem)
 278 {
 279         const struct cxsr_latency *latency;
 280         int i;
 281
 282         if (fsb == 0 || mem == 0)
 283                 return NULL;
 284
 285         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 286                 latency = &cxsr_latency_table[i];
 287                 if (is_desktop == latency->is_desktop &&
 288                     is_ddr3 == latency->is_ddr3 &&
 289                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 290                         return latency;
 291         }
 292
 293         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 294
 295         return NULL;
 296 }
 297
 298 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 299 {
 300         u32 val;
 301
 302         vlv_punit_get(dev_priv);
 303
 304         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 305         if (enable)
 306                 val &= ~FORCE_DDR_HIGH_FREQ;
 307         else
 308                 val |= FORCE_DDR_HIGH_FREQ;
 309         val &= ~FORCE_DDR_LOW_FREQ;
 310         val |= FORCE_DDR_FREQ_REQ_ACK;
 311         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 312
 313         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 314                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 315                 drm_err(&dev_priv->drm,
 316                         "timed out waiting for Punit DDR DVFS request\n");
 317
 318         vlv_punit_put(dev_priv);
 319 }
 320
 321 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 322 {
 323         u32 val;
 324
 325         vlv_punit_get(dev_priv);
 326
 327         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
 328         if (enable)
 329                 val |= DSP_MAXFIFO_PM5_ENABLE;
 330         else
 331                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 332         vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
 333
 334         vlv_punit_put(dev_priv);
 335 }
 336
 337 #define FW_WM(value, plane) \
 338         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 339
 340 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 341 {
 342         bool was_enabled;
 343         u32 val;
 344
 345         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
 346                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
 347                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 348                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
 349         } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
 350                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 351                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 352                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 353         } else if (IS_PINEVIEW(dev_priv)) {
 354                 val = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 355                 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
 356                 if (enable)
 357                         val |= PINEVIEW_SELF_REFRESH_EN;
 358                 else
 359                         val &= ~PINEVIEW_SELF_REFRESH_EN;
 360                 intel_uncore_write(&dev_priv->uncore, DSPFW3, val);
 361                 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3);
 362         } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
 363                 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
 364                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 365                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 366                 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
 367                 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
 368         } else if (IS_I915GM(dev_priv)) {
 369                 /*
 370                  * FIXME can't find a bit like this for 915G, and
 371                  * and yet it does have the related watermark in
 372                  * FW_BLC_SELF. What's going on?
 373                  */
 374                 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
 375                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 376                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 377                 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
 378                 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
 379         } else {
 380                 return false;
 381         }
 382
 383         trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
 384
 385         drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
 386                     str_enabled_disabled(enable),
 387                     str_enabled_disabled(was_enabled));
 388
 389         return was_enabled;
 390 }
 391
 392 /**
 393  * intel_set_memory_cxsr - Configure CxSR state
 394  * @dev_priv: i915 device
 395  * @enable: Allow vs. disallow CxSR
 396  *
 397  * Allow or disallow the system to enter a special CxSR
 398  * (C-state self refresh) state. What typically happens in CxSR mode
 399  * is that several display FIFOs may get combined into a single larger
 400  * FIFO for a particular plane (so called max FIFO mode) to allow the
 401  * system to defer memory fetches longer, and the memory will enter
 402  * self refresh.
 403  *
 404  * Note that enabling CxSR does not guarantee that the system enter
 405  * this special mode, nor does it guarantee that the system stays
 406  * in that mode once entered. So this just allows/disallows the system
 407  * to autonomously utilize the CxSR mode. Other factors such as core
 408  * C-states will affect when/if the system actually enters/exits the
 409  * CxSR mode.
 410  *
 411  * Note that on VLV/CHV this actually only controls the max FIFO mode,
 412  * and the system is free to enter/exit memory self refresh at any time
 413  * even when the use of CxSR has been disallowed.
 414  *
 415  * While the system is actually in the CxSR/max FIFO mode, some plane
 416  * control registers will not get latched on vblank. Thus in order to
 417  * guarantee the system will respond to changes in the plane registers
 418  * we must always disallow CxSR prior to making changes to those registers.
 419  * Unfortunately the system will re-evaluate the CxSR conditions at
 420  * frame start which happens after vblank start (which is when the plane
 421  * registers would get latched), so we can't proceed with the plane update
 422  * during the same frame where we disallowed CxSR.
 423  *
 424  * Certain platforms also have a deeper HPLL SR mode. Fortunately the
 425  * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
 426  * the hardware w.r.t. HPLL SR when writing to plane registers.
 427  * Disallowing just CxSR is sufficient.
 428  */
 429 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 430 {
 431         bool ret;
 432
 433         mutex_lock(&dev_priv->display.wm.wm_mutex);
 434         ret = _intel_set_memory_cxsr(dev_priv, enable);
 435         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
 436                 dev_priv->display.wm.vlv.cxsr = enable;
 437         else if (IS_G4X(dev_priv))
 438                 dev_priv->display.wm.g4x.cxsr = enable;
 439         mutex_unlock(&dev_priv->display.wm.wm_mutex);
 440
 441         return ret;
 442 }
 443
 444 /*
 445  * Latency for FIFO fetches is dependent on several factors:
 446  *   - memory configuration (speed, channels)
 447  *   - chipset
 448  *   - current MCH state
 449  * It can be fairly high in some situations, so here we assume a fairly
 450  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 451  * set this value too high, the FIFO will fetch frequently to stay full)
 452  * and power consumption (set it too low to save power and we might see
 453  * FIFO underruns and display "flicker").
 454  *
 455  * A value of 5us seems to be a good balance; safe for very low end
 456  * platforms but not overly aggressive on lower latency configs.
 457  */
 458 static const int pessimal_latency_ns = 5000;
 459
 460 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 461         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 462
 463 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
 464 {
 465         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 466         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 467         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
 468         enum pipe pipe = crtc->pipe;
 469         int sprite0_start, sprite1_start;
 470         u32 dsparb, dsparb2, dsparb3;
 471
 472         switch (pipe) {
 473         case PIPE_A:
 474                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 475                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 476                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 477                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 478                 break;
 479         case PIPE_B:
 480                 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 481                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 482                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 483                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 484                 break;
 485         case PIPE_C:
 486                 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
 487                 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
 488                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 489                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 490                 break;
 491         default:
 492                 MISSING_CASE(pipe);
 493                 return;
 494         }
 495
 496         fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
 497         fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
 498         fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
 499         fifo_state->plane[PLANE_CURSOR] = 63;
 500 }
 501
 502 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
 503                               enum i9xx_plane_id i9xx_plane)
 504 {
 505         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 506         int size;
 507
 508         size = dsparb & 0x7f;
 509         if (i9xx_plane == PLANE_B)
 510                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 511
 512         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 513                     dsparb, plane_name(i9xx_plane), size);
 514
 515         return size;
 516 }
 517
 518 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
 519                               enum i9xx_plane_id i9xx_plane)
 520 {
 521         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 522         int size;
 523
 524         size = dsparb & 0x1ff;
 525         if (i9xx_plane == PLANE_B)
 526                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 527         size >>= 1; /* Convert to cachelines */
 528
 529         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 530                     dsparb, plane_name(i9xx_plane), size);
 531
 532         return size;
 533 }
 534
 535 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
 536                               enum i9xx_plane_id i9xx_plane)
 537 {
 538         u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB);
 539         int size;
 540
 541         size = dsparb & 0x7f;
 542         size >>= 2; /* Convert to cachelines */
 543
 544         drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
 545                     dsparb, plane_name(i9xx_plane), size);
 546
 547         return size;
 548 }
 549
 550 /* Pineview has different values for various configs */
 551 static const struct intel_watermark_params pnv_display_wm = {
 552         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 553         .max_wm = PINEVIEW_MAX_WM,
 554         .default_wm = PINEVIEW_DFT_WM,
 555         .guard_size = PINEVIEW_GUARD_WM,
 556         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 557 };
 558
 559 static const struct intel_watermark_params pnv_display_hplloff_wm = {
 560         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 561         .max_wm = PINEVIEW_MAX_WM,
 562         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 563         .guard_size = PINEVIEW_GUARD_WM,
 564         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 565 };
 566
 567 static const struct intel_watermark_params pnv_cursor_wm = {
 568         .fifo_size = PINEVIEW_CURSOR_FIFO,
 569         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 570         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 571         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 572         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 573 };
 574
 575 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
 576         .fifo_size = PINEVIEW_CURSOR_FIFO,
 577         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 578         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 579         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 580         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 581 };
 582
 583 static const struct intel_watermark_params i965_cursor_wm_info = {
 584         .fifo_size = I965_CURSOR_FIFO,
 585         .max_wm = I965_CURSOR_MAX_WM,
 586         .default_wm = I965_CURSOR_DFT_WM,
 587         .guard_size = 2,
 588         .cacheline_size = I915_FIFO_LINE_SIZE,
 589 };
 590
 591 static const struct intel_watermark_params i945_wm_info = {
 592         .fifo_size = I945_FIFO_SIZE,
 593         .max_wm = I915_MAX_WM,
 594         .default_wm = 1,
 595         .guard_size = 2,
 596         .cacheline_size = I915_FIFO_LINE_SIZE,
 597 };
 598
 599 static const struct intel_watermark_params i915_wm_info = {
 600         .fifo_size = I915_FIFO_SIZE,
 601         .max_wm = I915_MAX_WM,
 602         .default_wm = 1,
 603         .guard_size = 2,
 604         .cacheline_size = I915_FIFO_LINE_SIZE,
 605 };
 606
 607 static const struct intel_watermark_params i830_a_wm_info = {
 608         .fifo_size = I855GM_FIFO_SIZE,
 609         .max_wm = I915_MAX_WM,
 610         .default_wm = 1,
 611         .guard_size = 2,
 612         .cacheline_size = I830_FIFO_LINE_SIZE,
 613 };
 614
 615 static const struct intel_watermark_params i830_bc_wm_info = {
 616         .fifo_size = I855GM_FIFO_SIZE,
 617         .max_wm = I915_MAX_WM/2,
 618         .default_wm = 1,
 619         .guard_size = 2,
 620         .cacheline_size = I830_FIFO_LINE_SIZE,
 621 };
 622
 623 static const struct intel_watermark_params i845_wm_info = {
 624         .fifo_size = I830_FIFO_SIZE,
 625         .max_wm = I915_MAX_WM,
 626         .default_wm = 1,
 627         .guard_size = 2,
 628         .cacheline_size = I830_FIFO_LINE_SIZE,
 629 };
 630
 631 /**
 632  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
 633  * @pixel_rate: Pipe pixel rate in kHz
 634  * @cpp: Plane bytes per pixel
 635  * @latency: Memory wakeup latency in 0.1us units
 636  *
 637  * Compute the watermark using the method 1 or "small buffer"
 638  * formula. The caller may additonally add extra cachelines
 639  * to account for TLB misses and clock crossings.
 640  *
 641  * This method is concerned with the short term drain rate
 642  * of the FIFO, ie. it does not account for blanking periods
 643  * which would effectively reduce the average drain rate across
 644  * a longer period. The name "small" refers to the fact the
 645  * FIFO is relatively small compared to the amount of data
 646  * fetched.
 647  *
 648  * The FIFO level vs. time graph might look something like:
 649  *
 650  *   |\   |\
 651  *   | \  | \
 652  * __---__---__ (- plane active, _ blanking)
 653  * -> time
 654  *
 655  * or perhaps like this:
 656  *
 657  *   |\|\  |\|\
 658  * __----__----__ (- plane active, _ blanking)
 659  * -> time
 660  *
 661  * Returns:
 662  * The watermark in bytes
 663  */
 664 static unsigned int intel_wm_method1(unsigned int pixel_rate,
 665                                      unsigned int cpp,
 666                                      unsigned int latency)
 667 {
 668         u64 ret;
 669
 670         ret = mul_u32_u32(pixel_rate, cpp * latency);
 671         ret = DIV_ROUND_UP_ULL(ret, 10000);
 672
 673         return ret;
 674 }
 675
 676 /**
 677  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
 678  * @pixel_rate: Pipe pixel rate in kHz
 679  * @htotal: Pipe horizontal total
 680  * @width: Plane width in pixels
 681  * @cpp: Plane bytes per pixel
 682  * @latency: Memory wakeup latency in 0.1us units
 683  *
 684  * Compute the watermark using the method 2 or "large buffer"
 685  * formula. The caller may additonally add extra cachelines
 686  * to account for TLB misses and clock crossings.
 687  *
 688  * This method is concerned with the long term drain rate
 689  * of the FIFO, ie. it does account for blanking periods
 690  * which effectively reduce the average drain rate across
 691  * a longer period. The name "large" refers to the fact the
 692  * FIFO is relatively large compared to the amount of data
 693  * fetched.
 694  *
 695  * The FIFO level vs. time graph might look something like:
 696  *
 697  *    |\___       |\___
 698  *    |    \___   |    \___
 699  *    |        \  |        \
 700  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
 701  * -> time
 702  *
 703  * Returns:
 704  * The watermark in bytes
 705  */
 706 static unsigned int intel_wm_method2(unsigned int pixel_rate,
 707                                      unsigned int htotal,
 708                                      unsigned int width,
 709                                      unsigned int cpp,
 710                                      unsigned int latency)
 711 {
 712         unsigned int ret;
 713
 714         /*
 715          * FIXME remove once all users are computing
 716          * watermarks in the correct place.
 717          */
 718         if (WARN_ON_ONCE(htotal == 0))
 719                 htotal = 1;
 720
 721         ret = (latency * pixel_rate) / (htotal * 10000);
 722         ret = (ret + 1) * width * cpp;
 723
 724         return ret;
 725 }
 726
 727 /**
 728  * intel_calculate_wm - calculate watermark level
 729  * @pixel_rate: pixel clock
 730  * @wm: chip FIFO params
 731  * @fifo_size: size of the FIFO buffer
 732  * @cpp: bytes per pixel
 733  * @latency_ns: memory latency for the platform
 734  *
 735  * Calculate the watermark level (the level at which the display plane will
 736  * start fetching from memory again).  Each chip has a different display
 737  * FIFO size and allocation, so the caller needs to figure that out and pass
 738  * in the correct intel_watermark_params structure.
 739  *
 740  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 741  * on the pixel size.  When it reaches the watermark level, it'll start
 742  * fetching FIFO line sized based chunks from memory until the FIFO fills
 743  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 744  * will occur, and a display engine hang could result.
 745  */
 746 static unsigned int intel_calculate_wm(int pixel_rate,
 747                                        const struct intel_watermark_params *wm,
 748                                        int fifo_size, int cpp,
 749                                        unsigned int latency_ns)
 750 {
 751         int entries, wm_size;
 752
 753         /*
 754          * Note: we need to make sure we don't overflow for various clock &
 755          * latency values.
 756          * clocks go from a few thousand to several hundred thousand.
 757          * latency is usually a few thousand
 758          */
 759         entries = intel_wm_method1(pixel_rate, cpp,
 760                                    latency_ns / 100);
 761         entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
 762                 wm->guard_size;
 763         DRM_DEBUG_KMS("FIFO entries required for mode: %d\n", entries);
 764
 765         wm_size = fifo_size - entries;
 766         DRM_DEBUG_KMS("FIFO watermark level: %d\n", wm_size);
 767
 768         /* Don't promote wm_size to unsigned... */
 769         if (wm_size > wm->max_wm)
 770                 wm_size = wm->max_wm;
 771         if (wm_size <= 0)
 772                 wm_size = wm->default_wm;
 773
 774         /*
 775          * Bspec seems to indicate that the value shouldn't be lower than
 776          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 777          * Lets go for 8 which is the burst size since certain platforms
 778          * already use a hardcoded 8 (which is what the spec says should be
 779          * done).
 780          */
 781         if (wm_size <= 8)
 782                 wm_size = 8;
 783
 784         return wm_size;
 785 }
 786
 787 static bool is_disabling(int old, int new, int threshold)
 788 {
 789         return old >= threshold && new < threshold;
 790 }
 791
 792 static bool is_enabling(int old, int new, int threshold)
 793 {
 794         return old < threshold && new >= threshold;
 795 }
 796
 797 static int intel_wm_num_levels(struct drm_i915_private *dev_priv)
 798 {
 799         return dev_priv->display.wm.max_level + 1;
 800 }
 801
 802 bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
 803                             const struct intel_plane_state *plane_state)
 804 {
 805         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 806
 807         /* FIXME check the 'enable' instead */
 808         if (!crtc_state->hw.active)
 809                 return false;
 810
 811         /*
 812          * Treat cursor with fb as always visible since cursor updates
 813          * can happen faster than the vrefresh rate, and the current
 814          * watermark code doesn't handle that correctly. Cursor updates
 815          * which set/clear the fb or change the cursor size are going
 816          * to get throttled by intel_legacy_cursor_update() to work
 817          * around this problem with the watermark code.
 818          */
 819         if (plane->id == PLANE_CURSOR)
 820                 return plane_state->hw.fb != NULL;
 821         else
 822                 return plane_state->uapi.visible;
 823 }
 824
 825 static bool intel_crtc_active(struct intel_crtc *crtc)
 826 {
 827         /* Be paranoid as we can arrive here with only partial
 828          * state retrieved from the hardware during setup.
 829          *
 830          * We can ditch the adjusted_mode.crtc_clock check as soon
 831          * as Haswell has gained clock readout/fastboot support.
 832          *
 833          * We can ditch the crtc->primary->state->fb check as soon as we can
 834          * properly reconstruct framebuffers.
 835          *
 836          * FIXME: The intel_crtc->active here should be switched to
 837          * crtc->state->active once we have proper CRTC states wired up
 838          * for atomic.
 839          */
 840         return crtc && crtc->active && crtc->base.primary->state->fb &&
 841                 crtc->config->hw.adjusted_mode.crtc_clock;
 842 }
 843
 844 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
 845 {
 846         struct intel_crtc *crtc, *enabled = NULL;
 847
 848         for_each_intel_crtc(&dev_priv->drm, crtc) {
 849                 if (intel_crtc_active(crtc)) {
 850                         if (enabled)
 851                                 return NULL;
 852                         enabled = crtc;
 853                 }
 854         }
 855
 856         return enabled;
 857 }
 858
 859 static void pnv_update_wm(struct drm_i915_private *dev_priv)
 860 {
 861         struct intel_crtc *crtc;
 862         const struct cxsr_latency *latency;
 863         u32 reg;
 864         unsigned int wm;
 865
 866         latency = intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
 867                                          dev_priv->is_ddr3,
 868                                          dev_priv->fsb_freq,
 869                                          dev_priv->mem_freq);
 870         if (!latency) {
 871                 drm_dbg_kms(&dev_priv->drm,
 872                             "Unknown FSB/MEM found, disable CxSR\n");
 873                 intel_set_memory_cxsr(dev_priv, false);
 874                 return;
 875         }
 876
 877         crtc = single_enabled_crtc(dev_priv);
 878         if (crtc) {
 879                 const struct drm_framebuffer *fb =
 880                         crtc->base.primary->state->fb;
 881                 int pixel_rate = crtc->config->pixel_rate;
 882                 int cpp = fb->format->cpp[0];
 883
 884                 /* Display SR */
 885                 wm = intel_calculate_wm(pixel_rate, &pnv_display_wm,
 886                                         pnv_display_wm.fifo_size,
 887                                         cpp, latency->display_sr);
 888                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1);
 889                 reg &= ~DSPFW_SR_MASK;
 890                 reg |= FW_WM(wm, SR);
 891                 intel_uncore_write(&dev_priv->uncore, DSPFW1, reg);
 892                 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
 893
 894                 /* cursor SR */
 895                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_wm,
 896                                         pnv_display_wm.fifo_size,
 897                                         4, latency->cursor_sr);
 898                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_CURSOR_SR_MASK,
 899                                  FW_WM(wm, CURSOR_SR));
 900
 901                 /* Display HPLL off SR */
 902                 wm = intel_calculate_wm(pixel_rate, &pnv_display_hplloff_wm,
 903                                         pnv_display_hplloff_wm.fifo_size,
 904                                         cpp, latency->display_hpll_disable);
 905                 intel_uncore_rmw(&dev_priv->uncore, DSPFW3, DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
 906
 907                 /* cursor HPLL off SR */
 908                 wm = intel_calculate_wm(pixel_rate, &pnv_cursor_hplloff_wm,
 909                                         pnv_display_hplloff_wm.fifo_size,
 910                                         4, latency->cursor_hpll_disable);
 911                 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3);
 912                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 913                 reg |= FW_WM(wm, HPLL_CURSOR);
 914                 intel_uncore_write(&dev_priv->uncore, DSPFW3, reg);
 915                 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
 916
 917                 intel_set_memory_cxsr(dev_priv, true);
 918         } else {
 919                 intel_set_memory_cxsr(dev_priv, false);
 920         }
 921 }
 922
 923 /*
 924  * Documentation says:
 925  * "If the line size is small, the TLB fetches can get in the way of the
 926  *  data fetches, causing some lag in the pixel data return which is not
 927  *  accounted for in the above formulas. The following adjustment only
 928  *  needs to be applied if eight whole lines fit in the buffer at once.
 929  *  The WM is adjusted upwards by the difference between the FIFO size
 930  *  and the size of 8 whole lines. This adjustment is always performed
 931  *  in the actual pixel depth regardless of whether FBC is enabled or not."
 932  */
 933 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 934 {
 935         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 936
 937         return max(0, tlb_miss);
 938 }
 939
 940 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
 941                                 const struct g4x_wm_values *wm)
 942 {
 943         enum pipe pipe;
 944
 945         for_each_pipe(dev_priv, pipe)
 946                 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 947
 948         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 949                    FW_WM(wm->sr.plane, SR) |
 950                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
 951                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
 952                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
 953         intel_uncore_write(&dev_priv->uncore, DSPFW2,
 954                    (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
 955                    FW_WM(wm->sr.fbc, FBC_SR) |
 956                    FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
 957                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
 958                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
 959                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
 960         intel_uncore_write(&dev_priv->uncore, DSPFW3,
 961                    (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
 962                    FW_WM(wm->sr.cursor, CURSOR_SR) |
 963                    FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
 964                    FW_WM(wm->hpll.plane, HPLL_SR));
 965
 966         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
 967 }
 968
 969 #define FW_WM_VLV(value, plane) \
 970         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 971
 972 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
 973                                 const struct vlv_wm_values *wm)
 974 {
 975         enum pipe pipe;
 976
 977         for_each_pipe(dev_priv, pipe) {
 978                 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
 979
 980                 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
 981                            (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
 982                            (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
 983                            (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
 984                            (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
 985         }
 986
 987         /*
 988          * Zero the (unused) WM1 watermarks, and also clear all the
 989          * high order bits so that there are no out of bounds values
 990          * present in the registers during the reprogramming.
 991          */
 992         intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
 993         intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
 994         intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
 995         intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
 996         intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
 997
 998         intel_uncore_write(&dev_priv->uncore, DSPFW1,
 999                    FW_WM(wm->sr.plane, SR) |
1000                    FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
1001                    FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
1002                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
1003         intel_uncore_write(&dev_priv->uncore, DSPFW2,
1004                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
1005                    FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
1006                    FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
1007         intel_uncore_write(&dev_priv->uncore, DSPFW3,
1008                    FW_WM(wm->sr.cursor, CURSOR_SR));
1009
1010         if (IS_CHERRYVIEW(dev_priv)) {
1011                 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
1012                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1013                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1014                 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
1015                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
1016                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
1017                 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
1018                            FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
1019                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
1020                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1021                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1022                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
1023                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
1024                            FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
1025                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1026                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1027                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1028                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1029                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1030                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1031         } else {
1032                 intel_uncore_write(&dev_priv->uncore, DSPFW7,
1033                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
1034                            FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
1035                 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
1036                            FW_WM(wm->sr.plane >> 9, SR_HI) |
1037                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
1038                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
1039                            FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
1040                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
1041                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
1042                            FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
1043         }
1044
1045         intel_uncore_posting_read(&dev_priv->uncore, DSPFW1);
1046 }
1047
1048 #undef FW_WM_VLV
1049
1050 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
1051 {
1052         /* all latencies in usec */
1053         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
1054         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
1055         dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
1056
1057         dev_priv->display.wm.max_level = G4X_WM_LEVEL_HPLL;
1058 }
1059
1060 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
1061 {
1062         /*
1063          * DSPCNTR[13] supposedly controls whether the
1064          * primary plane can use the FIFO space otherwise
1065          * reserved for the sprite plane. It's not 100% clear
1066          * what the actual FIFO size is, but it looks like we
1067          * can happily set both primary and sprite watermarks
1068          * up to 127 cachelines. So that would seem to mean
1069          * that either DSPCNTR[13] doesn't do anything, or that
1070          * the total FIFO is >= 256 cachelines in size. Either
1071          * way, we don't seem to have to worry about this
1072          * repartitioning as the maximum watermark value the
1073          * register can hold for each plane is lower than the
1074          * minimum FIFO size.
1075          */
1076         switch (plane_id) {
1077         case PLANE_CURSOR:
1078                 return 63;
1079         case PLANE_PRIMARY:
1080                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
1081         case PLANE_SPRITE0:
1082                 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
1083         default:
1084                 MISSING_CASE(plane_id);
1085                 return 0;
1086         }
1087 }
1088
1089 static int g4x_fbc_fifo_size(int level)
1090 {
1091         switch (level) {
1092         case G4X_WM_LEVEL_SR:
1093                 return 7;
1094         case G4X_WM_LEVEL_HPLL:
1095                 return 15;
1096         default:
1097                 MISSING_CASE(level);
1098                 return 0;
1099         }
1100 }
1101
1102 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
1103                           const struct intel_plane_state *plane_state,
1104                           int level)
1105 {
1106         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1107         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1108         const struct drm_display_mode *pipe_mode =
1109                 &crtc_state->hw.pipe_mode;
1110         unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
1111         unsigned int pixel_rate, htotal, cpp, width, wm;
1112
1113         if (latency == 0)
1114                 return USHRT_MAX;
1115
1116         if (!intel_wm_plane_visible(crtc_state, plane_state))
1117                 return 0;
1118
1119         cpp = plane_state->hw.fb->format->cpp[0];
1120
1121         /*
1122          * WaUse32BppForSRWM:ctg,elk
1123          *
1124          * The spec fails to list this restriction for the
1125          * HPLL watermark, which seems a little strange.
1126          * Let's use 32bpp for the HPLL watermark as well.
1127          */
1128         if (plane->id == PLANE_PRIMARY &&
1129             level != G4X_WM_LEVEL_NORMAL)
1130                 cpp = max(cpp, 4u);
1131
1132         pixel_rate = crtc_state->pixel_rate;
1133         htotal = pipe_mode->crtc_htotal;
1134         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1135
1136         if (plane->id == PLANE_CURSOR) {
1137                 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1138         } else if (plane->id == PLANE_PRIMARY &&
1139                    level == G4X_WM_LEVEL_NORMAL) {
1140                 wm = intel_wm_method1(pixel_rate, cpp, latency);
1141         } else {
1142                 unsigned int small, large;
1143
1144                 small = intel_wm_method1(pixel_rate, cpp, latency);
1145                 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
1146
1147                 wm = min(small, large);
1148         }
1149
1150         wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
1151                               width, cpp);
1152
1153         wm = DIV_ROUND_UP(wm, 64) + 2;
1154
1155         return min_t(unsigned int, wm, USHRT_MAX);
1156 }
1157
1158 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1159                                  int level, enum plane_id plane_id, u16 value)
1160 {
1161         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1162         bool dirty = false;
1163
1164         for (; level < intel_wm_num_levels(dev_priv); level++) {
1165                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1166
1167                 dirty |= raw->plane[plane_id] != value;
1168                 raw->plane[plane_id] = value;
1169         }
1170
1171         return dirty;
1172 }
1173
1174 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
1175                                int level, u16 value)
1176 {
1177         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1178         bool dirty = false;
1179
1180         /* NORMAL level doesn't have an FBC watermark */
1181         level = max(level, G4X_WM_LEVEL_SR);
1182
1183         for (; level < intel_wm_num_levels(dev_priv); level++) {
1184                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1185
1186                 dirty |= raw->fbc != value;
1187                 raw->fbc = value;
1188         }
1189
1190         return dirty;
1191 }
1192
1193 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
1194                               const struct intel_plane_state *plane_state,
1195                               u32 pri_val);
1196
1197 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1198                                      const struct intel_plane_state *plane_state)
1199 {
1200         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1201         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1202         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1203         enum plane_id plane_id = plane->id;
1204         bool dirty = false;
1205         int level;
1206
1207         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1208                 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1209                 if (plane_id == PLANE_PRIMARY)
1210                         dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
1211                 goto out;
1212         }
1213
1214         for (level = 0; level < num_levels; level++) {
1215                 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1216                 int wm, max_wm;
1217
1218                 wm = g4x_compute_wm(crtc_state, plane_state, level);
1219                 max_wm = g4x_plane_fifo_size(plane_id, level);
1220
1221                 if (wm > max_wm)
1222                         break;
1223
1224                 dirty |= raw->plane[plane_id] != wm;
1225                 raw->plane[plane_id] = wm;
1226
1227                 if (plane_id != PLANE_PRIMARY ||
1228                     level == G4X_WM_LEVEL_NORMAL)
1229                         continue;
1230
1231                 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1232                                         raw->plane[plane_id]);
1233                 max_wm = g4x_fbc_fifo_size(level);
1234
1235                 /*
1236                  * FBC wm is not mandatory as we
1237                  * can always just disable its use.
1238                  */
1239                 if (wm > max_wm)
1240                         wm = USHRT_MAX;
1241
1242                 dirty |= raw->fbc != wm;
1243                 raw->fbc = wm;
1244         }
1245
1246         /* mark watermarks as invalid */
1247         dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1248
1249         if (plane_id == PLANE_PRIMARY)
1250                 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1251
1252  out:
1253         if (dirty) {
1254                 drm_dbg_kms(&dev_priv->drm,
1255                             "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1256                             plane->base.name,
1257                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1258                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1259                             crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1260
1261                 if (plane_id == PLANE_PRIMARY)
1262                         drm_dbg_kms(&dev_priv->drm,
1263                                     "FBC watermarks: SR=%d, HPLL=%d\n",
1264                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1265                                     crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1266         }
1267
1268         return dirty;
1269 }
1270
1271 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1272                                       enum plane_id plane_id, int level)
1273 {
1274         const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1275
1276         return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1277 }
1278
1279 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1280                                      int level)
1281 {
1282         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1283
1284         if (level > dev_priv->display.wm.max_level)
1285                 return false;
1286
1287         return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1288                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1289                 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1290 }
1291
1292 /* mark all levels starting from 'level' as invalid */
1293 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1294                                struct g4x_wm_state *wm_state, int level)
1295 {
1296         if (level <= G4X_WM_LEVEL_NORMAL) {
1297                 enum plane_id plane_id;
1298
1299                 for_each_plane_id_on_crtc(crtc, plane_id)
1300                         wm_state->wm.plane[plane_id] = USHRT_MAX;
1301         }
1302
1303         if (level <= G4X_WM_LEVEL_SR) {
1304                 wm_state->cxsr = false;
1305                 wm_state->sr.cursor = USHRT_MAX;
1306                 wm_state->sr.plane = USHRT_MAX;
1307                 wm_state->sr.fbc = USHRT_MAX;
1308         }
1309
1310         if (level <= G4X_WM_LEVEL_HPLL) {
1311                 wm_state->hpll_en = false;
1312                 wm_state->hpll.cursor = USHRT_MAX;
1313                 wm_state->hpll.plane = USHRT_MAX;
1314                 wm_state->hpll.fbc = USHRT_MAX;
1315         }
1316 }
1317
1318 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1319                                int level)
1320 {
1321         if (level < G4X_WM_LEVEL_SR)
1322                 return false;
1323
1324         if (level >= G4X_WM_LEVEL_SR &&
1325             wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1326                 return false;
1327
1328         if (level >= G4X_WM_LEVEL_HPLL &&
1329             wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1330                 return false;
1331
1332         return true;
1333 }
1334
1335 static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1336 {
1337         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1338         struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1339         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1340         const struct g4x_pipe_wm *raw;
1341         enum plane_id plane_id;
1342         int level;
1343
1344         level = G4X_WM_LEVEL_NORMAL;
1345         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1346                 goto out;
1347
1348         raw = &crtc_state->wm.g4x.raw[level];
1349         for_each_plane_id_on_crtc(crtc, plane_id)
1350                 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1351
1352         level = G4X_WM_LEVEL_SR;
1353         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1354                 goto out;
1355
1356         raw = &crtc_state->wm.g4x.raw[level];
1357         wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1358         wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1359         wm_state->sr.fbc = raw->fbc;
1360
1361         wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1362
1363         level = G4X_WM_LEVEL_HPLL;
1364         if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1365                 goto out;
1366
1367         raw = &crtc_state->wm.g4x.raw[level];
1368         wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1369         wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1370         wm_state->hpll.fbc = raw->fbc;
1371
1372         wm_state->hpll_en = wm_state->cxsr;
1373
1374         level++;
1375
1376  out:
1377         if (level == G4X_WM_LEVEL_NORMAL)
1378                 return -EINVAL;
1379
1380         /* invalidate the higher levels */
1381         g4x_invalidate_wms(crtc, wm_state, level);
1382
1383         /*
1384          * Determine if the FBC watermark(s) can be used. IF
1385          * this isn't the case we prefer to disable the FBC
1386          * watermark(s) rather than disable the SR/HPLL
1387          * level(s) entirely. 'level-1' is the highest valid
1388          * level here.
1389          */
1390         wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1391
1392         return 0;
1393 }
1394
1395 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1396                                struct intel_crtc *crtc)
1397 {
1398         struct intel_crtc_state *crtc_state =
1399                 intel_atomic_get_new_crtc_state(state, crtc);
1400         const struct intel_plane_state *old_plane_state;
1401         const struct intel_plane_state *new_plane_state;
1402         struct intel_plane *plane;
1403         unsigned int dirty = 0;
1404         int i;
1405
1406         for_each_oldnew_intel_plane_in_state(state, plane,
1407                                              old_plane_state,
1408                                              new_plane_state, i) {
1409                 if (new_plane_state->hw.crtc != &crtc->base &&
1410                     old_plane_state->hw.crtc != &crtc->base)
1411                         continue;
1412
1413                 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1414                         dirty |= BIT(plane->id);
1415         }
1416
1417         if (!dirty)
1418                 return 0;
1419
1420         return _g4x_compute_pipe_wm(crtc_state);
1421 }
1422
1423 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1424                                        struct intel_crtc *crtc)
1425 {
1426         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1427         struct intel_crtc_state *new_crtc_state =
1428                 intel_atomic_get_new_crtc_state(state, crtc);
1429         const struct intel_crtc_state *old_crtc_state =
1430                 intel_atomic_get_old_crtc_state(state, crtc);
1431         struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1432         const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1433         const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1434         enum plane_id plane_id;
1435
1436         if (!new_crtc_state->hw.active ||
1437             drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
1438                 *intermediate = *optimal;
1439
1440                 intermediate->cxsr = false;
1441                 intermediate->hpll_en = false;
1442                 goto out;
1443         }
1444
1445         intermediate->cxsr = optimal->cxsr && active->cxsr &&
1446                 !new_crtc_state->disable_cxsr;
1447         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1448                 !new_crtc_state->disable_cxsr;
1449         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1450
1451         for_each_plane_id_on_crtc(crtc, plane_id) {
1452                 intermediate->wm.plane[plane_id] =
1453                         max(optimal->wm.plane[plane_id],
1454                             active->wm.plane[plane_id]);
1455
1456                 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1457                             g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1458         }
1459
1460         intermediate->sr.plane = max(optimal->sr.plane,
1461                                      active->sr.plane);
1462         intermediate->sr.cursor = max(optimal->sr.cursor,
1463                                       active->sr.cursor);
1464         intermediate->sr.fbc = max(optimal->sr.fbc,
1465                                    active->sr.fbc);
1466
1467         intermediate->hpll.plane = max(optimal->hpll.plane,
1468                                        active->hpll.plane);
1469         intermediate->hpll.cursor = max(optimal->hpll.cursor,
1470                                         active->hpll.cursor);
1471         intermediate->hpll.fbc = max(optimal->hpll.fbc,
1472                                      active->hpll.fbc);
1473
1474         drm_WARN_ON(&dev_priv->drm,
1475                     (intermediate->sr.plane >
1476                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1477                      intermediate->sr.cursor >
1478                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1479                     intermediate->cxsr);
1480         drm_WARN_ON(&dev_priv->drm,
1481                     (intermediate->sr.plane >
1482                      g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1483                      intermediate->sr.cursor >
1484                      g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1485                     intermediate->hpll_en);
1486
1487         drm_WARN_ON(&dev_priv->drm,
1488                     intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1489                     intermediate->fbc_en && intermediate->cxsr);
1490         drm_WARN_ON(&dev_priv->drm,
1491                     intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1492                     intermediate->fbc_en && intermediate->hpll_en);
1493
1494 out:
1495         /*
1496          * If our intermediate WM are identical to the final WM, then we can
1497          * omit the post-vblank programming; only update if it's different.
1498          */
1499         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1500                 new_crtc_state->wm.need_postvbl_update = true;
1501
1502         return 0;
1503 }
1504
1505 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1506                          struct g4x_wm_values *wm)
1507 {
1508         struct intel_crtc *crtc;
1509         int num_active_pipes = 0;
1510
1511         wm->cxsr = true;
1512         wm->hpll_en = true;
1513         wm->fbc_en = true;
1514
1515         for_each_intel_crtc(&dev_priv->drm, crtc) {
1516                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1517
1518                 if (!crtc->active)
1519                         continue;
1520
1521                 if (!wm_state->cxsr)
1522                         wm->cxsr = false;
1523                 if (!wm_state->hpll_en)
1524                         wm->hpll_en = false;
1525                 if (!wm_state->fbc_en)
1526                         wm->fbc_en = false;
1527
1528                 num_active_pipes++;
1529         }
1530
1531         if (num_active_pipes != 1) {
1532                 wm->cxsr = false;
1533                 wm->hpll_en = false;
1534                 wm->fbc_en = false;
1535         }
1536
1537         for_each_intel_crtc(&dev_priv->drm, crtc) {
1538                 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1539                 enum pipe pipe = crtc->pipe;
1540
1541                 wm->pipe[pipe] = wm_state->wm;
1542                 if (crtc->active && wm->cxsr)
1543                         wm->sr = wm_state->sr;
1544                 if (crtc->active && wm->hpll_en)
1545                         wm->hpll = wm_state->hpll;
1546         }
1547 }
1548
1549 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1550 {
1551         struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1552         struct g4x_wm_values new_wm = {};
1553
1554         g4x_merge_wm(dev_priv, &new_wm);
1555
1556         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1557                 return;
1558
1559         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1560                 _intel_set_memory_cxsr(dev_priv, false);
1561
1562         g4x_write_wm_values(dev_priv, &new_wm);
1563
1564         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1565                 _intel_set_memory_cxsr(dev_priv, true);
1566
1567         *old_wm = new_wm;
1568 }
1569
1570 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1571                                    struct intel_crtc *crtc)
1572 {
1573         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1574         const struct intel_crtc_state *crtc_state =
1575                 intel_atomic_get_new_crtc_state(state, crtc);
1576
1577         mutex_lock(&dev_priv->display.wm.wm_mutex);
1578         crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1579         g4x_program_watermarks(dev_priv);
1580         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1581 }
1582
1583 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1584                                     struct intel_crtc *crtc)
1585 {
1586         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1587         const struct intel_crtc_state *crtc_state =
1588                 intel_atomic_get_new_crtc_state(state, crtc);
1589
1590         if (!crtc_state->wm.need_postvbl_update)
1591                 return;
1592
1593         mutex_lock(&dev_priv->display.wm.wm_mutex);
1594         crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1595         g4x_program_watermarks(dev_priv);
1596         mutex_unlock(&dev_priv->display.wm.wm_mutex);
1597 }
1598
1599 /* latency must be in 0.1us units. */
1600 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1601                                    unsigned int htotal,
1602                                    unsigned int width,
1603                                    unsigned int cpp,
1604                                    unsigned int latency)
1605 {
1606         unsigned int ret;
1607
1608         ret = intel_wm_method2(pixel_rate, htotal,
1609                                width, cpp, latency);
1610         ret = DIV_ROUND_UP(ret, 64);
1611
1612         return ret;
1613 }
1614
1615 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1616 {
1617         /* all latencies in usec */
1618         dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1619
1620         dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM2;
1621
1622         if (IS_CHERRYVIEW(dev_priv)) {
1623                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1624                 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1625
1626                 dev_priv->display.wm.max_level = VLV_WM_LEVEL_DDR_DVFS;
1627         }
1628 }
1629
1630 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1631                                 const struct intel_plane_state *plane_state,
1632                                 int level)
1633 {
1634         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1635         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1636         const struct drm_display_mode *pipe_mode =
1637                 &crtc_state->hw.pipe_mode;
1638         unsigned int pixel_rate, htotal, cpp, width, wm;
1639
1640         if (dev_priv->display.wm.pri_latency[level] == 0)
1641                 return USHRT_MAX;
1642
1643         if (!intel_wm_plane_visible(crtc_state, plane_state))
1644                 return 0;
1645
1646         cpp = plane_state->hw.fb->format->cpp[0];
1647         pixel_rate = crtc_state->pixel_rate;
1648         htotal = pipe_mode->crtc_htotal;
1649         width = drm_rect_width(&plane_state->uapi.src) >> 16;
1650
1651         if (plane->id == PLANE_CURSOR) {
1652                 /*
1653                  * FIXME the formula gives values that are
1654                  * too big for the cursor FIFO, and hence we
1655                  * would never be able to use cursors. For
1656                  * now just hardcode the watermark.
1657                  */
1658                 wm = 63;
1659         } else {
1660                 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1661                                     dev_priv->display.wm.pri_latency[level] * 10);
1662         }
1663
1664         return min_t(unsigned int, wm, USHRT_MAX);
1665 }
1666
1667 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1668 {
1669         return (active_planes & (BIT(PLANE_SPRITE0) |
1670                                  BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1671 }
1672
1673 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1674 {
1675         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1676         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1677         const struct g4x_pipe_wm *raw =
1678                 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1679         struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1680         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1681         int num_active_planes = hweight8(active_planes);
1682         const int fifo_size = 511;
1683         int fifo_extra, fifo_left = fifo_size;
1684         int sprite0_fifo_extra = 0;
1685         unsigned int total_rate;
1686         enum plane_id plane_id;
1687
1688         /*
1689          * When enabling sprite0 after sprite1 has already been enabled
1690          * we tend to get an underrun unless sprite0 already has some
1691          * FIFO space allcoated. Hence we always allocate at least one
1692          * cacheline for sprite0 whenever sprite1 is enabled.
1693          *
1694          * All other plane enable sequences appear immune to this problem.
1695          */
1696         if (vlv_need_sprite0_fifo_workaround(active_planes))
1697                 sprite0_fifo_extra = 1;
1698
1699         total_rate = raw->plane[PLANE_PRIMARY] +
1700                 raw->plane[PLANE_SPRITE0] +
1701                 raw->plane[PLANE_SPRITE1] +
1702                 sprite0_fifo_extra;
1703
1704         if (total_rate > fifo_size)
1705                 return -EINVAL;
1706
1707         if (total_rate == 0)
1708                 total_rate = 1;
1709
1710         for_each_plane_id_on_crtc(crtc, plane_id) {
1711                 unsigned int rate;
1712
1713                 if ((active_planes & BIT(plane_id)) == 0) {
1714                         fifo_state->plane[plane_id] = 0;
1715                         continue;
1716                 }
1717
1718                 rate = raw->plane[plane_id];
1719                 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1720                 fifo_left -= fifo_state->plane[plane_id];
1721         }
1722
1723         fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1724         fifo_left -= sprite0_fifo_extra;
1725
1726         fifo_state->plane[PLANE_CURSOR] = 63;
1727
1728         fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1729
1730         /* spread the remainder evenly */
1731         for_each_plane_id_on_crtc(crtc, plane_id) {
1732                 int plane_extra;
1733
1734                 if (fifo_left == 0)
1735                         break;
1736
1737                 if ((active_planes & BIT(plane_id)) == 0)
1738                         continue;
1739
1740                 plane_extra = min(fifo_extra, fifo_left);
1741                 fifo_state->plane[plane_id] += plane_extra;
1742                 fifo_left -= plane_extra;
1743         }
1744
1745         drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1746
1747         /* give it all to the first plane if none are active */
1748         if (active_planes == 0) {
1749                 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1750                 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1751         }
1752
1753         return 0;
1754 }
1755
1756 /* mark all levels starting from 'level' as invalid */
1757 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1758                                struct vlv_wm_state *wm_state, int level)
1759 {
1760         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1761
1762         for (; level < intel_wm_num_levels(dev_priv); level++) {
1763                 enum plane_id plane_id;
1764
1765                 for_each_plane_id_on_crtc(crtc, plane_id)
1766                         wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1767
1768                 wm_state->sr[level].cursor = USHRT_MAX;
1769                 wm_state->sr[level].plane = USHRT_MAX;
1770         }
1771 }
1772
1773 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1774 {
1775         if (wm > fifo_size)
1776                 return USHRT_MAX;
1777         else
1778                 return fifo_size - wm;
1779 }
1780
1781 /*
1782  * Starting from 'level' set all higher
1783  * levels to 'value' in the "raw" watermarks.
1784  */
1785 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1786                                  int level, enum plane_id plane_id, u16 value)
1787 {
1788         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1789         int num_levels = intel_wm_num_levels(dev_priv);
1790         bool dirty = false;
1791
1792         for (; level < num_levels; level++) {
1793                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1794
1795                 dirty |= raw->plane[plane_id] != value;
1796                 raw->plane[plane_id] = value;
1797         }
1798
1799         return dirty;
1800 }
1801
1802 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1803                                      const struct intel_plane_state *plane_state)
1804 {
1805         struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1806         struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1807         enum plane_id plane_id = plane->id;
1808         int num_levels = intel_wm_num_levels(to_i915(plane->base.dev));
1809         int level;
1810         bool dirty = false;
1811
1812         if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1813                 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1814                 goto out;
1815         }
1816
1817         for (level = 0; level < num_levels; level++) {
1818                 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1819                 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1820                 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1821
1822                 if (wm > max_wm)
1823                         break;
1824
1825                 dirty |= raw->plane[plane_id] != wm;
1826                 raw->plane[plane_id] = wm;
1827         }
1828
1829         /* mark all higher levels as invalid */
1830         dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1831
1832 out:
1833         if (dirty)
1834                 drm_dbg_kms(&dev_priv->drm,
1835                             "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1836                             plane->base.name,
1837                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1838                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1839                             crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1840
1841         return dirty;
1842 }
1843
1844 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1845                                       enum plane_id plane_id, int level)
1846 {
1847         const struct g4x_pipe_wm *raw =
1848                 &crtc_state->wm.vlv.raw[level];
1849         const struct vlv_fifo_state *fifo_state =
1850                 &crtc_state->wm.vlv.fifo_state;
1851
1852         return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1853 }
1854
1855 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1856 {
1857         return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1858                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1859                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1860                 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1861 }
1862
1863 static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1864 {
1865         struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1866         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1867         struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1868         const struct vlv_fifo_state *fifo_state =
1869                 &crtc_state->wm.vlv.fifo_state;
1870         u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1871         int num_active_planes = hweight8(active_planes);
1872         enum plane_id plane_id;
1873         int level;
1874
1875         /* initially allow all levels */
1876         wm_state->num_levels = intel_wm_num_levels(dev_priv);
1877         /*
1878          * Note that enabling cxsr with no primary/sprite planes
1879          * enabled can wedge the pipe. Hence we only allow cxsr
1880          * with exactly one enabled primary/sprite plane.
1881          */
1882         wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1883
1884         for (level = 0; level < wm_state->num_levels; level++) {
1885                 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1886                 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1887
1888                 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1889                         break;
1890
1891                 for_each_plane_id_on_crtc(crtc, plane_id) {
1892                         wm_state->wm[level].plane[plane_id] =
1893                                 vlv_invert_wm_value(raw->plane[plane_id],
1894                                                     fifo_state->plane[plane_id]);
1895                 }
1896
1897                 wm_state->sr[level].plane =
1898                         vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1899                                                  raw->plane[PLANE_SPRITE0],
1900                                                  raw->plane[PLANE_SPRITE1]),
1901                                             sr_fifo_size);
1902
1903                 wm_state->sr[level].cursor =
1904                         vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1905                                             63);
1906         }
1907
1908         if (level == 0)
1909                 return -EINVAL;
1910
1911         /* limit to only levels we can actually handle */
1912         wm_state->num_levels = level;
1913
1914         /* invalidate the higher levels */
1915         vlv_invalidate_wms(crtc, wm_state, level);
1916
1917         return 0;
1918 }
1919
1920 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1921                                struct intel_crtc *crtc)
1922 {
1923         struct intel_crtc_state *crtc_state =
1924                 intel_atomic_get_new_crtc_state(state, crtc);
1925         bool needs_modeset = drm_atomic_crtc_needs_modeset(&crtc_state->uapi);
1926         const struct intel_plane_state *old_plane_state;
1927         const struct intel_plane_state *new_plane_state;
1928         struct intel_plane *plane;
1929         unsigned int dirty = 0;
1930         int i;
1931
1932         for_each_oldnew_intel_plane_in_state(state, plane,
1933                                              old_plane_state,
1934                                              new_plane_state, i) {
1935                 if (new_plane_state->hw.crtc != &crtc->base &&
1936                     old_plane_state->hw.crtc != &crtc->base)
1937                         continue;
1938
1939                 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1940                         dirty |= BIT(plane->id);
1941         }
1942
1943         /*
1944          * DSPARB registers may have been reset due to the
1945          * power well being turned off. Make sure we restore
1946          * them to a consistent state even if no primary/sprite
1947          * planes are initially active. We also force a FIFO
1948          * recomputation so that we are sure to sanitize the
1949          * FIFO setting we took over from the BIOS even if there
1950          * are no active planes on the crtc.
1951          */
1952         if (needs_modeset)
1953                 dirty = ~0;
1954
1955         if (!dirty)
1956                 return 0;
1957
1958         /* cursor changes don't warrant a FIFO recompute */
1959         if (dirty & ~BIT(PLANE_CURSOR)) {
1960                 const struct intel_crtc_state *old_crtc_state =
1961                         intel_atomic_get_old_crtc_state(state, crtc);
1962                 const struct vlv_fifo_state *old_fifo_state =
1963                         &old_crtc_state->wm.vlv.fifo_state;
1964                 const struct vlv_fifo_state *new_fifo_state =
1965                         &crtc_state->wm.vlv.fifo_state;
1966                 int ret;
1967
1968                 ret = vlv_compute_fifo(crtc_state);
1969                 if (ret)
1970                         return ret;
1971
1972                 if (needs_modeset ||
1973                     memcmp(old_fifo_state, new_fifo_state,
1974                            sizeof(*new_fifo_state)) != 0)
1975                         crtc_state->fifo_changed = true;
1976         }
1977
1978         return _vlv_compute_pipe_wm(crtc_state);
1979 }
1980
1981 #define VLV_FIFO(plane, value) \
1982         (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1983
1984 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1985                                    struct intel_crtc *crtc)
1986 {
1987         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1988         struct intel_uncore *uncore = &dev_priv->uncore;
1989         const struct intel_crtc_state *crtc_state =
1990                 intel_atomic_get_new_crtc_state(state, crtc);
1991         const struct vlv_fifo_state *fifo_state =
1992                 &crtc_state->wm.vlv.fifo_state;
1993         int sprite0_start, sprite1_start, fifo_size;
1994         u32 dsparb, dsparb2, dsparb3;
1995
1996         if (!crtc_state->fifo_changed)
1997                 return;
1998
1999         sprite0_start = fifo_state->plane[PLANE_PRIMARY];
2000         sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
2001         fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
2002
2003         drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
2004         drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
2005
2006         trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
2007
2008         /*
2009          * uncore.lock serves a double purpose here. It allows us to
2010          * use the less expensive I915_{READ,WRITE}_FW() functions, and
2011          * it protects the DSPARB registers from getting clobbered by
2012          * parallel updates from multiple pipes.
2013          *
2014          * intel_pipe_update_start() has already disabled interrupts
2015          * for us, so a plain spin_lock() is sufficient here.
2016          */
2017         spin_lock(&uncore->lock);
2018
2019         switch (crtc->pipe) {
2020         case PIPE_A:
2021                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2022                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2023
2024                 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
2025                             VLV_FIFO(SPRITEB, 0xff));
2026                 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
2027                            VLV_FIFO(SPRITEB, sprite1_start));
2028
2029                 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
2030                              VLV_FIFO(SPRITEB_HI, 0x1));
2031                 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
2032                            VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
2033
2034                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2035                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2036                 break;
2037         case PIPE_B:
2038                 dsparb = intel_uncore_read_fw(uncore, DSPARB);
2039                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2040
2041                 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
2042                             VLV_FIFO(SPRITED, 0xff));
2043                 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
2044                            VLV_FIFO(SPRITED, sprite1_start));
2045
2046                 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
2047                              VLV_FIFO(SPRITED_HI, 0xff));
2048                 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
2049                            VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
2050
2051                 intel_uncore_write_fw(uncore, DSPARB, dsparb);
2052                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2053                 break;
2054         case PIPE_C:
2055                 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
2056                 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
2057
2058                 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
2059                              VLV_FIFO(SPRITEF, 0xff));
2060                 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
2061                             VLV_FIFO(SPRITEF, sprite1_start));
2062
2063                 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
2064                              VLV_FIFO(SPRITEF_HI, 0xff));
2065                 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
2066                            VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
2067
2068                 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
2069                 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
2070                 break;
2071         default:
2072                 break;
2073         }
2074
2075         intel_uncore_posting_read_fw(uncore, DSPARB);
2076
2077         spin_unlock(&uncore->lock);
2078 }
2079
2080 #undef VLV_FIFO
2081
2082 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
2083                                        struct intel_crtc *crtc)
2084 {
2085         struct intel_crtc_state *new_crtc_state =
2086                 intel_atomic_get_new_crtc_state(state, crtc);
2087         const struct intel_crtc_state *old_crtc_state =
2088                 intel_atomic_get_old_crtc_state(state, crtc);
2089         struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
2090         const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
2091         const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
2092         int level;
2093
2094         if (!new_crtc_state->hw.active ||
2095             drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi)) {
2096                 *intermediate = *optimal;
2097
2098                 intermediate->cxsr = false;
2099                 goto out;
2100         }
2101
2102         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
2103         intermediate->cxsr = optimal->cxsr && active->cxsr &&
2104                 !new_crtc_state->disable_cxsr;
2105
2106         for (level = 0; level < intermediate->num_levels; level++) {
2107                 enum plane_id plane_id;
2108
2109                 for_each_plane_id_on_crtc(crtc, plane_id) {
2110                         intermediate->wm[level].plane[plane_id] =
2111                                 min(optimal->wm[level].plane[plane_id],
2112                                     active->wm[level].plane[plane_id]);
2113                 }
2114
2115                 intermediate->sr[level].plane = min(optimal->sr[level].plane,
2116                                                     active->sr[level].plane);
2117                 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
2118                                                      active->sr[level].cursor);
2119         }
2120
2121         vlv_invalidate_wms(crtc, intermediate, level);
2122
2123 out:
2124         /*
2125          * If our intermediate WM are identical to the final WM, then we can
2126          * omit the post-vblank programming; only update if it's different.
2127          */
2128         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
2129                 new_crtc_state->wm.need_postvbl_update = true;
2130
2131         return 0;
2132 }
2133
2134 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
2135                          struct vlv_wm_values *wm)
2136 {
2137         struct intel_crtc *crtc;
2138         int num_active_pipes = 0;
2139
2140         wm->level = dev_priv->display.wm.max_level;
2141         wm->cxsr = true;
2142
2143         for_each_intel_crtc(&dev_priv->drm, crtc) {
2144                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2145
2146                 if (!crtc->active)
2147                         continue;
2148
2149                 if (!wm_state->cxsr)
2150                         wm->cxsr = false;
2151
2152                 num_active_pipes++;
2153                 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
2154         }
2155
2156         if (num_active_pipes != 1)
2157                 wm->cxsr = false;
2158
2159         if (num_active_pipes > 1)
2160                 wm->level = VLV_WM_LEVEL_PM2;
2161
2162         for_each_intel_crtc(&dev_priv->drm, crtc) {
2163                 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
2164                 enum pipe pipe = crtc->pipe;
2165
2166                 wm->pipe[pipe] = wm_state->wm[wm->level];
2167                 if (crtc->active && wm->cxsr)
2168                         wm->sr = wm_state->sr[wm->level];
2169
2170                 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
2171                 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
2172                 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
2173                 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
2174         }
2175 }
2176
2177 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
2178 {
2179         struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
2180         struct vlv_wm_values new_wm = {};
2181
2182         vlv_merge_wm(dev_priv, &new_wm);
2183
2184         if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
2185                 return;
2186
2187         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2188                 chv_set_memory_dvfs(dev_priv, false);
2189
2190         if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2191                 chv_set_memory_pm5(dev_priv, false);
2192
2193         if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
2194                 _intel_set_memory_cxsr(dev_priv, false);
2195
2196         vlv_write_wm_values(dev_priv, &new_wm);
2197
2198         if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
2199                 _intel_set_memory_cxsr(dev_priv, true);
2200
2201         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
2202                 chv_set_memory_pm5(dev_priv, true);
2203
2204         if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
2205                 chv_set_memory_dvfs(dev_priv, true);
2206
2207         *old_wm = new_wm;
2208 }
2209
2210 static void vlv_initial_watermarks(struct intel_atomic_state *state,
2211                                    struct intel_crtc *crtc)
2212 {
2213         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2214         const struct intel_crtc_state *crtc_state =
2215                 intel_atomic_get_new_crtc_state(state, crtc);
2216
2217         mutex_lock(&dev_priv->display.wm.wm_mutex);
2218         crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
2219         vlv_program_watermarks(dev_priv);
2220         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2221 }
2222
2223 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
2224                                     struct intel_crtc *crtc)
2225 {
2226         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2227         const struct intel_crtc_state *crtc_state =
2228                 intel_atomic_get_new_crtc_state(state, crtc);
2229
2230         if (!crtc_state->wm.need_postvbl_update)
2231                 return;
2232
2233         mutex_lock(&dev_priv->display.wm.wm_mutex);
2234         crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2235         vlv_program_watermarks(dev_priv);
2236         mutex_unlock(&dev_priv->display.wm.wm_mutex);
2237 }
2238
2239 static void i965_update_wm(struct drm_i915_private *dev_priv)
2240 {
2241         struct intel_crtc *crtc;
2242         int srwm = 1;
2243         int cursor_sr = 16;
2244         bool cxsr_enabled;
2245
2246         /* Calc sr entries for one plane configs */
2247         crtc = single_enabled_crtc(dev_priv);
2248         if (crtc) {
2249                 /* self-refresh has much higher latency */
2250                 static const int sr_latency_ns = 12000;
2251                 const struct drm_display_mode *pipe_mode =
2252                         &crtc->config->hw.pipe_mode;
2253                 const struct drm_framebuffer *fb =
2254                         crtc->base.primary->state->fb;
2255                 int pixel_rate = crtc->config->pixel_rate;
2256                 int htotal = pipe_mode->crtc_htotal;
2257                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2258                 int cpp = fb->format->cpp[0];
2259                 int entries;
2260
2261                 entries = intel_wm_method2(pixel_rate, htotal,
2262                                            width, cpp, sr_latency_ns / 100);
2263                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2264                 srwm = I965_FIFO_SIZE - entries;
2265                 if (srwm < 0)
2266                         srwm = 1;
2267                 srwm &= 0x1ff;
2268                 drm_dbg_kms(&dev_priv->drm,
2269                             "self-refresh entries: %d, wm: %d\n",
2270                             entries, srwm);
2271
2272                 entries = intel_wm_method2(pixel_rate, htotal,
2273                                            crtc->base.cursor->state->crtc_w, 4,
2274                                            sr_latency_ns / 100);
2275                 entries = DIV_ROUND_UP(entries,
2276                                        i965_cursor_wm_info.cacheline_size) +
2277                         i965_cursor_wm_info.guard_size;
2278
2279                 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2280                 if (cursor_sr > i965_cursor_wm_info.max_wm)
2281                         cursor_sr = i965_cursor_wm_info.max_wm;
2282
2283                 drm_dbg_kms(&dev_priv->drm,
2284                             "self-refresh watermark: display plane %d "
2285                             "cursor %d\n", srwm, cursor_sr);
2286
2287                 cxsr_enabled = true;
2288         } else {
2289                 cxsr_enabled = false;
2290                 /* Turn off self refresh if both pipes are enabled */
2291                 intel_set_memory_cxsr(dev_priv, false);
2292         }
2293
2294         drm_dbg_kms(&dev_priv->drm,
2295                     "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2296                     srwm);
2297
2298         /* 965 has limitations... */
2299         intel_uncore_write(&dev_priv->uncore, DSPFW1, FW_WM(srwm, SR) |
2300                    FW_WM(8, CURSORB) |
2301                    FW_WM(8, PLANEB) |
2302                    FW_WM(8, PLANEA));
2303         intel_uncore_write(&dev_priv->uncore, DSPFW2, FW_WM(8, CURSORA) |
2304                    FW_WM(8, PLANEC_OLD));
2305         /* update cursor SR watermark */
2306         intel_uncore_write(&dev_priv->uncore, DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
2307
2308         if (cxsr_enabled)
2309                 intel_set_memory_cxsr(dev_priv, true);
2310 }
2311
2312 #undef FW_WM
2313
2314 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2315                                                enum i9xx_plane_id i9xx_plane)
2316 {
2317         struct intel_plane *plane;
2318
2319         for_each_intel_plane(&i915->drm, plane) {
2320                 if (plane->id == PLANE_PRIMARY &&
2321                     plane->i9xx_plane == i9xx_plane)
2322                         return intel_crtc_for_pipe(i915, plane->pipe);
2323         }
2324
2325         return NULL;
2326 }
2327
2328 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2329 {
2330         const struct intel_watermark_params *wm_info;
2331         u32 fwater_lo;
2332         u32 fwater_hi;
2333         int cwm, srwm = 1;
2334         int fifo_size;
2335         int planea_wm, planeb_wm;
2336         struct intel_crtc *crtc;
2337
2338         if (IS_I945GM(dev_priv))
2339                 wm_info = &i945_wm_info;
2340         else if (DISPLAY_VER(dev_priv) != 2)
2341                 wm_info = &i915_wm_info;
2342         else
2343                 wm_info = &i830_a_wm_info;
2344
2345         if (DISPLAY_VER(dev_priv) == 2)
2346                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2347         else
2348                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2349         crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2350         if (intel_crtc_active(crtc)) {
2351                 const struct drm_framebuffer *fb =
2352                         crtc->base.primary->state->fb;
2353                 int cpp;
2354
2355                 if (DISPLAY_VER(dev_priv) == 2)
2356                         cpp = 4;
2357                 else
2358                         cpp = fb->format->cpp[0];
2359
2360                 planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2361                                                wm_info, fifo_size, cpp,
2362                                                pessimal_latency_ns);
2363         } else {
2364                 planea_wm = fifo_size - wm_info->guard_size;
2365                 if (planea_wm > (long)wm_info->max_wm)
2366                         planea_wm = wm_info->max_wm;
2367         }
2368
2369         if (DISPLAY_VER(dev_priv) == 2)
2370                 wm_info = &i830_bc_wm_info;
2371
2372         if (DISPLAY_VER(dev_priv) == 2)
2373                 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2374         else
2375                 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2376         crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2377         if (intel_crtc_active(crtc)) {
2378                 const struct drm_framebuffer *fb =
2379                         crtc->base.primary->state->fb;
2380                 int cpp;
2381
2382                 if (DISPLAY_VER(dev_priv) == 2)
2383                         cpp = 4;
2384                 else
2385                         cpp = fb->format->cpp[0];
2386
2387                 planeb_wm = intel_calculate_wm(crtc->config->pixel_rate,
2388                                                wm_info, fifo_size, cpp,
2389                                                pessimal_latency_ns);
2390         } else {
2391                 planeb_wm = fifo_size - wm_info->guard_size;
2392                 if (planeb_wm > (long)wm_info->max_wm)
2393                         planeb_wm = wm_info->max_wm;
2394         }
2395
2396         drm_dbg_kms(&dev_priv->drm,
2397                     "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2398
2399         crtc = single_enabled_crtc(dev_priv);
2400         if (IS_I915GM(dev_priv) && crtc) {
2401                 struct drm_i915_gem_object *obj;
2402
2403                 obj = intel_fb_obj(crtc->base.primary->state->fb);
2404
2405                 /* self-refresh seems busted with untiled */
2406                 if (!i915_gem_object_is_tiled(obj))
2407                         crtc = NULL;
2408         }
2409
2410         /*
2411          * Overlay gets an aggressive default since video jitter is bad.
2412          */
2413         cwm = 2;
2414
2415         /* Play safe and disable self-refresh before adjusting watermarks. */
2416         intel_set_memory_cxsr(dev_priv, false);
2417
2418         /* Calc sr entries for one plane configs */
2419         if (HAS_FW_BLC(dev_priv) && crtc) {
2420                 /* self-refresh has much higher latency */
2421                 static const int sr_latency_ns = 6000;
2422                 const struct drm_display_mode *pipe_mode =
2423                         &crtc->config->hw.pipe_mode;
2424                 const struct drm_framebuffer *fb =
2425                         crtc->base.primary->state->fb;
2426                 int pixel_rate = crtc->config->pixel_rate;
2427                 int htotal = pipe_mode->crtc_htotal;
2428                 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2429                 int cpp;
2430                 int entries;
2431
2432                 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2433                         cpp = 4;
2434                 else
2435                         cpp = fb->format->cpp[0];
2436
2437                 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2438                                            sr_latency_ns / 100);
2439                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2440                 drm_dbg_kms(&dev_priv->drm,
2441                             "self-refresh entries: %d\n", entries);
2442                 srwm = wm_info->fifo_size - entries;
2443                 if (srwm < 0)
2444                         srwm = 1;
2445
2446                 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2447                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2448                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2449                 else
2450                         intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2451         }
2452
2453         drm_dbg_kms(&dev_priv->drm,
2454                     "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2455                      planea_wm, planeb_wm, cwm, srwm);
2456
2457         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2458         fwater_hi = (cwm & 0x1f);
2459
2460         /* Set request length to 8 cachelines per fetch */
2461         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2462         fwater_hi = fwater_hi | (1 << 8);
2463
2464         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2465         intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2466
2467         if (crtc)
2468                 intel_set_memory_cxsr(dev_priv, true);
2469 }
2470
2471 static void i845_update_wm(struct drm_i915_private *dev_priv)
2472 {
2473         struct intel_crtc *crtc;
2474         u32 fwater_lo;
2475         int planea_wm;
2476
2477         crtc = single_enabled_crtc(dev_priv);
2478         if (crtc == NULL)
2479                 return;
2480
2481         planea_wm = intel_calculate_wm(crtc->config->pixel_rate,
2482                                        &i845_wm_info,
2483                                        i845_get_fifo_size(dev_priv, PLANE_A),
2484                                        4, pessimal_latency_ns);
2485         fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2486         fwater_lo |= (3<<8) | planea_wm;
2487
2488         drm_dbg_kms(&dev_priv->drm,
2489                     "Setting FIFO watermarks - A: %d\n", planea_wm);
2490
2491         intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2492 }
2493
2494 /* latency must be in 0.1us units. */
2495 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2496                                    unsigned int cpp,
2497                                    unsigned int latency)
2498 {
2499         unsigned int ret;
2500
2501         ret = intel_wm_method1(pixel_rate, cpp, latency);
2502         ret = DIV_ROUND_UP(ret, 64) + 2;
2503
2504         return ret;
2505 }
2506
2507 /* latency must be in 0.1us units. */
2508 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2509                                    unsigned int htotal,
2510                                    unsigned int width,
2511                                    unsigned int cpp,
2512                                    unsigned int latency)
2513 {
2514         unsigned int ret;
2515
2516         ret = intel_wm_method2(pixel_rate, htotal,
2517                                width, cpp, latency);
2518         ret = DIV_ROUND_UP(ret, 64) + 2;
2519
2520         return ret;
2521 }
2522
2523 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2524 {
2525         /*
2526          * Neither of these should be possible since this function shouldn't be
2527          * called if the CRTC is off or the plane is invisible.  But let's be
2528          * extra paranoid to avoid a potential divide-by-zero if we screw up
2529          * elsewhere in the driver.
2530          */
2531         if (WARN_ON(!cpp))
2532                 return 0;
2533         if (WARN_ON(!horiz_pixels))
2534                 return 0;
2535
2536         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2537 }
2538
2539 struct ilk_wm_maximums {
2540         u16 pri;
2541         u16 spr;
2542         u16 cur;
2543         u16 fbc;
2544 };
2545
2546 /*
2547  * For both WM_PIPE and WM_LP.
2548  * mem_value must be in 0.1us units.
2549  */
2550 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2551                               const struct intel_plane_state *plane_state,
2552                               u32 mem_value, bool is_lp)
2553 {
2554         u32 method1, method2;
2555         int cpp;
2556
2557         if (mem_value == 0)
2558                 return U32_MAX;
2559
2560         if (!intel_wm_plane_visible(crtc_state, plane_state))
2561                 return 0;
2562
2563         cpp = plane_state->hw.fb->format->cpp[0];
2564
2565         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2566
2567         if (!is_lp)
2568                 return method1;
2569
2570         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2571                                  crtc_state->hw.pipe_mode.crtc_htotal,
2572                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2573                                  cpp, mem_value);
2574
2575         return min(method1, method2);
2576 }
2577
2578 /*
2579  * For both WM_PIPE and WM_LP.
2580  * mem_value must be in 0.1us units.
2581  */
2582 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2583                               const struct intel_plane_state *plane_state,
2584                               u32 mem_value)
2585 {
2586         u32 method1, method2;
2587         int cpp;
2588
2589         if (mem_value == 0)
2590                 return U32_MAX;
2591
2592         if (!intel_wm_plane_visible(crtc_state, plane_state))
2593                 return 0;
2594
2595         cpp = plane_state->hw.fb->format->cpp[0];
2596
2597         method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2598         method2 = ilk_wm_method2(crtc_state->pixel_rate,
2599                                  crtc_state->hw.pipe_mode.crtc_htotal,
2600                                  drm_rect_width(&plane_state->uapi.src) >> 16,
2601                                  cpp, mem_value);
2602         return min(method1, method2);
2603 }
2604
2605 /*
2606  * For both WM_PIPE and WM_LP.
2607  * mem_value must be in 0.1us units.
2608  */
2609 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2610                               const struct intel_plane_state *plane_state,
2611                               u32 mem_value)
2612 {
2613         int cpp;
2614
2615         if (mem_value == 0)
2616                 return U32_MAX;
2617
2618         if (!intel_wm_plane_visible(crtc_state, plane_state))
2619                 return 0;
2620
2621         cpp = plane_state->hw.fb->format->cpp[0];
2622
2623         return ilk_wm_method2(crtc_state->pixel_rate,
2624                               crtc_state->hw.pipe_mode.crtc_htotal,
2625                               drm_rect_width(&plane_state->uapi.src) >> 16,
2626                               cpp, mem_value);
2627 }
2628
2629 /* Only for WM_LP. */
2630 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2631                               const struct intel_plane_state *plane_state,
2632                               u32 pri_val)
2633 {
2634         int cpp;
2635
2636         if (!intel_wm_plane_visible(crtc_state, plane_state))
2637                 return 0;
2638
2639         cpp = plane_state->hw.fb->format->cpp[0];
2640
2641         return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2642                           cpp);
2643 }
2644
2645 static unsigned int
2646 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2647 {
2648         if (DISPLAY_VER(dev_priv) >= 8)
2649                 return 3072;
2650         else if (DISPLAY_VER(dev_priv) >= 7)
2651                 return 768;
2652         else
2653                 return 512;
2654 }
2655
2656 static unsigned int
2657 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2658                      int level, bool is_sprite)
2659 {
2660         if (DISPLAY_VER(dev_priv) >= 8)
2661                 /* BDW primary/sprite plane watermarks */
2662                 return level == 0 ? 255 : 2047;
2663         else if (DISPLAY_VER(dev_priv) >= 7)
2664                 /* IVB/HSW primary/sprite plane watermarks */
2665                 return level == 0 ? 127 : 1023;
2666         else if (!is_sprite)
2667                 /* ILK/SNB primary plane watermarks */
2668                 return level == 0 ? 127 : 511;
2669         else
2670                 /* ILK/SNB sprite plane watermarks */
2671                 return level == 0 ? 63 : 255;
2672 }
2673
2674 static unsigned int
2675 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2676 {
2677         if (DISPLAY_VER(dev_priv) >= 7)
2678                 return level == 0 ? 63 : 255;
2679         else
2680                 return level == 0 ? 31 : 63;
2681 }
2682
2683 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2684 {
2685         if (DISPLAY_VER(dev_priv) >= 8)
2686                 return 31;
2687         else
2688                 return 15;
2689 }
2690
2691 /* Calculate the maximum primary/sprite plane watermark */
2692 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2693                                      int level,
2694                                      const struct intel_wm_config *config,
2695                                      enum intel_ddb_partitioning ddb_partitioning,
2696                                      bool is_sprite)
2697 {
2698         unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2699
2700         /* if sprites aren't enabled, sprites get nothing */
2701         if (is_sprite && !config->sprites_enabled)
2702                 return 0;
2703
2704         /* HSW allows LP1+ watermarks even with multiple pipes */
2705         if (level == 0 || config->num_pipes_active > 1) {
2706                 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2707
2708                 /*
2709                  * For some reason the non self refresh
2710                  * FIFO size is only half of the self
2711                  * refresh FIFO size on ILK/SNB.
2712                  */
2713                 if (DISPLAY_VER(dev_priv) <= 6)
2714                         fifo_size /= 2;
2715         }
2716
2717         if (config->sprites_enabled) {
2718                 /* level 0 is always calculated with 1:1 split */
2719                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2720                         if (is_sprite)
2721                                 fifo_size *= 5;
2722                         fifo_size /= 6;
2723                 } else {
2724                         fifo_size /= 2;
2725                 }
2726         }
2727
2728         /* clamp to max that the registers can hold */
2729         return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2730 }
2731
2732 /* Calculate the maximum cursor plane watermark */
2733 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2734                                       int level,
2735                                       const struct intel_wm_config *config)
2736 {
2737         /* HSW LP1+ watermarks w/ multiple pipes */
2738         if (level > 0 && config->num_pipes_active > 1)
2739                 return 64;
2740
2741         /* otherwise just report max that registers can hold */
2742         return ilk_cursor_wm_reg_max(dev_priv, level);
2743 }
2744
2745 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2746                                     int level,
2747                                     const struct intel_wm_config *config,
2748                                     enum intel_ddb_partitioning ddb_partitioning,
2749                                     struct ilk_wm_maximums *max)
2750 {
2751         max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2752         max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2753         max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2754         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2755 }
2756
2757 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2758                                         int level,
2759                                         struct ilk_wm_maximums *max)
2760 {
2761         max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2762         max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2763         max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2764         max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2765 }
2766
2767 static bool ilk_validate_wm_level(int level,
2768                                   const struct ilk_wm_maximums *max,
2769                                   struct intel_wm_level *result)
2770 {
2771         bool ret;
2772
2773         /* already determined to be invalid? */
2774         if (!result->enable)
2775                 return false;
2776
2777         result->enable = result->pri_val <= max->pri &&
2778                          result->spr_val <= max->spr &&
2779                          result->cur_val <= max->cur;
2780
2781         ret = result->enable;
2782
2783         /*
2784          * HACK until we can pre-compute everything,
2785          * and thus fail gracefully if LP0 watermarks
2786          * are exceeded...
2787          */
2788         if (level == 0 && !result->enable) {
2789                 if (result->pri_val > max->pri)
2790                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
2791                                       level, result->pri_val, max->pri);
2792                 if (result->spr_val > max->spr)
2793                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
2794                                       level, result->spr_val, max->spr);
2795                 if (result->cur_val > max->cur)
2796                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
2797                                       level, result->cur_val, max->cur);
2798
2799                 result->pri_val = min_t(u32, result->pri_val, max->pri);
2800                 result->spr_val = min_t(u32, result->spr_val, max->spr);
2801                 result->cur_val = min_t(u32, result->cur_val, max->cur);
2802                 result->enable = true;
2803         }
2804
2805         return ret;
2806 }
2807
2808 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2809                                  const struct intel_crtc *crtc,
2810                                  int level,
2811                                  struct intel_crtc_state *crtc_state,
2812                                  const struct intel_plane_state *pristate,
2813                                  const struct intel_plane_state *sprstate,
2814                                  const struct intel_plane_state *curstate,
2815                                  struct intel_wm_level *result)
2816 {
2817         u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2818         u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2819         u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2820
2821         /* WM1+ latency values stored in 0.5us units */
2822         if (level > 0) {
2823                 pri_latency *= 5;
2824                 spr_latency *= 5;
2825                 cur_latency *= 5;
2826         }
2827
2828         if (pristate) {
2829                 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2830                                                      pri_latency, level);
2831                 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2832         }
2833
2834         if (sprstate)
2835                 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2836
2837         if (curstate)
2838                 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2839
2840         result->enable = true;
2841 }
2842
2843 static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2844 {
2845         u64 sskpd;
2846
2847         sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2848
2849         wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2850         if (wm[0] == 0)
2851                 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2852         wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2853         wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2854         wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2855         wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2856 }
2857
2858 static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2859 {
2860         u32 sskpd;
2861
2862         sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2863
2864         wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2865         wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2866         wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2867         wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2868 }
2869
2870 static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2871 {
2872         u32 mltr;
2873
2874         mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2875
2876         /* ILK primary LP0 latency is 700 ns */
2877         wm[0] = 7;
2878         wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2879         wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2880 }
2881
2882 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2883                                        u16 wm[5])
2884 {
2885         /* ILK sprite LP0 latency is 1300 ns */
2886         if (DISPLAY_VER(dev_priv) == 5)
2887                 wm[0] = 13;
2888 }
2889
2890 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2891                                        u16 wm[5])
2892 {
2893         /* ILK cursor LP0 latency is 1300 ns */
2894         if (DISPLAY_VER(dev_priv) == 5)
2895                 wm[0] = 13;
2896 }
2897
2898 int ilk_wm_max_level(const struct drm_i915_private *dev_priv)
2899 {
2900         /* how many WM levels are we expecting */
2901         if (HAS_HW_SAGV_WM(dev_priv))
2902                 return 5;
2903         else if (DISPLAY_VER(dev_priv) >= 9)
2904                 return 7;
2905         else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
2906                 return 4;
2907         else if (DISPLAY_VER(dev_priv) >= 6)
2908                 return 3;
2909         else
2910                 return 2;
2911 }
2912
2913 void intel_print_wm_latency(struct drm_i915_private *dev_priv,
2914                             const char *name, const u16 wm[])
2915 {
2916         int level, max_level = ilk_wm_max_level(dev_priv);
2917
2918         for (level = 0; level <= max_level; level++) {
2919                 unsigned int latency = wm[level];
2920
2921                 if (latency == 0) {
2922                         drm_dbg_kms(&dev_priv->drm,
2923                                     "%s WM%d latency not provided\n",
2924                                     name, level);
2925                         continue;
2926                 }
2927
2928                 /*
2929                  * - latencies are in us on gen9.
2930                  * - before then, WM1+ latency values are in 0.5us units
2931                  */
2932                 if (DISPLAY_VER(dev_priv) >= 9)
2933                         latency *= 10;
2934                 else if (level > 0)
2935                         latency *= 5;
2936
2937                 drm_dbg_kms(&dev_priv->drm,
2938                             "%s WM%d latency %u (%u.%u usec)\n", name, level,
2939                             wm[level], latency / 10, latency % 10);
2940         }
2941 }
2942
2943 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2944                                     u16 wm[5], u16 min)
2945 {
2946         int level, max_level = ilk_wm_max_level(dev_priv);
2947
2948         if (wm[0] >= min)
2949                 return false;
2950
2951         wm[0] = max(wm[0], min);
2952         for (level = 1; level <= max_level; level++)
2953                 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2954
2955         return true;
2956 }
2957
2958 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2959 {
2960         bool changed;
2961
2962         /*
2963          * The BIOS provided WM memory latency values are often
2964          * inadequate for high resolution displays. Adjust them.
2965          */
2966         changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2967         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2968         changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2969
2970         if (!changed)
2971                 return;
2972
2973         drm_dbg_kms(&dev_priv->drm,
2974                     "WM latency values increased to avoid potential underruns\n");
2975         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2976         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2977         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2978 }
2979
2980 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2981 {
2982         /*
2983          * On some SNB machines (Thinkpad X220 Tablet at least)
2984          * LP3 usage can cause vblank interrupts to be lost.
2985          * The DEIIR bit will go high but it looks like the CPU
2986          * never gets interrupted.
2987          *
2988          * It's not clear whether other interrupt source could
2989          * be affected or if this is somehow limited to vblank
2990          * interrupts only. To play it safe we disable LP3
2991          * watermarks entirely.
2992          */
2993         if (dev_priv->display.wm.pri_latency[3] == 0 &&
2994             dev_priv->display.wm.spr_latency[3] == 0 &&
2995             dev_priv->display.wm.cur_latency[3] == 0)
2996                 return;
2997
2998         dev_priv->display.wm.pri_latency[3] = 0;
2999         dev_priv->display.wm.spr_latency[3] = 0;
3000         dev_priv->display.wm.cur_latency[3] = 0;
3001
3002         drm_dbg_kms(&dev_priv->drm,
3003                     "LP3 watermarks disabled due to potential for lost interrupts\n");
3004         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3005         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3006         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3007 }
3008
3009 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
3010 {
3011         if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
3012                 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3013         else if (DISPLAY_VER(dev_priv) >= 6)
3014                 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3015         else
3016                 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
3017
3018         memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
3019                sizeof(dev_priv->display.wm.pri_latency));
3020         memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
3021                sizeof(dev_priv->display.wm.pri_latency));
3022
3023         intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
3024         intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
3025
3026         intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
3027         intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
3028         intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
3029
3030         if (DISPLAY_VER(dev_priv) == 6) {
3031                 snb_wm_latency_quirk(dev_priv);
3032                 snb_wm_lp3_irq_quirk(dev_priv);
3033         }
3034 }
3035
3036 static bool ilk_validate_pipe_wm(const struct drm_i915_private *dev_priv,
3037                                  struct intel_pipe_wm *pipe_wm)
3038 {
3039         /* LP0 watermark maximums depend on this pipe alone */
3040         const struct intel_wm_config config = {
3041                 .num_pipes_active = 1,
3042                 .sprites_enabled = pipe_wm->sprites_enabled,
3043                 .sprites_scaled = pipe_wm->sprites_scaled,
3044         };
3045         struct ilk_wm_maximums max;
3046
3047         /* LP0 watermarks always use 1/2 DDB partitioning */
3048         ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
3049
3050         /* At least LP0 must be valid */
3051         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0])) {
3052                 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
3053                 return false;
3054         }
3055
3056         return true;
3057 }
3058
3059 /* Compute new watermarks for the pipe */
3060 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
3061                                struct intel_crtc *crtc)
3062 {
3063         struct drm_i915_private *dev_priv = to_i915(state->base.dev);
3064         struct intel_crtc_state *crtc_state =
3065                 intel_atomic_get_new_crtc_state(state, crtc);
3066         struct intel_pipe_wm *pipe_wm;
3067         struct intel_plane *plane;
3068         const struct intel_plane_state *plane_state;
3069         const struct intel_plane_state *pristate = NULL;
3070         const struct intel_plane_state *sprstate = NULL;
3071         const struct intel_plane_state *curstate = NULL;
3072         int level, max_level = ilk_wm_max_level(dev_priv), usable_level;
3073         struct ilk_wm_maximums max;
3074
3075         pipe_wm = &crtc_state->wm.ilk.optimal;
3076
3077         intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
3078                 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
3079                         pristate = plane_state;
3080                 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
3081                         sprstate = plane_state;
3082                 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
3083                         curstate = plane_state;
3084         }
3085
3086         pipe_wm->pipe_enabled = crtc_state->hw.active;
3087         pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
3088         pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
3089
3090         usable_level = max_level;
3091
3092         /* ILK/SNB: LP2+ watermarks only w/o sprites */
3093         if (DISPLAY_VER(dev_priv) <= 6 && pipe_wm->sprites_enabled)
3094                 usable_level = 1;
3095
3096         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
3097         if (pipe_wm->sprites_scaled)
3098                 usable_level = 0;
3099
3100         memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
3101         ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
3102                              pristate, sprstate, curstate, &pipe_wm->wm[0]);
3103
3104         if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
3105                 return -EINVAL;
3106
3107         ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
3108
3109         for (level = 1; level <= usable_level; level++) {
3110                 struct intel_wm_level *wm = &pipe_wm->wm[level];
3111
3112                 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
3113                                      pristate, sprstate, curstate, wm);
3114
3115                 /*
3116                  * Disable any watermark level that exceeds the
3117                  * register maximums since such watermarks are
3118                  * always invalid.
3119                  */
3120                 if (!ilk_validate_wm_level(level, &max, wm)) {
3121                         memset(wm, 0, sizeof(*wm));
3122                         break;
3123                 }
3124         }
3125
3126         return 0;
3127 }
3128
3129 /*
3130  * Build a set of 'intermediate' watermark values that satisfy both the old
3131  * state and the new state.  These can be programmed to the hardware
3132  * immediately.
3133  */
3134 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
3135                                        struct intel_crtc *crtc)
3136 {
3137         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3138         struct intel_crtc_state *new_crtc_state =
3139                 intel_atomic_get_new_crtc_state(state, crtc);
3140         const struct intel_crtc_state *old_crtc_state =
3141                 intel_atomic_get_old_crtc_state(state, crtc);
3142         struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
3143         const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
3144         int level, max_level = ilk_wm_max_level(dev_priv);
3145
3146         /*
3147          * Start with the final, target watermarks, then combine with the
3148          * currently active watermarks to get values that are safe both before
3149          * and after the vblank.
3150          */
3151         *a = new_crtc_state->wm.ilk.optimal;
3152         if (!new_crtc_state->hw.active ||
3153             drm_atomic_crtc_needs_modeset(&new_crtc_state->uapi) ||
3154             state->skip_intermediate_wm)
3155                 return 0;
3156
3157         a->pipe_enabled |= b->pipe_enabled;
3158         a->sprites_enabled |= b->sprites_enabled;
3159         a->sprites_scaled |= b->sprites_scaled;
3160
3161         for (level = 0; level <= max_level; level++) {
3162                 struct intel_wm_level *a_wm = &a->wm[level];
3163                 const struct intel_wm_level *b_wm = &b->wm[level];
3164
3165                 a_wm->enable &= b_wm->enable;
3166                 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
3167                 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
3168                 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
3169                 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
3170         }
3171
3172         /*
3173          * We need to make sure that these merged watermark values are
3174          * actually a valid configuration themselves.  If they're not,
3175          * there's no safe way to transition from the old state to
3176          * the new state, so we need to fail the atomic transaction.
3177          */
3178         if (!ilk_validate_pipe_wm(dev_priv, a))
3179                 return -EINVAL;
3180
3181         /*
3182          * If our intermediate WM are identical to the final WM, then we can
3183          * omit the post-vblank programming; only update if it's different.
3184          */
3185         if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
3186                 new_crtc_state->wm.need_postvbl_update = true;
3187
3188         return 0;
3189 }
3190
3191 /*
3192  * Merge the watermarks from all active pipes for a specific level.
3193  */
3194 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
3195                                int level,
3196                                struct intel_wm_level *ret_wm)
3197 {
3198         const struct intel_crtc *crtc;
3199
3200         ret_wm->enable = true;
3201
3202         for_each_intel_crtc(&dev_priv->drm, crtc) {
3203                 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
3204                 const struct intel_wm_level *wm = &active->wm[level];
3205
3206                 if (!active->pipe_enabled)
3207                         continue;
3208
3209                 /*
3210                  * The watermark values may have been used in the past,
3211                  * so we must maintain them in the registers for some
3212                  * time even if the level is now disabled.
3213                  */
3214                 if (!wm->enable)
3215                         ret_wm->enable = false;
3216
3217                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
3218                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
3219                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
3220                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
3221         }
3222 }
3223
3224 /*
3225  * Merge all low power watermarks for all active pipes.
3226  */
3227 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
3228                          const struct intel_wm_config *config,
3229                          const struct ilk_wm_maximums *max,
3230                          struct intel_pipe_wm *merged)
3231 {
3232         int level, max_level = ilk_wm_max_level(dev_priv);
3233         int last_enabled_level = max_level;
3234
3235         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
3236         if ((DISPLAY_VER(dev_priv) <= 6 || IS_IVYBRIDGE(dev_priv)) &&
3237             config->num_pipes_active > 1)
3238                 last_enabled_level = 0;
3239
3240         /* ILK: FBC WM must be disabled always */
3241         merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
3242
3243         /* merge each WM1+ level */
3244         for (level = 1; level <= max_level; level++) {
3245                 struct intel_wm_level *wm = &merged->wm[level];
3246
3247                 ilk_merge_wm_level(dev_priv, level, wm);
3248
3249                 if (level > last_enabled_level)
3250                         wm->enable = false;
3251                 else if (!ilk_validate_wm_level(level, max, wm))
3252                         /* make sure all following levels get disabled */
3253                         last_enabled_level = level - 1;
3254
3255                 /*
3256                  * The spec says it is preferred to disable
3257                  * FBC WMs instead of disabling a WM level.
3258                  */
3259                 if (wm->fbc_val > max->fbc) {
3260                         if (wm->enable)
3261                                 merged->fbc_wm_enabled = false;
3262                         wm->fbc_val = 0;
3263                 }
3264         }
3265
3266         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3267         if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3268             dev_priv->params.enable_fbc && !merged->fbc_wm_enabled) {
3269                 for (level = 2; level <= max_level; level++) {
3270                         struct intel_wm_level *wm = &merged->wm[level];
3271
3272                         wm->enable = false;
3273                 }
3274         }
3275 }
3276
3277 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3278 {
3279         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3280         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3281 }
3282
3283 /* The value we need to program into the WM_LPx latency field */
3284 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3285                                       int level)
3286 {
3287         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3288                 return 2 * level;
3289         else
3290                 return dev_priv->display.wm.pri_latency[level];
3291 }
3292
3293 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3294                                    const struct intel_pipe_wm *merged,
3295                                    enum intel_ddb_partitioning partitioning,
3296                                    struct ilk_wm_values *results)
3297 {
3298         struct intel_crtc *crtc;
3299         int level, wm_lp;
3300
3301         results->enable_fbc_wm = merged->fbc_wm_enabled;
3302         results->partitioning = partitioning;
3303
3304         /* LP1+ register values */
3305         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3306                 const struct intel_wm_level *r;
3307
3308                 level = ilk_wm_lp_to_level(wm_lp, merged);
3309
3310                 r = &merged->wm[level];
3311
3312                 /*
3313                  * Maintain the watermark values even if the level is
3314                  * disabled. Doing otherwise could cause underruns.
3315                  */
3316                 results->wm_lp[wm_lp - 1] =
3317                         WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3318                         WM_LP_PRIMARY(r->pri_val) |
3319                         WM_LP_CURSOR(r->cur_val);
3320
3321                 if (r->enable)
3322                         results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3323
3324                 if (DISPLAY_VER(dev_priv) >= 8)
3325                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3326                 else
3327                         results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3328
3329                 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3330
3331                 /*
3332                  * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3333                  * level is disabled. Doing otherwise could cause underruns.
3334                  */
3335                 if (DISPLAY_VER(dev_priv) <= 6 && r->spr_val) {
3336                         drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3337                         results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3338                 }
3339         }
3340
3341         /* LP0 register values */
3342         for_each_intel_crtc(&dev_priv->drm, crtc) {
3343                 enum pipe pipe = crtc->pipe;
3344                 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3345                 const struct intel_wm_level *r = &pipe_wm->wm[0];
3346
3347                 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3348                         continue;
3349
3350                 results->wm_pipe[pipe] =
3351                         WM0_PIPE_PRIMARY(r->pri_val) |
3352                         WM0_PIPE_SPRITE(r->spr_val) |
3353                         WM0_PIPE_CURSOR(r->cur_val);
3354         }
3355 }
3356
3357 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
3358  * case both are at the same level. Prefer r1 in case they're the same. */
3359 static struct intel_pipe_wm *
3360 ilk_find_best_result(struct drm_i915_private *dev_priv,
3361                      struct intel_pipe_wm *r1,
3362                      struct intel_pipe_wm *r2)
3363 {
3364         int level, max_level = ilk_wm_max_level(dev_priv);
3365         int level1 = 0, level2 = 0;
3366
3367         for (level = 1; level <= max_level; level++) {
3368                 if (r1->wm[level].enable)
3369                         level1 = level;
3370                 if (r2->wm[level].enable)
3371                         level2 = level;
3372         }
3373
3374         if (level1 == level2) {
3375                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3376                         return r2;
3377                 else
3378                         return r1;
3379         } else if (level1 > level2) {
3380                 return r1;
3381         } else {
3382                 return r2;
3383         }
3384 }
3385
3386 /* dirty bits used to track which watermarks need changes */
3387 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3388 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3389 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3390 #define WM_DIRTY_FBC (1 << 24)
3391 #define WM_DIRTY_DDB (1 << 25)
3392
3393 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3394                                          const struct ilk_wm_values *old,
3395                                          const struct ilk_wm_values *new)
3396 {
3397         unsigned int dirty = 0;
3398         enum pipe pipe;
3399         int wm_lp;
3400
3401         for_each_pipe(dev_priv, pipe) {
3402                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3403                         dirty |= WM_DIRTY_PIPE(pipe);
3404                         /* Must disable LP1+ watermarks too */
3405                         dirty |= WM_DIRTY_LP_ALL;
3406                 }
3407         }
3408
3409         if (old->enable_fbc_wm != new->enable_fbc_wm) {
3410                 dirty |= WM_DIRTY_FBC;
3411                 /* Must disable LP1+ watermarks too */
3412                 dirty |= WM_DIRTY_LP_ALL;
3413         }
3414
3415         if (old->partitioning != new->partitioning) {
3416                 dirty |= WM_DIRTY_DDB;
3417                 /* Must disable LP1+ watermarks too */
3418                 dirty |= WM_DIRTY_LP_ALL;
3419         }
3420
3421         /* LP1+ watermarks already deemed dirty, no need to continue */
3422         if (dirty & WM_DIRTY_LP_ALL)
3423                 return dirty;
3424
3425         /* Find the lowest numbered LP1+ watermark in need of an update... */
3426         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3427                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3428                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3429                         break;
3430         }
3431
3432         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3433         for (; wm_lp <= 3; wm_lp++)
3434                 dirty |= WM_DIRTY_LP(wm_lp);
3435
3436         return dirty;
3437 }
3438
3439 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3440                                unsigned int dirty)
3441 {
3442         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3443         bool changed = false;
3444
3445         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3446                 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3447                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3448                 changed = true;
3449         }
3450         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3451                 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3452                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3453                 changed = true;
3454         }
3455         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3456                 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3457                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3458                 changed = true;
3459         }
3460
3461         /*
3462          * Don't touch WM_LP_SPRITE_ENABLE here.
3463          * Doing so could cause underruns.
3464          */
3465
3466         return changed;
3467 }
3468
3469 /*
3470  * The spec says we shouldn't write when we don't need, because every write
3471  * causes WMs to be re-evaluated, expending some power.
3472  */
3473 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3474                                 struct ilk_wm_values *results)
3475 {
3476         struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3477         unsigned int dirty;
3478
3479         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3480         if (!dirty)
3481                 return;
3482
3483         _ilk_disable_lp_wm(dev_priv, dirty);
3484
3485         if (dirty & WM_DIRTY_PIPE(PIPE_A))
3486                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3487         if (dirty & WM_DIRTY_PIPE(PIPE_B))
3488                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3489         if (dirty & WM_DIRTY_PIPE(PIPE_C))
3490                 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3491
3492         if (dirty & WM_DIRTY_DDB) {
3493                 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3494                         intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3495                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3496                                          WM_MISC_DATA_PARTITION_5_6);
3497                 else
3498                         intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3499                                          results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3500                                          DISP_DATA_PARTITION_5_6);
3501         }
3502
3503         if (dirty & WM_DIRTY_FBC)
3504                 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3505                                  results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3506
3507         if (dirty & WM_DIRTY_LP(1) &&
3508             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3509                 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3510
3511         if (DISPLAY_VER(dev_priv) >= 7) {
3512                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3513                         intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3514                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3515                         intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3516         }
3517
3518         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3519                 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3520         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3521                 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3522         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3523                 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3524
3525         dev_priv->display.wm.hw = *results;
3526 }
3527
3528 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3529 {
3530         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3531 }
3532
3533 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3534                                   struct intel_wm_config *config)
3535 {
3536         struct intel_crtc *crtc;
3537
3538         /* Compute the currently _active_ config */
3539         for_each_intel_crtc(&dev_priv->drm, crtc) {
3540                 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3541
3542                 if (!wm->pipe_enabled)
3543                         continue;
3544
3545                 config->sprites_enabled |= wm->sprites_enabled;
3546                 config->sprites_scaled |= wm->sprites_scaled;
3547                 config->num_pipes_active++;
3548         }
3549 }
3550
3551 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3552 {
3553         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3554         struct ilk_wm_maximums max;
3555         struct intel_wm_config config = {};
3556         struct ilk_wm_values results = {};
3557         enum intel_ddb_partitioning partitioning;
3558
3559         ilk_compute_wm_config(dev_priv, &config);
3560
3561         ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3562         ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3563
3564         /* 5/6 split only in single pipe config on IVB+ */
3565         if (DISPLAY_VER(dev_priv) >= 7 &&
3566             config.num_pipes_active == 1 && config.sprites_enabled) {
3567                 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3568                 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3569
3570                 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3571         } else {
3572                 best_lp_wm = &lp_wm_1_2;
3573         }
3574
3575         partitioning = (best_lp_wm == &lp_wm_1_2) ?
3576                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3577
3578         ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3579
3580         ilk_write_wm_values(dev_priv, &results);
3581 }
3582
3583 static void ilk_initial_watermarks(struct intel_atomic_state *state,
3584                                    struct intel_crtc *crtc)
3585 {
3586         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3587         const struct intel_crtc_state *crtc_state =
3588                 intel_atomic_get_new_crtc_state(state, crtc);
3589
3590         mutex_lock(&dev_priv->display.wm.wm_mutex);
3591         crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3592         ilk_program_watermarks(dev_priv);
3593         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3594 }
3595
3596 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3597                                     struct intel_crtc *crtc)
3598 {
3599         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3600         const struct intel_crtc_state *crtc_state =
3601                 intel_atomic_get_new_crtc_state(state, crtc);
3602
3603         if (!crtc_state->wm.need_postvbl_update)
3604                 return;
3605
3606         mutex_lock(&dev_priv->display.wm.wm_mutex);
3607         crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3608         ilk_program_watermarks(dev_priv);
3609         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3610 }
3611
3612 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3613 {
3614         struct drm_device *dev = crtc->base.dev;
3615         struct drm_i915_private *dev_priv = to_i915(dev);
3616         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3617         struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3618         struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3619         enum pipe pipe = crtc->pipe;
3620
3621         hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3622
3623         memset(active, 0, sizeof(*active));
3624
3625         active->pipe_enabled = crtc->active;
3626
3627         if (active->pipe_enabled) {
3628                 u32 tmp = hw->wm_pipe[pipe];
3629
3630                 /*
3631                  * For active pipes LP0 watermark is marked as
3632                  * enabled, and LP1+ watermaks as disabled since
3633                  * we can't really reverse compute them in case
3634                  * multiple pipes are active.
3635                  */
3636                 active->wm[0].enable = true;
3637                 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3638                 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3639                 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3640         } else {
3641                 int level, max_level = ilk_wm_max_level(dev_priv);
3642
3643                 /*
3644                  * For inactive pipes, all watermark levels
3645                  * should be marked as enabled but zeroed,
3646                  * which is what we'd compute them to.
3647                  */
3648                 for (level = 0; level <= max_level; level++)
3649                         active->wm[level].enable = true;
3650         }
3651
3652         crtc->wm.active.ilk = *active;
3653 }
3654
3655 #define _FW_WM(value, plane) \
3656         (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3657 #define _FW_WM_VLV(value, plane) \
3658         (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3659
3660 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3661                                struct g4x_wm_values *wm)
3662 {
3663         u32 tmp;
3664
3665         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3666         wm->sr.plane = _FW_WM(tmp, SR);
3667         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3668         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3669         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3670
3671         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3672         wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3673         wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3674         wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3675         wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3676         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3677         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3678
3679         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3680         wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3681         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3682         wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3683         wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3684 }
3685
3686 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3687                                struct vlv_wm_values *wm)
3688 {
3689         enum pipe pipe;
3690         u32 tmp;
3691
3692         for_each_pipe(dev_priv, pipe) {
3693                 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3694
3695                 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3696                         (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3697                 wm->ddl[pipe].plane[PLANE_CURSOR] =
3698                         (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3699                 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3700                         (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3701                 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3702                         (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3703         }
3704
3705         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1);
3706         wm->sr.plane = _FW_WM(tmp, SR);
3707         wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3708         wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3709         wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3710
3711         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2);
3712         wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3713         wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3714         wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3715
3716         tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3);
3717         wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3718
3719         if (IS_CHERRYVIEW(dev_priv)) {
3720                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3721                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3722                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3723
3724                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3725                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3726                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3727
3728                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3729                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3730                 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3731
3732                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3733                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3734                 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3735                 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3736                 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3737                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3738                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3739                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3740                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3741                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3742                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3743         } else {
3744                 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3745                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3746                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3747
3748                 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3749                 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3750                 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3751                 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3752                 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3753                 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3754                 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3755                 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3756         }
3757 }
3758
3759 #undef _FW_WM
3760 #undef _FW_WM_VLV
3761
3762 void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3763 {
3764         struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3765         struct intel_crtc *crtc;
3766
3767         g4x_read_wm_values(dev_priv, wm);
3768
3769         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3770
3771         for_each_intel_crtc(&dev_priv->drm, crtc) {
3772                 struct intel_crtc_state *crtc_state =
3773                         to_intel_crtc_state(crtc->base.state);
3774                 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3775                 struct g4x_pipe_wm *raw;
3776                 enum pipe pipe = crtc->pipe;
3777                 enum plane_id plane_id;
3778                 int level, max_level;
3779
3780                 active->cxsr = wm->cxsr;
3781                 active->hpll_en = wm->hpll_en;
3782                 active->fbc_en = wm->fbc_en;
3783
3784                 active->sr = wm->sr;
3785                 active->hpll = wm->hpll;
3786
3787                 for_each_plane_id_on_crtc(crtc, plane_id) {
3788                         active->wm.plane[plane_id] =
3789                                 wm->pipe[pipe].plane[plane_id];
3790                 }
3791
3792                 if (wm->cxsr && wm->hpll_en)
3793                         max_level = G4X_WM_LEVEL_HPLL;
3794                 else if (wm->cxsr)
3795                         max_level = G4X_WM_LEVEL_SR;
3796                 else
3797                         max_level = G4X_WM_LEVEL_NORMAL;
3798
3799                 level = G4X_WM_LEVEL_NORMAL;
3800                 raw = &crtc_state->wm.g4x.raw[level];
3801                 for_each_plane_id_on_crtc(crtc, plane_id)
3802                         raw->plane[plane_id] = active->wm.plane[plane_id];
3803
3804                 level = G4X_WM_LEVEL_SR;
3805                 if (level > max_level)
3806                         goto out;
3807
3808                 raw = &crtc_state->wm.g4x.raw[level];
3809                 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3810                 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3811                 raw->plane[PLANE_SPRITE0] = 0;
3812                 raw->fbc = active->sr.fbc;
3813
3814                 level = G4X_WM_LEVEL_HPLL;
3815                 if (level > max_level)
3816                         goto out;
3817
3818                 raw = &crtc_state->wm.g4x.raw[level];
3819                 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3820                 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3821                 raw->plane[PLANE_SPRITE0] = 0;
3822                 raw->fbc = active->hpll.fbc;
3823
3824                 level++;
3825         out:
3826                 for_each_plane_id_on_crtc(crtc, plane_id)
3827                         g4x_raw_plane_wm_set(crtc_state, level,
3828                                              plane_id, USHRT_MAX);
3829                 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3830
3831                 g4x_invalidate_wms(crtc, active, level);
3832
3833                 crtc_state->wm.g4x.optimal = *active;
3834                 crtc_state->wm.g4x.intermediate = *active;
3835
3836                 drm_dbg_kms(&dev_priv->drm,
3837                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3838                             pipe_name(pipe),
3839                             wm->pipe[pipe].plane[PLANE_PRIMARY],
3840                             wm->pipe[pipe].plane[PLANE_CURSOR],
3841                             wm->pipe[pipe].plane[PLANE_SPRITE0]);
3842         }
3843
3844         drm_dbg_kms(&dev_priv->drm,
3845                     "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3846                     wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3847         drm_dbg_kms(&dev_priv->drm,
3848                     "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3849                     wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3850         drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3851                     str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3852                     str_yes_no(wm->fbc_en));
3853 }
3854
3855 void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3856 {
3857         struct intel_plane *plane;
3858         struct intel_crtc *crtc;
3859
3860         mutex_lock(&dev_priv->display.wm.wm_mutex);
3861
3862         for_each_intel_plane(&dev_priv->drm, plane) {
3863                 struct intel_crtc *crtc =
3864                         intel_crtc_for_pipe(dev_priv, plane->pipe);
3865                 struct intel_crtc_state *crtc_state =
3866                         to_intel_crtc_state(crtc->base.state);
3867                 struct intel_plane_state *plane_state =
3868                         to_intel_plane_state(plane->base.state);
3869                 enum plane_id plane_id = plane->id;
3870                 int level, num_levels = intel_wm_num_levels(dev_priv);
3871
3872                 if (plane_state->uapi.visible)
3873                         continue;
3874
3875                 for (level = 0; level < num_levels; level++) {
3876                         struct g4x_pipe_wm *raw =
3877                                 &crtc_state->wm.g4x.raw[level];
3878
3879                         raw->plane[plane_id] = 0;
3880
3881                         if (plane_id == PLANE_PRIMARY)
3882                                 raw->fbc = 0;
3883                 }
3884         }
3885
3886         for_each_intel_crtc(&dev_priv->drm, crtc) {
3887                 struct intel_crtc_state *crtc_state =
3888                         to_intel_crtc_state(crtc->base.state);
3889                 int ret;
3890
3891                 ret = _g4x_compute_pipe_wm(crtc_state);
3892                 drm_WARN_ON(&dev_priv->drm, ret);
3893
3894                 crtc_state->wm.g4x.intermediate =
3895                         crtc_state->wm.g4x.optimal;
3896                 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3897         }
3898
3899         g4x_program_watermarks(dev_priv);
3900
3901         mutex_unlock(&dev_priv->display.wm.wm_mutex);
3902 }
3903
3904 void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3905 {
3906         struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3907         struct intel_crtc *crtc;
3908         u32 val;
3909
3910         vlv_read_wm_values(dev_priv, wm);
3911
3912         wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3913         wm->level = VLV_WM_LEVEL_PM2;
3914
3915         if (IS_CHERRYVIEW(dev_priv)) {
3916                 vlv_punit_get(dev_priv);
3917
3918                 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3919                 if (val & DSP_MAXFIFO_PM5_ENABLE)
3920                         wm->level = VLV_WM_LEVEL_PM5;
3921
3922                 /*
3923                  * If DDR DVFS is disabled in the BIOS, Punit
3924                  * will never ack the request. So if that happens
3925                  * assume we don't have to enable/disable DDR DVFS
3926                  * dynamically. To test that just set the REQ_ACK
3927                  * bit to poke the Punit, but don't change the
3928                  * HIGH/LOW bits so that we don't actually change
3929                  * the current state.
3930                  */
3931                 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3932                 val |= FORCE_DDR_FREQ_REQ_ACK;
3933                 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3934
3935                 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3936                               FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3937                         drm_dbg_kms(&dev_priv->drm,
3938                                     "Punit not acking DDR DVFS request, "
3939                                     "assuming DDR DVFS is disabled\n");
3940                         dev_priv->display.wm.max_level = VLV_WM_LEVEL_PM5;
3941                 } else {
3942                         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3943                         if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3944                                 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3945                 }
3946
3947                 vlv_punit_put(dev_priv);
3948         }
3949
3950         for_each_intel_crtc(&dev_priv->drm, crtc) {
3951                 struct intel_crtc_state *crtc_state =
3952                         to_intel_crtc_state(crtc->base.state);
3953                 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3954                 const struct vlv_fifo_state *fifo_state =
3955                         &crtc_state->wm.vlv.fifo_state;
3956                 enum pipe pipe = crtc->pipe;
3957                 enum plane_id plane_id;
3958                 int level;
3959
3960                 vlv_get_fifo_size(crtc_state);
3961
3962                 active->num_levels = wm->level + 1;
3963                 active->cxsr = wm->cxsr;
3964
3965                 for (level = 0; level < active->num_levels; level++) {
3966                         struct g4x_pipe_wm *raw =
3967                                 &crtc_state->wm.vlv.raw[level];
3968
3969                         active->sr[level].plane = wm->sr.plane;
3970                         active->sr[level].cursor = wm->sr.cursor;
3971
3972                         for_each_plane_id_on_crtc(crtc, plane_id) {
3973                                 active->wm[level].plane[plane_id] =
3974                                         wm->pipe[pipe].plane[plane_id];
3975
3976                                 raw->plane[plane_id] =
3977                                         vlv_invert_wm_value(active->wm[level].plane[plane_id],
3978                                                             fifo_state->plane[plane_id]);
3979                         }
3980                 }
3981
3982                 for_each_plane_id_on_crtc(crtc, plane_id)
3983                         vlv_raw_plane_wm_set(crtc_state, level,
3984                                              plane_id, USHRT_MAX);
3985                 vlv_invalidate_wms(crtc, active, level);
3986
3987                 crtc_state->wm.vlv.optimal = *active;
3988                 crtc_state->wm.vlv.intermediate = *active;
3989
3990                 drm_dbg_kms(&dev_priv->drm,
3991                             "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3992                             pipe_name(pipe),
3993                             wm->pipe[pipe].plane[PLANE_PRIMARY],
3994                             wm->pipe[pipe].plane[PLANE_CURSOR],
3995                             wm->pipe[pipe].plane[PLANE_SPRITE0],
3996                             wm->pipe[pipe].plane[PLANE_SPRITE1]);
3997         }
3998
3999         drm_dbg_kms(&dev_priv->drm,
4000                     "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
4001                     wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
4002 }
4003
4004 void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
4005 {
4006         struct intel_plane *plane;
4007         struct intel_crtc *crtc;
4008
4009         mutex_lock(&dev_priv->display.wm.wm_mutex);
4010
4011         for_each_intel_plane(&dev_priv->drm, plane) {
4012                 struct intel_crtc *crtc =
4013                         intel_crtc_for_pipe(dev_priv, plane->pipe);
4014                 struct intel_crtc_state *crtc_state =
4015                         to_intel_crtc_state(crtc->base.state);
4016                 struct intel_plane_state *plane_state =
4017                         to_intel_plane_state(plane->base.state);
4018                 enum plane_id plane_id = plane->id;
4019                 int level, num_levels = intel_wm_num_levels(dev_priv);
4020
4021                 if (plane_state->uapi.visible)
4022                         continue;
4023
4024                 for (level = 0; level < num_levels; level++) {
4025                         struct g4x_pipe_wm *raw =
4026                                 &crtc_state->wm.vlv.raw[level];
4027
4028                         raw->plane[plane_id] = 0;
4029                 }
4030         }
4031
4032         for_each_intel_crtc(&dev_priv->drm, crtc) {
4033                 struct intel_crtc_state *crtc_state =
4034                         to_intel_crtc_state(crtc->base.state);
4035                 int ret;
4036
4037                 ret = _vlv_compute_pipe_wm(crtc_state);
4038                 drm_WARN_ON(&dev_priv->drm, ret);
4039
4040                 crtc_state->wm.vlv.intermediate =
4041                         crtc_state->wm.vlv.optimal;
4042                 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
4043         }
4044
4045         vlv_program_watermarks(dev_priv);
4046
4047         mutex_unlock(&dev_priv->display.wm.wm_mutex);
4048 }
4049
4050 /*
4051  * FIXME should probably kill this and improve
4052  * the real watermark readout/sanitation instead
4053  */
4054 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
4055 {
4056         intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM_LP_ENABLE);
4057         intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM_LP_ENABLE);
4058         intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM_LP_ENABLE);
4059
4060         /*
4061          * Don't touch WM_LP_SPRITE_ENABLE here.
4062          * Doing so could cause underruns.
4063          */
4064 }
4065
4066 void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
4067 {
4068         struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
4069         struct intel_crtc *crtc;
4070
4071         ilk_init_lp_watermarks(dev_priv);
4072
4073         for_each_intel_crtc(&dev_priv->drm, crtc)
4074                 ilk_pipe_wm_get_hw_state(crtc);
4075
4076         hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
4077         hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
4078         hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
4079
4080         hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
4081         if (DISPLAY_VER(dev_priv) >= 7) {
4082                 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
4083                 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
4084         }
4085
4086         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
4087                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
4088                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4089         else if (IS_IVYBRIDGE(dev_priv))
4090                 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
4091                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
4092
4093         hw->enable_fbc_wm =
4094                 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
4095 }
4096
4097 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
4098 {
4099         /*
4100          * On Ibex Peak and Cougar Point, we need to disable clock
4101          * gating for the panel power sequencer or it will fail to
4102          * start up when no ports are active.
4103          */
4104         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
4105 }
4106
4107 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
4108 {
4109         enum pipe pipe;
4110
4111         for_each_pipe(dev_priv, pipe) {
4112                 intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
4113                            intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
4114                            DISP_TRICKLE_FEED_DISABLE);
4115
4116                 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
4117                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
4118         }
4119 }
4120
4121 static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
4122 {
4123         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4124
4125         /*
4126          * Required for FBC
4127          * WaFbcDisableDpfcClockGating:ilk
4128          */
4129         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
4130                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
4131                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
4132
4133         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS0,
4134                    MARIUNIT_CLOCK_GATE_DISABLE |
4135                    SVSMUNIT_CLOCK_GATE_DISABLE);
4136         intel_uncore_write(&dev_priv->uncore, PCH_3DCGDIS1,
4137                    VFMUNIT_CLOCK_GATE_DISABLE);
4138
4139         /*
4140          * According to the spec the following bits should be set in
4141          * order to enable memory self-refresh
4142          * The bit 22/21 of 0x42004
4143          * The bit 5 of 0x42020
4144          * The bit 15 of 0x45000
4145          */
4146         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4147                    (intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4148                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
4149         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
4150         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL,
4151                    (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4152                     DISP_FBC_WM_DIS));
4153
4154         /*
4155          * Based on the document from hardware guys the following bits
4156          * should be set unconditionally in order to enable FBC.
4157          * The bit 22 of 0x42000
4158          * The bit 22 of 0x42004
4159          * The bit 7,8,9 of 0x42020.
4160          */
4161         if (IS_IRONLAKE_M(dev_priv)) {
4162                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
4163                 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
4164                            intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
4165                            ILK_FBCQ_DIS);
4166                 intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4167                            intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4168                            ILK_DPARB_GATE);
4169         }
4170
4171         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4172
4173         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4174                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4175                    ILK_ELPIN_409_SELECT);
4176
4177         g4x_disable_trickle_feed(dev_priv);
4178
4179         ibx_init_clock_gating(dev_priv);
4180 }
4181
4182 static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
4183 {
4184         enum pipe pipe;
4185         u32 val;
4186
4187         /*
4188          * On Ibex Peak and Cougar Point, we need to disable clock
4189          * gating for the panel power sequencer or it will fail to
4190          * start up when no ports are active.
4191          */
4192         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
4193                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
4194                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
4195         intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
4196                    DPLS_EDP_PPS_FIX_DIS);
4197         /* The below fixes the weird display corruption, a few pixels shifted
4198          * downward, on (only) LVDS of some HP laptops with IVY.
4199          */
4200         for_each_pipe(dev_priv, pipe) {
4201                 val = intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN2(pipe));
4202                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
4203                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4204                 if (dev_priv->display.vbt.fdi_rx_polarity_inverted)
4205                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
4206                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
4207                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
4208                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN2(pipe), val);
4209         }
4210         /* WADP0ClockGatingDisable */
4211         for_each_pipe(dev_priv, pipe) {
4212                 intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(pipe),
4213                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4214         }
4215 }
4216
4217 static void gen6_check_mch_setup(struct drm_i915_private *dev_priv)
4218 {
4219         u32 tmp;
4220
4221         tmp = intel_uncore_read(&dev_priv->uncore, MCH_SSKPD);
4222         if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
4223                 drm_dbg_kms(&dev_priv->drm,
4224                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
4225                             tmp);
4226 }
4227
4228 static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
4229 {
4230         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
4231
4232         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
4233
4234         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4235                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4236                    ILK_ELPIN_409_SELECT);
4237
4238         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
4239                    intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
4240                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
4241                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4242
4243         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
4244          * gating disable must be set.  Failure to set it results in
4245          * flickering pixels due to Z write ordering failures after
4246          * some amount of runtime in the Mesa "fire" demo, and Unigine
4247          * Sanctuary and Tropics, and apparently anything else with
4248          * alpha test or pixel discard.
4249          *
4250          * According to the spec, bit 11 (RCCUNIT) must also be set,
4251          * but we didn't debug actual testcases to find it out.
4252          *
4253          * WaDisableRCCUnitClockGating:snb
4254          * WaDisableRCPBUnitClockGating:snb
4255          */
4256         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4257                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
4258                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
4259
4260         /*
4261          * According to the spec the following bits should be
4262          * set in order to enable memory self-refresh and fbc:
4263          * The bit21 and bit22 of 0x42000
4264          * The bit21 and bit22 of 0x42004
4265          * The bit5 and bit7 of 0x42020
4266          * The bit14 of 0x70180
4267          * The bit14 of 0x71180
4268          *
4269          * WaFbcAsynchFlipDisableFbcQueue:snb
4270          */
4271         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
4272                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
4273                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
4274         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
4275                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
4276                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
4277         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D,
4278                    intel_uncore_read(&dev_priv->uncore, ILK_DSPCLK_GATE_D) |
4279                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
4280                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
4281
4282         g4x_disable_trickle_feed(dev_priv);
4283
4284         cpt_init_clock_gating(dev_priv);
4285
4286         gen6_check_mch_setup(dev_priv);
4287 }
4288
4289 static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
4290 {
4291         /*
4292          * TODO: this bit should only be enabled when really needed, then
4293          * disabled when not needed anymore in order to save power.
4294          */
4295         if (HAS_PCH_LPT_LP(dev_priv))
4296                 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
4297                            intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
4298                            PCH_LP_PARTITION_LEVEL_DISABLE);
4299
4300         /* WADPOClockGatingDisable:hsw */
4301         intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
4302                    intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
4303                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
4304 }
4305
4306 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
4307 {
4308         if (HAS_PCH_LPT_LP(dev_priv)) {
4309                 u32 val = intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D);
4310
4311                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
4312                 intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, val);
4313         }
4314 }
4315
4316 static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
4317                                    int general_prio_credits,
4318                                    int high_prio_credits)
4319 {
4320         u32 misccpctl;
4321         u32 val;
4322
4323         /* WaTempDisableDOPClkGating:bdw */
4324         misccpctl = intel_uncore_rmw(&dev_priv->uncore, GEN7_MISCCPCTL,
4325                                      GEN7_DOP_CLOCK_GATE_ENABLE, 0);
4326
4327         val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
4328         val &= ~L3_PRIO_CREDITS_MASK;
4329         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
4330         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
4331         intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
4332
4333         /*
4334          * Wait at least 100 clocks before re-enabling clock gating.
4335          * See the definition of L3SQCREG1 in BSpec.
4336          */
4337         intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
4338         udelay(1);
4339         intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
4340 }
4341
4342 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
4343 {
4344         /* Wa_1409120013:icl,ehl */
4345         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4346                            DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4347
4348         /*Wa_14010594013:icl, ehl */
4349         intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
4350                          0, ICL_DELAY_PMRSP);
4351 }
4352
4353 static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
4354 {
4355         /* Wa_1409120013 */
4356         if (DISPLAY_VER(dev_priv) == 12)
4357                 intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4358                                    DPFC_CHICKEN_COMP_DUMMY_PIXEL);
4359
4360         /* Wa_1409825376:tgl (pre-prod)*/
4361         if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
4362                 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
4363                            TGL_VRH_GATING_DIS);
4364
4365         /* Wa_14013723622:tgl,rkl,dg1,adl-s */
4366         if (DISPLAY_VER(dev_priv) == 12)
4367                 intel_uncore_rmw(&dev_priv->uncore, CLKREQ_POLICY,
4368                                  CLKREQ_POLICY_MEM_UP_OVRD, 0);
4369 }
4370
4371 static void adlp_init_clock_gating(struct drm_i915_private *dev_priv)
4372 {
4373         gen12lp_init_clock_gating(dev_priv);
4374
4375         /* Wa_22011091694:adlp */
4376         intel_de_rmw(dev_priv, GEN9_CLKGATE_DIS_5, 0, DPCE_GATING_DIS);
4377
4378         /* Bspec/49189 Initialize Sequence */
4379         intel_de_rmw(dev_priv, GEN8_CHICKEN_DCPR_1, DDI_CLOCK_REG_ACCESS, 0);
4380 }
4381
4382 static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
4383 {
4384         gen12lp_init_clock_gating(dev_priv);
4385
4386         /* Wa_1409836686:dg1[a0] */
4387         if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4388                 intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
4389                            DPT_GATING_DIS);
4390 }
4391
4392 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
4393 {
4394         /* Wa_22010146351:xehpsdv */
4395         if (IS_XEHPSDV_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
4396                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4397 }
4398
4399 static void dg2_init_clock_gating(struct drm_i915_private *i915)
4400 {
4401         /* Wa_22010954014:dg2 */
4402         intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4403                          SGSI_SIDECLK_DIS);
4404
4405         /*
4406          * Wa_14010733611:dg2_g10
4407          * Wa_22010146351:dg2_g10
4408          */
4409         if (IS_DG2_GRAPHICS_STEP(i915, G10, STEP_A0, STEP_B0))
4410                 intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
4411                                  SGR_DIS | SGGI_DIS);
4412 }
4413
4414 static void pvc_init_clock_gating(struct drm_i915_private *dev_priv)
4415 {
4416         /* Wa_14012385139:pvc */
4417         if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4418                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGR_DIS);
4419
4420         /* Wa_22010954014:pvc */
4421         if (IS_PVC_BD_STEP(dev_priv, STEP_A0, STEP_B0))
4422                 intel_uncore_rmw(&dev_priv->uncore, XEHP_CLOCK_GATE_DIS, 0, SGSI_SIDECLK_DIS);
4423 }
4424
4425 static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
4426 {
4427         if (!HAS_PCH_CNP(dev_priv))
4428                 return;
4429
4430         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
4431         intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
4432                    CNP_PWM_CGE_GATING_DISABLE);
4433 }
4434
4435 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
4436 {
4437         cnp_init_clock_gating(dev_priv);
4438         gen9_init_clock_gating(dev_priv);
4439
4440         /* WAC6entrylatency:cfl */
4441         intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
4442                    FBC_LLC_FULLY_OPEN);
4443
4444         /*
4445          * WaFbcTurnOffFbcWatermark:cfl
4446          * Display WA #0562: cfl
4447          */
4448         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4449                    DISP_FBC_WM_DIS);
4450
4451         /*
4452          * WaFbcNukeOnHostModify:cfl
4453          * Display WA #0873: cfl
4454          */
4455         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4456                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
4457                            DPFC_NUKE_ON_ANY_MODIFICATION);
4458 }
4459
4460 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
4461 {
4462         gen9_init_clock_gating(dev_priv);
4463
4464         /* WAC6entrylatency:kbl */
4465         intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
4466                    FBC_LLC_FULLY_OPEN);
4467
4468         /* WaDisableSDEUnitClockGating:kbl */
4469         if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4470                 intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
4471                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4472
4473         /* WaDisableGamClockGating:kbl */
4474         if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
4475                 intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
4476                            GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
4477
4478         /*
4479          * WaFbcTurnOffFbcWatermark:kbl
4480          * Display WA #0562: kbl
4481          */
4482         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4483                    DISP_FBC_WM_DIS);
4484
4485         /*
4486          * WaFbcNukeOnHostModify:kbl
4487          * Display WA #0873: kbl
4488          */
4489         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4490                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
4491                            DPFC_NUKE_ON_ANY_MODIFICATION);
4492 }
4493
4494 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
4495 {
4496         gen9_init_clock_gating(dev_priv);
4497
4498         /* WaDisableDopClockGating:skl */
4499         intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
4500                    ~GEN7_DOP_CLOCK_GATE_ENABLE);
4501
4502         /* WAC6entrylatency:skl */
4503         intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
4504                    FBC_LLC_FULLY_OPEN);
4505
4506         /*
4507          * WaFbcTurnOffFbcWatermark:skl
4508          * Display WA #0562: skl
4509          */
4510         intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
4511                    DISP_FBC_WM_DIS);
4512
4513         /*
4514          * WaFbcNukeOnHostModify:skl
4515          * Display WA #0873: skl
4516          */
4517         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4518                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
4519                            DPFC_NUKE_ON_ANY_MODIFICATION);
4520
4521         /*
4522          * WaFbcHighMemBwCorruptionAvoidance:skl
4523          * Display WA #0883: skl
4524          */
4525         intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
4526                            intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
4527                            DPFC_DISABLE_DUMMY0);
4528 }
4529
4530 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
4531 {
4532         enum pipe pipe;
4533
4534         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4535         intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
4536                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
4537                    HSW_FBCQ_DIS);
4538
4539         /* WaSwitchSolVfFArbitrationPriority:bdw */
4540         intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
4541
4542         /* WaPsrDPAMaskVBlankInSRD:bdw */
4543         intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
4544                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
4545
4546         for_each_pipe(dev_priv, pipe) {
4547                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
4548                 intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
4549                            intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
4550                            BDW_DPRS_MASK_VBLANK_SRD);
4551         }
4552
4553         /* WaVSRefCountFullforceMissDisable:bdw */
4554         /* WaDSRefCountFullforceMissDisable:bdw */
4555         intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4556                    intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
4557                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
4558
4559         intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4560                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4561
4562         /* WaDisableSDEUnitClockGating:bdw */
4563         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
4564                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4565
4566         /* WaProgramL3SqcReg1Default:bdw */
4567         gen8_set_l3sqc_credits(dev_priv, 30, 2);
4568
4569         /* WaKVMNotificationOnConfigChange:bdw */
4570         intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
4571                    | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
4572
4573         lpt_init_clock_gating(dev_priv);
4574
4575         /* WaDisableDopClockGating:bdw
4576          *
4577          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
4578          * clock gating.
4579          */
4580         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
4581                    intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
4582 }
4583
4584 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
4585 {
4586         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
4587         intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
4588                    intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
4589                    HSW_FBCQ_DIS);
4590
4591         /* This is required by WaCatErrorRejectionIssue:hsw */
4592         intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4593                    intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
4594                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4595
4596         /* WaSwitchSolVfFArbitrationPriority:hsw */
4597         intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
4598
4599         lpt_init_clock_gating(dev_priv);
4600 }
4601
4602 static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
4603 {
4604         intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
4605
4606         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
4607         intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
4608                    intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
4609                    ILK_FBCQ_DIS);
4610
4611         /* WaDisableBackToBackFlipFix:ivb */
4612         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4613                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4614                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
4615
4616         if (IS_IVB_GT1(dev_priv))
4617                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4618                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4619         else {
4620                 /* must write both registers */
4621                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4622                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4623                 intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2_GT2,
4624                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4625         }
4626
4627         /*
4628          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4629          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
4630          */
4631         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4632                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4633
4634         /* This is required by WaCatErrorRejectionIssue:ivb */
4635         intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4636                         intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
4637                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4638
4639         g4x_disable_trickle_feed(dev_priv);
4640
4641         intel_uncore_rmw(&dev_priv->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
4642                          GEN6_MBC_SNPCR_MED);
4643
4644         if (!HAS_PCH_NOP(dev_priv))
4645                 cpt_init_clock_gating(dev_priv);
4646
4647         gen6_check_mch_setup(dev_priv);
4648 }
4649
4650 static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
4651 {
4652         /* WaDisableBackToBackFlipFix:vlv */
4653         intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
4654                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
4655                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
4656
4657         /* WaDisableDopClockGating:vlv */
4658         intel_uncore_write(&dev_priv->uncore, GEN7_ROW_CHICKEN2,
4659                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
4660
4661         /* This is required by WaCatErrorRejectionIssue:vlv */
4662         intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
4663                    intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
4664                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
4665
4666         /*
4667          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
4668          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
4669          */
4670         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL2,
4671                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
4672
4673         /* WaDisableL3Bank2xClockGate:vlv
4674          * Disabling L3 clock gating- MMIO 940c[25] = 1
4675          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
4676         intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
4677                    intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
4678
4679         /*
4680          * WaDisableVLVClockGating_VBIIssue:vlv
4681          * Disable clock gating on th GCFG unit to prevent a delay
4682          * in the reporting of vblank events.
4683          */
4684         intel_uncore_write(&dev_priv->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
4685 }
4686
4687 static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
4688 {
4689         /* WaVSRefCountFullforceMissDisable:chv */
4690         /* WaDSRefCountFullforceMissDisable:chv */
4691         intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
4692                    intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
4693                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
4694
4695         /* WaDisableSemaphoreAndSyncFlipWait:chv */
4696         intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
4697                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
4698
4699         /* WaDisableCSUnitClockGating:chv */
4700         intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
4701                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
4702
4703         /* WaDisableSDEUnitClockGating:chv */
4704         intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
4705                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
4706
4707         /*
4708          * WaProgramL3SqcReg1Default:chv
4709          * See gfxspecs/Related Documents/Performance Guide/
4710          * LSQC Setting Recommendations.
4711          */
4712         gen8_set_l3sqc_credits(dev_priv, 38, 2);
4713 }
4714
4715 static void g4x_init_clock_gating(struct drm_i915_private *dev_priv)
4716 {
4717         u32 dspclk_gate;
4718
4719         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, 0);
4720         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
4721                    GS_UNIT_CLOCK_GATE_DISABLE |
4722                    CL_UNIT_CLOCK_GATE_DISABLE);
4723         intel_uncore_write(&dev_priv->uncore, RAMCLK_GATE_D, 0);
4724         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
4725                 OVRUNIT_CLOCK_GATE_DISABLE |
4726                 OVCUNIT_CLOCK_GATE_DISABLE;
4727         if (IS_GM45(dev_priv))
4728                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
4729         intel_uncore_write(&dev_priv->uncore, DSPCLK_GATE_D(dev_priv), dspclk_gate);
4730
4731         g4x_disable_trickle_feed(dev_priv);
4732 }
4733
4734 static void i965gm_init_clock_gating(struct drm_i915_private *dev_priv)
4735 {
4736         struct intel_uncore *uncore = &dev_priv->uncore;
4737
4738         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
4739         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
4740         intel_uncore_write(uncore, DSPCLK_GATE_D(dev_priv), 0);
4741         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
4742         intel_uncore_write16(uncore, DEUC, 0);
4743         intel_uncore_write(uncore,
4744                            MI_ARB_STATE,
4745                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4746 }
4747
4748 static void i965g_init_clock_gating(struct drm_i915_private *dev_priv)
4749 {
4750         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
4751                    I965_RCC_CLOCK_GATE_DISABLE |
4752                    I965_RCPB_CLOCK_GATE_DISABLE |
4753                    I965_ISC_CLOCK_GATE_DISABLE |
4754                    I965_FBC_CLOCK_GATE_DISABLE);
4755         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D2, 0);
4756         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4757                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4758 }
4759
4760 static void gen3_init_clock_gating(struct drm_i915_private *dev_priv)
4761 {
4762         u32 dstate = intel_uncore_read(&dev_priv->uncore, D_STATE);
4763
4764         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
4765                 DSTATE_DOT_CLOCK_GATING;
4766         intel_uncore_write(&dev_priv->uncore, D_STATE, dstate);
4767
4768         if (IS_PINEVIEW(dev_priv))
4769                 intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4770                                    _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
4771
4772         /* IIR "flip pending" means done if this bit is set */
4773         intel_uncore_write(&dev_priv->uncore, ECOSKPD(RENDER_RING_BASE),
4774                            _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
4775
4776         /* interrupts should cause a wake up from C3 */
4777         intel_uncore_write(&dev_priv->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
4778
4779         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4780         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
4781
4782         intel_uncore_write(&dev_priv->uncore, MI_ARB_STATE,
4783                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
4784 }
4785
4786 static void i85x_init_clock_gating(struct drm_i915_private *dev_priv)
4787 {
4788         intel_uncore_write(&dev_priv->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
4789
4790         /* interrupts should cause a wake up from C3 */
4791         intel_uncore_write(&dev_priv->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
4792                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
4793
4794         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4795                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
4796
4797         /*
4798          * Have FBC ignore 3D activity since we use software
4799          * render tracking, and otherwise a pure 3D workload
4800          * (even if it just renders a single frame and then does
4801          * abosultely nothing) would not allow FBC to recompress
4802          * until a 2D blit occurs.
4803          */
4804         intel_uncore_write(&dev_priv->uncore, SCPD0,
4805                    _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
4806 }
4807
4808 static void i830_init_clock_gating(struct drm_i915_private *dev_priv)
4809 {
4810         intel_uncore_write(&dev_priv->uncore, MEM_MODE,
4811                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
4812                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
4813 }
4814
4815 void intel_init_clock_gating(struct drm_i915_private *dev_priv)
4816 {
4817         dev_priv->clock_gating_funcs->init_clock_gating(dev_priv);
4818 }
4819
4820 void intel_suspend_hw(struct drm_i915_private *dev_priv)
4821 {
4822         if (HAS_PCH_LPT(dev_priv))
4823                 lpt_suspend_hw(dev_priv);
4824 }
4825
4826 static void nop_init_clock_gating(struct drm_i915_private *dev_priv)
4827 {
4828         drm_dbg_kms(&dev_priv->drm,
4829                     "No clock gating settings or workarounds applied.\n");
4830 }
4831
4832 #define CG_FUNCS(platform)                                              \
4833 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
4834         .init_clock_gating = platform##_init_clock_gating,              \
4835 }
4836
4837 CG_FUNCS(pvc);
4838 CG_FUNCS(dg2);
4839 CG_FUNCS(xehpsdv);
4840 CG_FUNCS(adlp);
4841 CG_FUNCS(dg1);
4842 CG_FUNCS(gen12lp);
4843 CG_FUNCS(icl);
4844 CG_FUNCS(cfl);
4845 CG_FUNCS(skl);
4846 CG_FUNCS(kbl);
4847 CG_FUNCS(bxt);
4848 CG_FUNCS(glk);
4849 CG_FUNCS(bdw);
4850 CG_FUNCS(chv);
4851 CG_FUNCS(hsw);
4852 CG_FUNCS(ivb);
4853 CG_FUNCS(vlv);
4854 CG_FUNCS(gen6);
4855 CG_FUNCS(ilk);
4856 CG_FUNCS(g4x);
4857 CG_FUNCS(i965gm);
4858 CG_FUNCS(i965g);
4859 CG_FUNCS(gen3);
4860 CG_FUNCS(i85x);
4861 CG_FUNCS(i830);
4862 CG_FUNCS(nop);
4863 #undef CG_FUNCS
4864
4865 /**
4866  * intel_init_clock_gating_hooks - setup the clock gating hooks
4867  * @dev_priv: device private
4868  *
4869  * Setup the hooks that configure which clocks of a given platform can be
4870  * gated and also apply various GT and display specific workarounds for these
4871  * platforms. Note that some GT specific workarounds are applied separately
4872  * when GPU contexts or batchbuffers start their execution.
4873  */
4874 void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
4875 {
4876         if (IS_PONTEVECCHIO(dev_priv))
4877                 dev_priv->clock_gating_funcs = &pvc_clock_gating_funcs;
4878         else if (IS_DG2(dev_priv))
4879                 dev_priv->clock_gating_funcs = &dg2_clock_gating_funcs;
4880         else if (IS_XEHPSDV(dev_priv))
4881                 dev_priv->clock_gating_funcs = &xehpsdv_clock_gating_funcs;
4882         else if (IS_ALDERLAKE_P(dev_priv))
4883                 dev_priv->clock_gating_funcs = &adlp_clock_gating_funcs;
4884         else if (IS_DG1(dev_priv))
4885                 dev_priv->clock_gating_funcs = &dg1_clock_gating_funcs;
4886         else if (GRAPHICS_VER(dev_priv) == 12)
4887                 dev_priv->clock_gating_funcs = &gen12lp_clock_gating_funcs;
4888         else if (GRAPHICS_VER(dev_priv) == 11)
4889                 dev_priv->clock_gating_funcs = &icl_clock_gating_funcs;
4890         else if (IS_COFFEELAKE(dev_priv) || IS_COMETLAKE(dev_priv))
4891                 dev_priv->clock_gating_funcs = &cfl_clock_gating_funcs;
4892         else if (IS_SKYLAKE(dev_priv))
4893                 dev_priv->clock_gating_funcs = &skl_clock_gating_funcs;
4894         else if (IS_KABYLAKE(dev_priv))
4895                 dev_priv->clock_gating_funcs = &kbl_clock_gating_funcs;
4896         else if (IS_BROXTON(dev_priv))
4897                 dev_priv->clock_gating_funcs = &bxt_clock_gating_funcs;
4898         else if (IS_GEMINILAKE(dev_priv))
4899                 dev_priv->clock_gating_funcs = &glk_clock_gating_funcs;
4900         else if (IS_BROADWELL(dev_priv))
4901                 dev_priv->clock_gating_funcs = &bdw_clock_gating_funcs;
4902         else if (IS_CHERRYVIEW(dev_priv))
4903                 dev_priv->clock_gating_funcs = &chv_clock_gating_funcs;
4904         else if (IS_HASWELL(dev_priv))
4905                 dev_priv->clock_gating_funcs = &hsw_clock_gating_funcs;
4906         else if (IS_IVYBRIDGE(dev_priv))
4907                 dev_priv->clock_gating_funcs = &ivb_clock_gating_funcs;
4908         else if (IS_VALLEYVIEW(dev_priv))
4909                 dev_priv->clock_gating_funcs = &vlv_clock_gating_funcs;
4910         else if (GRAPHICS_VER(dev_priv) == 6)
4911                 dev_priv->clock_gating_funcs = &gen6_clock_gating_funcs;
4912         else if (GRAPHICS_VER(dev_priv) == 5)
4913                 dev_priv->clock_gating_funcs = &ilk_clock_gating_funcs;
4914         else if (IS_G4X(dev_priv))
4915                 dev_priv->clock_gating_funcs = &g4x_clock_gating_funcs;
4916         else if (IS_I965GM(dev_priv))
4917                 dev_priv->clock_gating_funcs = &i965gm_clock_gating_funcs;
4918         else if (IS_I965G(dev_priv))
4919                 dev_priv->clock_gating_funcs = &i965g_clock_gating_funcs;
4920         else if (GRAPHICS_VER(dev_priv) == 3)
4921                 dev_priv->clock_gating_funcs = &gen3_clock_gating_funcs;
4922         else if (IS_I85X(dev_priv) || IS_I865G(dev_priv))
4923                 dev_priv->clock_gating_funcs = &i85x_clock_gating_funcs;
4924         else if (GRAPHICS_VER(dev_priv) == 2)
4925                 dev_priv->clock_gating_funcs = &i830_clock_gating_funcs;
4926         else {
4927                 MISSING_CASE(INTEL_DEVID(dev_priv));
4928                 dev_priv->clock_gating_funcs = &nop_clock_gating_funcs;
4929         }
4930 }
4931
4932 static const struct intel_wm_funcs ilk_wm_funcs = {
4933         .compute_pipe_wm = ilk_compute_pipe_wm,
4934         .compute_intermediate_wm = ilk_compute_intermediate_wm,
4935         .initial_watermarks = ilk_initial_watermarks,
4936         .optimize_watermarks = ilk_optimize_watermarks,
4937 };
4938
4939 static const struct intel_wm_funcs vlv_wm_funcs = {
4940         .compute_pipe_wm = vlv_compute_pipe_wm,
4941         .compute_intermediate_wm = vlv_compute_intermediate_wm,
4942         .initial_watermarks = vlv_initial_watermarks,
4943         .optimize_watermarks = vlv_optimize_watermarks,
4944         .atomic_update_watermarks = vlv_atomic_update_fifo,
4945 };
4946
4947 static const struct intel_wm_funcs g4x_wm_funcs = {
4948         .compute_pipe_wm = g4x_compute_pipe_wm,
4949         .compute_intermediate_wm = g4x_compute_intermediate_wm,
4950         .initial_watermarks = g4x_initial_watermarks,
4951         .optimize_watermarks = g4x_optimize_watermarks,
4952 };
4953
4954 static const struct intel_wm_funcs pnv_wm_funcs = {
4955         .update_wm = pnv_update_wm,
4956 };
4957
4958 static const struct intel_wm_funcs i965_wm_funcs = {
4959         .update_wm = i965_update_wm,
4960 };
4961
4962 static const struct intel_wm_funcs i9xx_wm_funcs = {
4963         .update_wm = i9xx_update_wm,
4964 };
4965
4966 static const struct intel_wm_funcs i845_wm_funcs = {
4967         .update_wm = i845_update_wm,
4968 };
4969
4970 static const struct intel_wm_funcs nop_funcs = {
4971 };
4972
4973 /* Set up chip specific power management-related functions */
4974 void intel_init_pm(struct drm_i915_private *dev_priv)
4975 {
4976         if (DISPLAY_VER(dev_priv) >= 9) {
4977                 skl_wm_init(dev_priv);
4978                 return;
4979         }
4980
4981         /* For cxsr */
4982         if (IS_PINEVIEW(dev_priv))
4983                 pnv_get_mem_freq(dev_priv);
4984         else if (GRAPHICS_VER(dev_priv) == 5)
4985                 ilk_get_mem_freq(dev_priv);
4986
4987         /* For FIFO watermark updates */
4988         if (HAS_PCH_SPLIT(dev_priv)) {
4989                 ilk_setup_wm_latency(dev_priv);
4990
4991                 if ((DISPLAY_VER(dev_priv) == 5 && dev_priv->display.wm.pri_latency[1] &&
4992                      dev_priv->display.wm.spr_latency[1] && dev_priv->display.wm.cur_latency[1]) ||
4993                     (DISPLAY_VER(dev_priv) != 5 && dev_priv->display.wm.pri_latency[0] &&
4994                      dev_priv->display.wm.spr_latency[0] && dev_priv->display.wm.cur_latency[0])) {
4995                         dev_priv->display.funcs.wm = &ilk_wm_funcs;
4996                 } else {
4997                         drm_dbg_kms(&dev_priv->drm,
4998                                     "Failed to read display plane latency. "
4999                                     "Disable CxSR\n");
5000                         dev_priv->display.funcs.wm = &nop_funcs;
5001                 }
5002         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
5003                 vlv_setup_wm_latency(dev_priv);
5004                 dev_priv->display.funcs.wm = &vlv_wm_funcs;
5005         } else if (IS_G4X(dev_priv)) {
5006                 g4x_setup_wm_latency(dev_priv);
5007                 dev_priv->display.funcs.wm = &g4x_wm_funcs;
5008         } else if (IS_PINEVIEW(dev_priv)) {
5009                 if (!intel_get_cxsr_latency(!IS_MOBILE(dev_priv),
5010                                             dev_priv->is_ddr3,
5011                                             dev_priv->fsb_freq,
5012                                             dev_priv->mem_freq)) {
5013                         drm_info(&dev_priv->drm,
5014                                  "failed to find known CxSR latency "
5015                                  "(found ddr%s fsb freq %d, mem freq %d), "
5016                                  "disabling CxSR\n",
5017                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
5018                                  dev_priv->fsb_freq, dev_priv->mem_freq);
5019                         /* Disable CxSR and never update its watermark again */
5020                         intel_set_memory_cxsr(dev_priv, false);
5021                         dev_priv->display.funcs.wm = &nop_funcs;
5022                 } else
5023                         dev_priv->display.funcs.wm = &pnv_wm_funcs;
5024         } else if (DISPLAY_VER(dev_priv) == 4) {
5025                 dev_priv->display.funcs.wm = &i965_wm_funcs;
5026         } else if (DISPLAY_VER(dev_priv) == 3) {
5027                 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
5028         } else if (DISPLAY_VER(dev_priv) == 2) {
5029                 if (INTEL_NUM_PIPES(dev_priv) == 1)
5030                         dev_priv->display.funcs.wm = &i845_wm_funcs;
5031                 else
5032                         dev_priv->display.funcs.wm = &i9xx_wm_funcs;
5033         } else {
5034                 drm_err(&dev_priv->drm,
5035                         "unexpected fall-through in %s\n", __func__);
5036                 dev_priv->display.funcs.wm = &nop_funcs;
5037         }
5038 }
5039
5040 void intel_pm_setup(struct drm_i915_private *dev_priv)
5041 {
5042         dev_priv->runtime_pm.suspended = false;
5043         atomic_set(&dev_priv->runtime_pm.wakeref_count, 0);
5044 }