drivers/gpu/drm/i915/intel_pm.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include <linux/cpufreq.h>
  29 #include "i915_drv.h"
  30 #include "intel_drv.h"
  31 #include "../../../platform/x86/intel_ips.h"
  32 #include <linux/module.h>
  33
  34 /**
  35  * RC6 is a special power stage which allows the GPU to enter an very
  36  * low-voltage mode when idle, using down to 0V while at this stage.  This
  37  * stage is entered automatically when the GPU is idle when RC6 support is
  38  * enabled, and as soon as new workload arises GPU wakes up automatically as well.
  39  *
  40  * There are different RC6 modes available in Intel GPU, which differentiate
  41  * among each other with the latency required to enter and leave RC6 and
  42  * voltage consumed by the GPU in different states.
  43  *
  44  * The combination of the following flags define which states GPU is allowed
  45  * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
  46  * RC6pp is deepest RC6. Their support by hardware varies according to the
  47  * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
  48  * which brings the most power savings; deeper states save more power, but
  49  * require higher latency to switch to and wake up.
  50  */
  51 #define INTEL_RC6_ENABLE                        (1<<0)
  52 #define INTEL_RC6p_ENABLE                       (1<<1)
  53 #define INTEL_RC6pp_ENABLE                      (1<<2)
  54
  55 static void gen9_init_clock_gating(struct drm_device *dev)
  56 {
  57         struct drm_i915_private *dev_priv = dev->dev_private;
  58
  59         /* WaEnableLbsSlaRetryTimerDecrement:skl */
  60         I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
  61                    GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
  62 }
  63
  64 static void skl_init_clock_gating(struct drm_device *dev)
  65 {
  66         struct drm_i915_private *dev_priv = dev->dev_private;
  67
  68         gen9_init_clock_gating(dev);
  69
  70         if (INTEL_REVID(dev) == SKL_REVID_A0) {
  71                 /*
  72                  * WaDisableSDEUnitClockGating:skl
  73                  * WaSetGAPSunitClckGateDisable:skl
  74                  */
  75                 I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
  76                            GEN8_GAPSUNIT_CLOCK_GATE_DISABLE |
  77                            GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  78         }
  79
  80         if (INTEL_REVID(dev) <= SKL_REVID_D0) {
  81                 /* WaDisableHDCInvalidation:skl */
  82                 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
  83                            BDW_DISABLE_HDC_INVALIDATION);
  84
  85                 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
  86                 I915_WRITE(FF_SLICE_CS_CHICKEN2,
  87                            I915_READ(FF_SLICE_CS_CHICKEN2) |
  88                            GEN9_TSG_BARRIER_ACK_DISABLE);
  89         }
  90
  91         if (INTEL_REVID(dev) <= SKL_REVID_E0)
  92                 /* WaDisableLSQCROPERFforOCL:skl */
  93                 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
  94                            GEN8_LQSC_RO_PERF_DIS);
  95 }
  96
  97 static void i915_pineview_get_mem_freq(struct drm_device *dev)
  98 {
  99         struct drm_i915_private *dev_priv = dev->dev_private;
 100         u32 tmp;
 101
 102         tmp = I915_READ(CLKCFG);
 103
 104         switch (tmp & CLKCFG_FSB_MASK) {
 105         case CLKCFG_FSB_533:
 106                 dev_priv->fsb_freq = 533; /* 133*4 */
 107                 break;
 108         case CLKCFG_FSB_800:
 109                 dev_priv->fsb_freq = 800; /* 200*4 */
 110                 break;
 111         case CLKCFG_FSB_667:
 112                 dev_priv->fsb_freq =  667; /* 167*4 */
 113                 break;
 114         case CLKCFG_FSB_400:
 115                 dev_priv->fsb_freq = 400; /* 100*4 */
 116                 break;
 117         }
 118
 119         switch (tmp & CLKCFG_MEM_MASK) {
 120         case CLKCFG_MEM_533:
 121                 dev_priv->mem_freq = 533;
 122                 break;
 123         case CLKCFG_MEM_667:
 124                 dev_priv->mem_freq = 667;
 125                 break;
 126         case CLKCFG_MEM_800:
 127                 dev_priv->mem_freq = 800;
 128                 break;
 129         }
 130
 131         /* detect pineview DDR3 setting */
 132         tmp = I915_READ(CSHRDDR3CTL);
 133         dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
 134 }
 135
 136 static void i915_ironlake_get_mem_freq(struct drm_device *dev)
 137 {
 138         struct drm_i915_private *dev_priv = dev->dev_private;
 139         u16 ddrpll, csipll;
 140
 141         ddrpll = I915_READ16(DDRMPLL1);
 142         csipll = I915_READ16(CSIPLL0);
 143
 144         switch (ddrpll & 0xff) {
 145         case 0xc:
 146                 dev_priv->mem_freq = 800;
 147                 break;
 148         case 0x10:
 149                 dev_priv->mem_freq = 1066;
 150                 break;
 151         case 0x14:
 152                 dev_priv->mem_freq = 1333;
 153                 break;
 154         case 0x18:
 155                 dev_priv->mem_freq = 1600;
 156                 break;
 157         default:
 158                 DRM_DEBUG_DRIVER("unknown memory frequency 0x%02x\n",
 159                                  ddrpll & 0xff);
 160                 dev_priv->mem_freq = 0;
 161                 break;
 162         }
 163
 164         dev_priv->ips.r_t = dev_priv->mem_freq;
 165
 166         switch (csipll & 0x3ff) {
 167         case 0x00c:
 168                 dev_priv->fsb_freq = 3200;
 169                 break;
 170         case 0x00e:
 171                 dev_priv->fsb_freq = 3733;
 172                 break;
 173         case 0x010:
 174                 dev_priv->fsb_freq = 4266;
 175                 break;
 176         case 0x012:
 177                 dev_priv->fsb_freq = 4800;
 178                 break;
 179         case 0x014:
 180                 dev_priv->fsb_freq = 5333;
 181                 break;
 182         case 0x016:
 183                 dev_priv->fsb_freq = 5866;
 184                 break;
 185         case 0x018:
 186                 dev_priv->fsb_freq = 6400;
 187                 break;
 188         default:
 189                 DRM_DEBUG_DRIVER("unknown fsb frequency 0x%04x\n",
 190                                  csipll & 0x3ff);
 191                 dev_priv->fsb_freq = 0;
 192                 break;
 193         }
 194
 195         if (dev_priv->fsb_freq == 3200) {
 196                 dev_priv->ips.c_m = 0;
 197         } else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
 198                 dev_priv->ips.c_m = 1;
 199         } else {
 200                 dev_priv->ips.c_m = 2;
 201         }
 202 }
 203
 204 static const struct cxsr_latency cxsr_latency_table[] = {
 205         {1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
 206         {1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
 207         {1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
 208         {1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
 209         {1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
 210
 211         {1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
 212         {1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
 213         {1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
 214         {1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
 215         {1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
 216
 217         {1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
 218         {1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
 219         {1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
 220         {1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
 221         {1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
 222
 223         {0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
 224         {0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
 225         {0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
 226         {0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
 227         {0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
 228
 229         {0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
 230         {0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
 231         {0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
 232         {0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
 233         {0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
 234
 235         {0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
 236         {0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
 237         {0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
 238         {0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
 239         {0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
 240 };
 241
 242 static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
 243                                                          int is_ddr3,
 244                                                          int fsb,
 245                                                          int mem)
 246 {
 247         const struct cxsr_latency *latency;
 248         int i;
 249
 250         if (fsb == 0 || mem == 0)
 251                 return NULL;
 252
 253         for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
 254                 latency = &cxsr_latency_table[i];
 255                 if (is_desktop == latency->is_desktop &&
 256                     is_ddr3 == latency->is_ddr3 &&
 257                     fsb == latency->fsb_freq && mem == latency->mem_freq)
 258                         return latency;
 259         }
 260
 261         DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 262
 263         return NULL;
 264 }
 265
 266 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
 267 {
 268         u32 val;
 269
 270         mutex_lock(&dev_priv->rps.hw_lock);
 271
 272         val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
 273         if (enable)
 274                 val &= ~FORCE_DDR_HIGH_FREQ;
 275         else
 276                 val |= FORCE_DDR_HIGH_FREQ;
 277         val &= ~FORCE_DDR_LOW_FREQ;
 278         val |= FORCE_DDR_FREQ_REQ_ACK;
 279         vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
 280
 281         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
 282                       FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
 283                 DRM_ERROR("timed out waiting for Punit DDR DVFS request\n");
 284
 285         mutex_unlock(&dev_priv->rps.hw_lock);
 286 }
 287
 288 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 289 {
 290         u32 val;
 291
 292         mutex_lock(&dev_priv->rps.hw_lock);
 293
 294         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
 295         if (enable)
 296                 val |= DSP_MAXFIFO_PM5_ENABLE;
 297         else
 298                 val &= ~DSP_MAXFIFO_PM5_ENABLE;
 299         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
 300
 301         mutex_unlock(&dev_priv->rps.hw_lock);
 302 }
 303
 304 #define FW_WM(value, plane) \
 305         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
 306
 307 void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 308 {
 309         struct drm_device *dev = dev_priv->dev;
 310         u32 val;
 311
 312         if (IS_VALLEYVIEW(dev)) {
 313                 I915_WRITE(FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
 314                 if (IS_CHERRYVIEW(dev))
 315                         chv_set_memory_pm5(dev_priv, enable);
 316         } else if (IS_G4X(dev) || IS_CRESTLINE(dev)) {
 317                 I915_WRITE(FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
 318         } else if (IS_PINEVIEW(dev)) {
 319                 val = I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN;
 320                 val |= enable ? PINEVIEW_SELF_REFRESH_EN : 0;
 321                 I915_WRITE(DSPFW3, val);
 322         } else if (IS_I945G(dev) || IS_I945GM(dev)) {
 323                 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
 324                                _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
 325                 I915_WRITE(FW_BLC_SELF, val);
 326         } else if (IS_I915GM(dev)) {
 327                 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
 328                                _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
 329                 I915_WRITE(INSTPM, val);
 330         } else {
 331                 return;
 332         }
 333
 334         DRM_DEBUG_KMS("memory self-refresh is %s\n",
 335                       enable ? "enabled" : "disabled");
 336 }
 337
 338
 339 /*
 340  * Latency for FIFO fetches is dependent on several factors:
 341  *   - memory configuration (speed, channels)
 342  *   - chipset
 343  *   - current MCH state
 344  * It can be fairly high in some situations, so here we assume a fairly
 345  * pessimal value.  It's a tradeoff between extra memory fetches (if we
 346  * set this value too high, the FIFO will fetch frequently to stay full)
 347  * and power consumption (set it too low to save power and we might see
 348  * FIFO underruns and display "flicker").
 349  *
 350  * A value of 5us seems to be a good balance; safe for very low end
 351  * platforms but not overly aggressive on lower latency configs.
 352  */
 353 static const int pessimal_latency_ns = 5000;
 354
 355 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
 356         ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
 357
 358 static int vlv_get_fifo_size(struct drm_device *dev,
 359                               enum pipe pipe, int plane)
 360 {
 361         struct drm_i915_private *dev_priv = dev->dev_private;
 362         int sprite0_start, sprite1_start, size;
 363
 364         switch (pipe) {
 365                 uint32_t dsparb, dsparb2, dsparb3;
 366         case PIPE_A:
 367                 dsparb = I915_READ(DSPARB);
 368                 dsparb2 = I915_READ(DSPARB2);
 369                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
 370                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
 371                 break;
 372         case PIPE_B:
 373                 dsparb = I915_READ(DSPARB);
 374                 dsparb2 = I915_READ(DSPARB2);
 375                 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
 376                 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
 377                 break;
 378         case PIPE_C:
 379                 dsparb2 = I915_READ(DSPARB2);
 380                 dsparb3 = I915_READ(DSPARB3);
 381                 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
 382                 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
 383                 break;
 384         default:
 385                 return 0;
 386         }
 387
 388         switch (plane) {
 389         case 0:
 390                 size = sprite0_start;
 391                 break;
 392         case 1:
 393                 size = sprite1_start - sprite0_start;
 394                 break;
 395         case 2:
 396                 size = 512 - 1 - sprite1_start;
 397                 break;
 398         default:
 399                 return 0;
 400         }
 401
 402         DRM_DEBUG_KMS("Pipe %c %s %c FIFO size: %d\n",
 403                       pipe_name(pipe), plane == 0 ? "primary" : "sprite",
 404                       plane == 0 ? plane_name(pipe) : sprite_name(pipe, plane - 1),
 405                       size);
 406
 407         return size;
 408 }
 409
 410 static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
 411 {
 412         struct drm_i915_private *dev_priv = dev->dev_private;
 413         uint32_t dsparb = I915_READ(DSPARB);
 414         int size;
 415
 416         size = dsparb & 0x7f;
 417         if (plane)
 418                 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
 419
 420         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 421                       plane ? "B" : "A", size);
 422
 423         return size;
 424 }
 425
 426 static int i830_get_fifo_size(struct drm_device *dev, int plane)
 427 {
 428         struct drm_i915_private *dev_priv = dev->dev_private;
 429         uint32_t dsparb = I915_READ(DSPARB);
 430         int size;
 431
 432         size = dsparb & 0x1ff;
 433         if (plane)
 434                 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
 435         size >>= 1; /* Convert to cachelines */
 436
 437         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 438                       plane ? "B" : "A", size);
 439
 440         return size;
 441 }
 442
 443 static int i845_get_fifo_size(struct drm_device *dev, int plane)
 444 {
 445         struct drm_i915_private *dev_priv = dev->dev_private;
 446         uint32_t dsparb = I915_READ(DSPARB);
 447         int size;
 448
 449         size = dsparb & 0x7f;
 450         size >>= 2; /* Convert to cachelines */
 451
 452         DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
 453                       plane ? "B" : "A",
 454                       size);
 455
 456         return size;
 457 }
 458
 459 /* Pineview has different values for various configs */
 460 static const struct intel_watermark_params pineview_display_wm = {
 461         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 462         .max_wm = PINEVIEW_MAX_WM,
 463         .default_wm = PINEVIEW_DFT_WM,
 464         .guard_size = PINEVIEW_GUARD_WM,
 465         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 466 };
 467 static const struct intel_watermark_params pineview_display_hplloff_wm = {
 468         .fifo_size = PINEVIEW_DISPLAY_FIFO,
 469         .max_wm = PINEVIEW_MAX_WM,
 470         .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
 471         .guard_size = PINEVIEW_GUARD_WM,
 472         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 473 };
 474 static const struct intel_watermark_params pineview_cursor_wm = {
 475         .fifo_size = PINEVIEW_CURSOR_FIFO,
 476         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 477         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 478         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 479         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 480 };
 481 static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
 482         .fifo_size = PINEVIEW_CURSOR_FIFO,
 483         .max_wm = PINEVIEW_CURSOR_MAX_WM,
 484         .default_wm = PINEVIEW_CURSOR_DFT_WM,
 485         .guard_size = PINEVIEW_CURSOR_GUARD_WM,
 486         .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
 487 };
 488 static const struct intel_watermark_params g4x_wm_info = {
 489         .fifo_size = G4X_FIFO_SIZE,
 490         .max_wm = G4X_MAX_WM,
 491         .default_wm = G4X_MAX_WM,
 492         .guard_size = 2,
 493         .cacheline_size = G4X_FIFO_LINE_SIZE,
 494 };
 495 static const struct intel_watermark_params g4x_cursor_wm_info = {
 496         .fifo_size = I965_CURSOR_FIFO,
 497         .max_wm = I965_CURSOR_MAX_WM,
 498         .default_wm = I965_CURSOR_DFT_WM,
 499         .guard_size = 2,
 500         .cacheline_size = G4X_FIFO_LINE_SIZE,
 501 };
 502 static const struct intel_watermark_params valleyview_wm_info = {
 503         .fifo_size = VALLEYVIEW_FIFO_SIZE,
 504         .max_wm = VALLEYVIEW_MAX_WM,
 505         .default_wm = VALLEYVIEW_MAX_WM,
 506         .guard_size = 2,
 507         .cacheline_size = G4X_FIFO_LINE_SIZE,
 508 };
 509 static const struct intel_watermark_params valleyview_cursor_wm_info = {
 510         .fifo_size = I965_CURSOR_FIFO,
 511         .max_wm = VALLEYVIEW_CURSOR_MAX_WM,
 512         .default_wm = I965_CURSOR_DFT_WM,
 513         .guard_size = 2,
 514         .cacheline_size = G4X_FIFO_LINE_SIZE,
 515 };
 516 static const struct intel_watermark_params i965_cursor_wm_info = {
 517         .fifo_size = I965_CURSOR_FIFO,
 518         .max_wm = I965_CURSOR_MAX_WM,
 519         .default_wm = I965_CURSOR_DFT_WM,
 520         .guard_size = 2,
 521         .cacheline_size = I915_FIFO_LINE_SIZE,
 522 };
 523 static const struct intel_watermark_params i945_wm_info = {
 524         .fifo_size = I945_FIFO_SIZE,
 525         .max_wm = I915_MAX_WM,
 526         .default_wm = 1,
 527         .guard_size = 2,
 528         .cacheline_size = I915_FIFO_LINE_SIZE,
 529 };
 530 static const struct intel_watermark_params i915_wm_info = {
 531         .fifo_size = I915_FIFO_SIZE,
 532         .max_wm = I915_MAX_WM,
 533         .default_wm = 1,
 534         .guard_size = 2,
 535         .cacheline_size = I915_FIFO_LINE_SIZE,
 536 };
 537 static const struct intel_watermark_params i830_a_wm_info = {
 538         .fifo_size = I855GM_FIFO_SIZE,
 539         .max_wm = I915_MAX_WM,
 540         .default_wm = 1,
 541         .guard_size = 2,
 542         .cacheline_size = I830_FIFO_LINE_SIZE,
 543 };
 544 static const struct intel_watermark_params i830_bc_wm_info = {
 545         .fifo_size = I855GM_FIFO_SIZE,
 546         .max_wm = I915_MAX_WM/2,
 547         .default_wm = 1,
 548         .guard_size = 2,
 549         .cacheline_size = I830_FIFO_LINE_SIZE,
 550 };
 551 static const struct intel_watermark_params i845_wm_info = {
 552         .fifo_size = I830_FIFO_SIZE,
 553         .max_wm = I915_MAX_WM,
 554         .default_wm = 1,
 555         .guard_size = 2,
 556         .cacheline_size = I830_FIFO_LINE_SIZE,
 557 };
 558
 559 /**
 560  * intel_calculate_wm - calculate watermark level
 561  * @clock_in_khz: pixel clock
 562  * @wm: chip FIFO params
 563  * @pixel_size: display pixel size
 564  * @latency_ns: memory latency for the platform
 565  *
 566  * Calculate the watermark level (the level at which the display plane will
 567  * start fetching from memory again).  Each chip has a different display
 568  * FIFO size and allocation, so the caller needs to figure that out and pass
 569  * in the correct intel_watermark_params structure.
 570  *
 571  * As the pixel clock runs, the FIFO will be drained at a rate that depends
 572  * on the pixel size.  When it reaches the watermark level, it'll start
 573  * fetching FIFO line sized based chunks from memory until the FIFO fills
 574  * past the watermark point.  If the FIFO drains completely, a FIFO underrun
 575  * will occur, and a display engine hang could result.
 576  */
 577 static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
 578                                         const struct intel_watermark_params *wm,
 579                                         int fifo_size,
 580                                         int pixel_size,
 581                                         unsigned long latency_ns)
 582 {
 583         long entries_required, wm_size;
 584
 585         /*
 586          * Note: we need to make sure we don't overflow for various clock &
 587          * latency values.
 588          * clocks go from a few thousand to several hundred thousand.
 589          * latency is usually a few thousand
 590          */
 591         entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
 592                 1000;
 593         entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
 594
 595         DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
 596
 597         wm_size = fifo_size - (entries_required + wm->guard_size);
 598
 599         DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
 600
 601         /* Don't promote wm_size to unsigned... */
 602         if (wm_size > (long)wm->max_wm)
 603                 wm_size = wm->max_wm;
 604         if (wm_size <= 0)
 605                 wm_size = wm->default_wm;
 606
 607         /*
 608          * Bspec seems to indicate that the value shouldn't be lower than
 609          * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
 610          * Lets go for 8 which is the burst size since certain platforms
 611          * already use a hardcoded 8 (which is what the spec says should be
 612          * done).
 613          */
 614         if (wm_size <= 8)
 615                 wm_size = 8;
 616
 617         return wm_size;
 618 }
 619
 620 static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
 621 {
 622         struct drm_crtc *crtc, *enabled = NULL;
 623
 624         for_each_crtc(dev, crtc) {
 625                 if (intel_crtc_active(crtc)) {
 626                         if (enabled)
 627                                 return NULL;
 628                         enabled = crtc;
 629                 }
 630         }
 631
 632         return enabled;
 633 }
 634
 635 static void pineview_update_wm(struct drm_crtc *unused_crtc)
 636 {
 637         struct drm_device *dev = unused_crtc->dev;
 638         struct drm_i915_private *dev_priv = dev->dev_private;
 639         struct drm_crtc *crtc;
 640         const struct cxsr_latency *latency;
 641         u32 reg;
 642         unsigned long wm;
 643
 644         latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
 645                                          dev_priv->fsb_freq, dev_priv->mem_freq);
 646         if (!latency) {
 647                 DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
 648                 intel_set_memory_cxsr(dev_priv, false);
 649                 return;
 650         }
 651
 652         crtc = single_enabled_crtc(dev);
 653         if (crtc) {
 654                 const struct drm_display_mode *adjusted_mode;
 655                 int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
 656                 int clock;
 657
 658                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 659                 clock = adjusted_mode->crtc_clock;
 660
 661                 /* Display SR */
 662                 wm = intel_calculate_wm(clock, &pineview_display_wm,
 663                                         pineview_display_wm.fifo_size,
 664                                         pixel_size, latency->display_sr);
 665                 reg = I915_READ(DSPFW1);
 666                 reg &= ~DSPFW_SR_MASK;
 667                 reg |= FW_WM(wm, SR);
 668                 I915_WRITE(DSPFW1, reg);
 669                 DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
 670
 671                 /* cursor SR */
 672                 wm = intel_calculate_wm(clock, &pineview_cursor_wm,
 673                                         pineview_display_wm.fifo_size,
 674                                         pixel_size, latency->cursor_sr);
 675                 reg = I915_READ(DSPFW3);
 676                 reg &= ~DSPFW_CURSOR_SR_MASK;
 677                 reg |= FW_WM(wm, CURSOR_SR);
 678                 I915_WRITE(DSPFW3, reg);
 679
 680                 /* Display HPLL off SR */
 681                 wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
 682                                         pineview_display_hplloff_wm.fifo_size,
 683                                         pixel_size, latency->display_hpll_disable);
 684                 reg = I915_READ(DSPFW3);
 685                 reg &= ~DSPFW_HPLL_SR_MASK;
 686                 reg |= FW_WM(wm, HPLL_SR);
 687                 I915_WRITE(DSPFW3, reg);
 688
 689                 /* cursor HPLL off SR */
 690                 wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
 691                                         pineview_display_hplloff_wm.fifo_size,
 692                                         pixel_size, latency->cursor_hpll_disable);
 693                 reg = I915_READ(DSPFW3);
 694                 reg &= ~DSPFW_HPLL_CURSOR_MASK;
 695                 reg |= FW_WM(wm, HPLL_CURSOR);
 696                 I915_WRITE(DSPFW3, reg);
 697                 DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
 698
 699                 intel_set_memory_cxsr(dev_priv, true);
 700         } else {
 701                 intel_set_memory_cxsr(dev_priv, false);
 702         }
 703 }
 704
 705 static bool g4x_compute_wm0(struct drm_device *dev,
 706                             int plane,
 707                             const struct intel_watermark_params *display,
 708                             int display_latency_ns,
 709                             const struct intel_watermark_params *cursor,
 710                             int cursor_latency_ns,
 711                             int *plane_wm,
 712                             int *cursor_wm)
 713 {
 714         struct drm_crtc *crtc;
 715         const struct drm_display_mode *adjusted_mode;
 716         int htotal, hdisplay, clock, pixel_size;
 717         int line_time_us, line_count;
 718         int entries, tlb_miss;
 719
 720         crtc = intel_get_crtc_for_plane(dev, plane);
 721         if (!intel_crtc_active(crtc)) {
 722                 *cursor_wm = cursor->guard_size;
 723                 *plane_wm = display->guard_size;
 724                 return false;
 725         }
 726
 727         adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 728         clock = adjusted_mode->crtc_clock;
 729         htotal = adjusted_mode->crtc_htotal;
 730         hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
 731         pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
 732
 733         /* Use the small buffer method to calculate plane watermark */
 734         entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
 735         tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
 736         if (tlb_miss > 0)
 737                 entries += tlb_miss;
 738         entries = DIV_ROUND_UP(entries, display->cacheline_size);
 739         *plane_wm = entries + display->guard_size;
 740         if (*plane_wm > (int)display->max_wm)
 741                 *plane_wm = display->max_wm;
 742
 743         /* Use the large buffer method to calculate cursor watermark */
 744         line_time_us = max(htotal * 1000 / clock, 1);
 745         line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
 746         entries = line_count * crtc->cursor->state->crtc_w * pixel_size;
 747         tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
 748         if (tlb_miss > 0)
 749                 entries += tlb_miss;
 750         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
 751         *cursor_wm = entries + cursor->guard_size;
 752         if (*cursor_wm > (int)cursor->max_wm)
 753                 *cursor_wm = (int)cursor->max_wm;
 754
 755         return true;
 756 }
 757
 758 /*
 759  * Check the wm result.
 760  *
 761  * If any calculated watermark values is larger than the maximum value that
 762  * can be programmed into the associated watermark register, that watermark
 763  * must be disabled.
 764  */
 765 static bool g4x_check_srwm(struct drm_device *dev,
 766                            int display_wm, int cursor_wm,
 767                            const struct intel_watermark_params *display,
 768                            const struct intel_watermark_params *cursor)
 769 {
 770         DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
 771                       display_wm, cursor_wm);
 772
 773         if (display_wm > display->max_wm) {
 774                 DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
 775                               display_wm, display->max_wm);
 776                 return false;
 777         }
 778
 779         if (cursor_wm > cursor->max_wm) {
 780                 DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
 781                               cursor_wm, cursor->max_wm);
 782                 return false;
 783         }
 784
 785         if (!(display_wm || cursor_wm)) {
 786                 DRM_DEBUG_KMS("SR latency is 0, disabling\n");
 787                 return false;
 788         }
 789
 790         return true;
 791 }
 792
 793 static bool g4x_compute_srwm(struct drm_device *dev,
 794                              int plane,
 795                              int latency_ns,
 796                              const struct intel_watermark_params *display,
 797                              const struct intel_watermark_params *cursor,
 798                              int *display_wm, int *cursor_wm)
 799 {
 800         struct drm_crtc *crtc;
 801         const struct drm_display_mode *adjusted_mode;
 802         int hdisplay, htotal, pixel_size, clock;
 803         unsigned long line_time_us;
 804         int line_count, line_size;
 805         int small, large;
 806         int entries;
 807
 808         if (!latency_ns) {
 809                 *display_wm = *cursor_wm = 0;
 810                 return false;
 811         }
 812
 813         crtc = intel_get_crtc_for_plane(dev, plane);
 814         adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
 815         clock = adjusted_mode->crtc_clock;
 816         htotal = adjusted_mode->crtc_htotal;
 817         hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
 818         pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
 819
 820         line_time_us = max(htotal * 1000 / clock, 1);
 821         line_count = (latency_ns / line_time_us + 1000) / 1000;
 822         line_size = hdisplay * pixel_size;
 823
 824         /* Use the minimum of the small and large buffer method for primary */
 825         small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
 826         large = line_count * line_size;
 827
 828         entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
 829         *display_wm = entries + display->guard_size;
 830
 831         /* calculate the self-refresh watermark for display cursor */
 832         entries = line_count * pixel_size * crtc->cursor->state->crtc_w;
 833         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
 834         *cursor_wm = entries + cursor->guard_size;
 835
 836         return g4x_check_srwm(dev,
 837                               *display_wm, *cursor_wm,
 838                               display, cursor);
 839 }
 840
 841 #define FW_WM_VLV(value, plane) \
 842         (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
 843
 844 static void vlv_write_wm_values(struct intel_crtc *crtc,
 845                                 const struct vlv_wm_values *wm)
 846 {
 847         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 848         enum pipe pipe = crtc->pipe;
 849
 850         I915_WRITE(VLV_DDL(pipe),
 851                    (wm->ddl[pipe].cursor << DDL_CURSOR_SHIFT) |
 852                    (wm->ddl[pipe].sprite[1] << DDL_SPRITE_SHIFT(1)) |
 853                    (wm->ddl[pipe].sprite[0] << DDL_SPRITE_SHIFT(0)) |
 854                    (wm->ddl[pipe].primary << DDL_PLANE_SHIFT));
 855
 856         I915_WRITE(DSPFW1,
 857                    FW_WM(wm->sr.plane, SR) |
 858                    FW_WM(wm->pipe[PIPE_B].cursor, CURSORB) |
 859                    FW_WM_VLV(wm->pipe[PIPE_B].primary, PLANEB) |
 860                    FW_WM_VLV(wm->pipe[PIPE_A].primary, PLANEA));
 861         I915_WRITE(DSPFW2,
 862                    FW_WM_VLV(wm->pipe[PIPE_A].sprite[1], SPRITEB) |
 863                    FW_WM(wm->pipe[PIPE_A].cursor, CURSORA) |
 864                    FW_WM_VLV(wm->pipe[PIPE_A].sprite[0], SPRITEA));
 865         I915_WRITE(DSPFW3,
 866                    FW_WM(wm->sr.cursor, CURSOR_SR));
 867
 868         if (IS_CHERRYVIEW(dev_priv)) {
 869                 I915_WRITE(DSPFW7_CHV,
 870                            FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) |
 871                            FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
 872                 I915_WRITE(DSPFW8_CHV,
 873                            FW_WM_VLV(wm->pipe[PIPE_C].sprite[1], SPRITEF) |
 874                            FW_WM_VLV(wm->pipe[PIPE_C].sprite[0], SPRITEE));
 875                 I915_WRITE(DSPFW9_CHV,
 876                            FW_WM_VLV(wm->pipe[PIPE_C].primary, PLANEC) |
 877                            FW_WM(wm->pipe[PIPE_C].cursor, CURSORC));
 878                 I915_WRITE(DSPHOWM,
 879                            FW_WM(wm->sr.plane >> 9, SR_HI) |
 880                            FW_WM(wm->pipe[PIPE_C].sprite[1] >> 8, SPRITEF_HI) |
 881                            FW_WM(wm->pipe[PIPE_C].sprite[0] >> 8, SPRITEE_HI) |
 882                            FW_WM(wm->pipe[PIPE_C].primary >> 8, PLANEC_HI) |
 883                            FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) |
 884                            FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) |
 885                            FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) |
 886                            FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) |
 887                            FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) |
 888                            FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
 889         } else {
 890                 I915_WRITE(DSPFW7,
 891                            FW_WM_VLV(wm->pipe[PIPE_B].sprite[1], SPRITED) |
 892                            FW_WM_VLV(wm->pipe[PIPE_B].sprite[0], SPRITEC));
 893                 I915_WRITE(DSPHOWM,
 894                            FW_WM(wm->sr.plane >> 9, SR_HI) |
 895                            FW_WM(wm->pipe[PIPE_B].sprite[1] >> 8, SPRITED_HI) |
 896                            FW_WM(wm->pipe[PIPE_B].sprite[0] >> 8, SPRITEC_HI) |
 897                            FW_WM(wm->pipe[PIPE_B].primary >> 8, PLANEB_HI) |
 898                            FW_WM(wm->pipe[PIPE_A].sprite[1] >> 8, SPRITEB_HI) |
 899                            FW_WM(wm->pipe[PIPE_A].sprite[0] >> 8, SPRITEA_HI) |
 900                            FW_WM(wm->pipe[PIPE_A].primary >> 8, PLANEA_HI));
 901         }
 902
 903         POSTING_READ(DSPFW1);
 904
 905         dev_priv->wm.vlv = *wm;
 906 }
 907
 908 #undef FW_WM_VLV
 909
 910 static uint8_t vlv_compute_drain_latency(struct drm_crtc *crtc,
 911                                          struct drm_plane *plane)
 912 {
 913         struct drm_device *dev = crtc->dev;
 914         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 915         int entries, prec_mult, drain_latency, pixel_size;
 916         int clock = intel_crtc->config->base.adjusted_mode.crtc_clock;
 917         const int high_precision = IS_CHERRYVIEW(dev) ? 16 : 64;
 918
 919         /*
 920          * FIXME the plane might have an fb
 921          * but be invisible (eg. due to clipping)
 922          */
 923         if (!intel_crtc->active || !plane->state->fb)
 924                 return 0;
 925
 926         if (WARN(clock == 0, "Pixel clock is zero!\n"))
 927                 return 0;
 928
 929         pixel_size = drm_format_plane_cpp(plane->state->fb->pixel_format, 0);
 930
 931         if (WARN(pixel_size == 0, "Pixel size is zero!\n"))
 932                 return 0;
 933
 934         entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
 935
 936         prec_mult = high_precision;
 937         drain_latency = 64 * prec_mult * 4 / entries;
 938
 939         if (drain_latency > DRAIN_LATENCY_MASK) {
 940                 prec_mult /= 2;
 941                 drain_latency = 64 * prec_mult * 4 / entries;
 942         }
 943
 944         if (drain_latency > DRAIN_LATENCY_MASK)
 945                 drain_latency = DRAIN_LATENCY_MASK;
 946
 947         return drain_latency | (prec_mult == high_precision ?
 948                                 DDL_PRECISION_HIGH : DDL_PRECISION_LOW);
 949 }
 950
 951 static int vlv_compute_wm(struct intel_crtc *crtc,
 952                           struct intel_plane *plane,
 953                           int fifo_size)
 954 {
 955         int clock, entries, pixel_size;
 956
 957         /*
 958          * FIXME the plane might have an fb
 959          * but be invisible (eg. due to clipping)
 960          */
 961         if (!crtc->active || !plane->base.state->fb)
 962                 return 0;
 963
 964         pixel_size = drm_format_plane_cpp(plane->base.state->fb->pixel_format, 0);
 965         clock = crtc->config->base.adjusted_mode.crtc_clock;
 966
 967         entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
 968
 969         /*
 970          * Set up the watermark such that we don't start issuing memory
 971          * requests until we are within PND's max deadline value (256us).
 972          * Idea being to be idle as long as possible while still taking
 973          * advatange of PND's deadline scheduling. The limit of 8
 974          * cachelines (used when the FIFO will anyway drain in less time
 975          * than 256us) should match what we would be done if trickle
 976          * feed were enabled.
 977          */
 978         return fifo_size - clamp(DIV_ROUND_UP(256 * entries, 64), 0, fifo_size - 8);
 979 }
 980
 981 static bool vlv_compute_sr_wm(struct drm_device *dev,
 982                               struct vlv_wm_values *wm)
 983 {
 984         struct drm_i915_private *dev_priv = to_i915(dev);
 985         struct drm_crtc *crtc;
 986         enum pipe pipe = INVALID_PIPE;
 987         int num_planes = 0;
 988         int fifo_size = 0;
 989         struct intel_plane *plane;
 990
 991         wm->sr.cursor = wm->sr.plane = 0;
 992
 993         crtc = single_enabled_crtc(dev);
 994         /* maxfifo not supported on pipe C */
 995         if (crtc && to_intel_crtc(crtc)->pipe != PIPE_C) {
 996                 pipe = to_intel_crtc(crtc)->pipe;
 997                 num_planes = !!wm->pipe[pipe].primary +
 998                         !!wm->pipe[pipe].sprite[0] +
 999                         !!wm->pipe[pipe].sprite[1];
1000                 fifo_size = INTEL_INFO(dev_priv)->num_pipes * 512 - 1;
1001         }
1002
1003         if (fifo_size == 0 || num_planes > 1)
1004                 return false;
1005
1006         wm->sr.cursor = vlv_compute_wm(to_intel_crtc(crtc),
1007                                        to_intel_plane(crtc->cursor), 0x3f);
1008
1009         list_for_each_entry(plane, &dev->mode_config.plane_list, base.head) {
1010                 if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
1011                         continue;
1012
1013                 if (plane->pipe != pipe)
1014                         continue;
1015
1016                 wm->sr.plane = vlv_compute_wm(to_intel_crtc(crtc),
1017                                               plane, fifo_size);
1018                 if (wm->sr.plane != 0)
1019                         break;
1020         }
1021
1022         return true;
1023 }
1024
1025 static void valleyview_update_wm(struct drm_crtc *crtc)
1026 {
1027         struct drm_device *dev = crtc->dev;
1028         struct drm_i915_private *dev_priv = dev->dev_private;
1029         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1030         enum pipe pipe = intel_crtc->pipe;
1031         bool cxsr_enabled;
1032         struct vlv_wm_values wm = dev_priv->wm.vlv;
1033
1034         wm.ddl[pipe].primary = vlv_compute_drain_latency(crtc, crtc->primary);
1035         wm.pipe[pipe].primary = vlv_compute_wm(intel_crtc,
1036                                                to_intel_plane(crtc->primary),
1037                                                vlv_get_fifo_size(dev, pipe, 0));
1038
1039         wm.ddl[pipe].cursor = vlv_compute_drain_latency(crtc, crtc->cursor);
1040         wm.pipe[pipe].cursor = vlv_compute_wm(intel_crtc,
1041                                               to_intel_plane(crtc->cursor),
1042                                               0x3f);
1043
1044         cxsr_enabled = vlv_compute_sr_wm(dev, &wm);
1045
1046         if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0)
1047                 return;
1048
1049         DRM_DEBUG_KMS("Setting FIFO watermarks - %c: plane=%d, cursor=%d, "
1050                       "SR: plane=%d, cursor=%d\n", pipe_name(pipe),
1051                       wm.pipe[pipe].primary, wm.pipe[pipe].cursor,
1052                       wm.sr.plane, wm.sr.cursor);
1053
1054         /*
1055          * FIXME DDR DVFS introduces massive memory latencies which
1056          * are not known to system agent so any deadline specified
1057          * by the display may not be respected. To support DDR DVFS
1058          * the watermark code needs to be rewritten to essentially
1059          * bypass deadline mechanism and rely solely on the
1060          * watermarks. For now disable DDR DVFS.
1061          */
1062         if (IS_CHERRYVIEW(dev_priv))
1063                 chv_set_memory_dvfs(dev_priv, false);
1064
1065         if (!cxsr_enabled)
1066                 intel_set_memory_cxsr(dev_priv, false);
1067
1068         vlv_write_wm_values(intel_crtc, &wm);
1069
1070         if (cxsr_enabled)
1071                 intel_set_memory_cxsr(dev_priv, true);
1072 }
1073
1074 static void valleyview_update_sprite_wm(struct drm_plane *plane,
1075                                         struct drm_crtc *crtc,
1076                                         uint32_t sprite_width,
1077                                         uint32_t sprite_height,
1078                                         int pixel_size,
1079                                         bool enabled, bool scaled)
1080 {
1081         struct drm_device *dev = crtc->dev;
1082         struct drm_i915_private *dev_priv = dev->dev_private;
1083         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1084         enum pipe pipe = intel_crtc->pipe;
1085         int sprite = to_intel_plane(plane)->plane;
1086         bool cxsr_enabled;
1087         struct vlv_wm_values wm = dev_priv->wm.vlv;
1088
1089         if (enabled) {
1090                 wm.ddl[pipe].sprite[sprite] =
1091                         vlv_compute_drain_latency(crtc, plane);
1092
1093                 wm.pipe[pipe].sprite[sprite] =
1094                         vlv_compute_wm(intel_crtc,
1095                                        to_intel_plane(plane),
1096                                        vlv_get_fifo_size(dev, pipe, sprite+1));
1097         } else {
1098                 wm.ddl[pipe].sprite[sprite] = 0;
1099                 wm.pipe[pipe].sprite[sprite] = 0;
1100         }
1101
1102         cxsr_enabled = vlv_compute_sr_wm(dev, &wm);
1103
1104         if (memcmp(&wm, &dev_priv->wm.vlv, sizeof(wm)) == 0)
1105                 return;
1106
1107         DRM_DEBUG_KMS("Setting FIFO watermarks - %c: sprite %c=%d, "
1108                       "SR: plane=%d, cursor=%d\n", pipe_name(pipe),
1109                       sprite_name(pipe, sprite),
1110                       wm.pipe[pipe].sprite[sprite],
1111                       wm.sr.plane, wm.sr.cursor);
1112
1113         if (!cxsr_enabled)
1114                 intel_set_memory_cxsr(dev_priv, false);
1115
1116         vlv_write_wm_values(intel_crtc, &wm);
1117
1118         if (cxsr_enabled)
1119                 intel_set_memory_cxsr(dev_priv, true);
1120 }
1121
1122 #define single_plane_enabled(mask) is_power_of_2(mask)
1123
1124 static void g4x_update_wm(struct drm_crtc *crtc)
1125 {
1126         struct drm_device *dev = crtc->dev;
1127         static const int sr_latency_ns = 12000;
1128         struct drm_i915_private *dev_priv = dev->dev_private;
1129         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
1130         int plane_sr, cursor_sr;
1131         unsigned int enabled = 0;
1132         bool cxsr_enabled;
1133
1134         if (g4x_compute_wm0(dev, PIPE_A,
1135                             &g4x_wm_info, pessimal_latency_ns,
1136                             &g4x_cursor_wm_info, pessimal_latency_ns,
1137                             &planea_wm, &cursora_wm))
1138                 enabled |= 1 << PIPE_A;
1139
1140         if (g4x_compute_wm0(dev, PIPE_B,
1141                             &g4x_wm_info, pessimal_latency_ns,
1142                             &g4x_cursor_wm_info, pessimal_latency_ns,
1143                             &planeb_wm, &cursorb_wm))
1144                 enabled |= 1 << PIPE_B;
1145
1146         if (single_plane_enabled(enabled) &&
1147             g4x_compute_srwm(dev, ffs(enabled) - 1,
1148                              sr_latency_ns,
1149                              &g4x_wm_info,
1150                              &g4x_cursor_wm_info,
1151                              &plane_sr, &cursor_sr)) {
1152                 cxsr_enabled = true;
1153         } else {
1154                 cxsr_enabled = false;
1155                 intel_set_memory_cxsr(dev_priv, false);
1156                 plane_sr = cursor_sr = 0;
1157         }
1158
1159         DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
1160                       "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
1161                       planea_wm, cursora_wm,
1162                       planeb_wm, cursorb_wm,
1163                       plane_sr, cursor_sr);
1164
1165         I915_WRITE(DSPFW1,
1166                    FW_WM(plane_sr, SR) |
1167                    FW_WM(cursorb_wm, CURSORB) |
1168                    FW_WM(planeb_wm, PLANEB) |
1169                    FW_WM(planea_wm, PLANEA));
1170         I915_WRITE(DSPFW2,
1171                    (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
1172                    FW_WM(cursora_wm, CURSORA));
1173         /* HPLL off in SR has some issues on G4x... disable it */
1174         I915_WRITE(DSPFW3,
1175                    (I915_READ(DSPFW3) & ~(DSPFW_HPLL_SR_EN | DSPFW_CURSOR_SR_MASK)) |
1176                    FW_WM(cursor_sr, CURSOR_SR));
1177
1178         if (cxsr_enabled)
1179                 intel_set_memory_cxsr(dev_priv, true);
1180 }
1181
1182 static void i965_update_wm(struct drm_crtc *unused_crtc)
1183 {
1184         struct drm_device *dev = unused_crtc->dev;
1185         struct drm_i915_private *dev_priv = dev->dev_private;
1186         struct drm_crtc *crtc;
1187         int srwm = 1;
1188         int cursor_sr = 16;
1189         bool cxsr_enabled;
1190
1191         /* Calc sr entries for one plane configs */
1192         crtc = single_enabled_crtc(dev);
1193         if (crtc) {
1194                 /* self-refresh has much higher latency */
1195                 static const int sr_latency_ns = 12000;
1196                 const struct drm_display_mode *adjusted_mode =
1197                         &to_intel_crtc(crtc)->config->base.adjusted_mode;
1198                 int clock = adjusted_mode->crtc_clock;
1199                 int htotal = adjusted_mode->crtc_htotal;
1200                 int hdisplay = to_intel_crtc(crtc)->config->pipe_src_w;
1201                 int pixel_size = crtc->primary->state->fb->bits_per_pixel / 8;
1202                 unsigned long line_time_us;
1203                 int entries;
1204
1205                 line_time_us = max(htotal * 1000 / clock, 1);
1206
1207                 /* Use ns/us then divide to preserve precision */
1208                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1209                         pixel_size * hdisplay;
1210                 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
1211                 srwm = I965_FIFO_SIZE - entries;
1212                 if (srwm < 0)
1213                         srwm = 1;
1214                 srwm &= 0x1ff;
1215                 DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
1216                               entries, srwm);
1217
1218                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1219                         pixel_size * crtc->cursor->state->crtc_w;
1220                 entries = DIV_ROUND_UP(entries,
1221                                           i965_cursor_wm_info.cacheline_size);
1222                 cursor_sr = i965_cursor_wm_info.fifo_size -
1223                         (entries + i965_cursor_wm_info.guard_size);
1224
1225                 if (cursor_sr > i965_cursor_wm_info.max_wm)
1226                         cursor_sr = i965_cursor_wm_info.max_wm;
1227
1228                 DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
1229                               "cursor %d\n", srwm, cursor_sr);
1230
1231                 cxsr_enabled = true;
1232         } else {
1233                 cxsr_enabled = false;
1234                 /* Turn off self refresh if both pipes are enabled */
1235                 intel_set_memory_cxsr(dev_priv, false);
1236         }
1237
1238         DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
1239                       srwm);
1240
1241         /* 965 has limitations... */
1242         I915_WRITE(DSPFW1, FW_WM(srwm, SR) |
1243                    FW_WM(8, CURSORB) |
1244                    FW_WM(8, PLANEB) |
1245                    FW_WM(8, PLANEA));
1246         I915_WRITE(DSPFW2, FW_WM(8, CURSORA) |
1247                    FW_WM(8, PLANEC_OLD));
1248         /* update cursor SR watermark */
1249         I915_WRITE(DSPFW3, FW_WM(cursor_sr, CURSOR_SR));
1250
1251         if (cxsr_enabled)
1252                 intel_set_memory_cxsr(dev_priv, true);
1253 }
1254
1255 #undef FW_WM
1256
1257 static void i9xx_update_wm(struct drm_crtc *unused_crtc)
1258 {
1259         struct drm_device *dev = unused_crtc->dev;
1260         struct drm_i915_private *dev_priv = dev->dev_private;
1261         const struct intel_watermark_params *wm_info;
1262         uint32_t fwater_lo;
1263         uint32_t fwater_hi;
1264         int cwm, srwm = 1;
1265         int fifo_size;
1266         int planea_wm, planeb_wm;
1267         struct drm_crtc *crtc, *enabled = NULL;
1268
1269         if (IS_I945GM(dev))
1270                 wm_info = &i945_wm_info;
1271         else if (!IS_GEN2(dev))
1272                 wm_info = &i915_wm_info;
1273         else
1274                 wm_info = &i830_a_wm_info;
1275
1276         fifo_size = dev_priv->display.get_fifo_size(dev, 0);
1277         crtc = intel_get_crtc_for_plane(dev, 0);
1278         if (intel_crtc_active(crtc)) {
1279                 const struct drm_display_mode *adjusted_mode;
1280                 int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
1281                 if (IS_GEN2(dev))
1282                         cpp = 4;
1283
1284                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1285                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1286                                                wm_info, fifo_size, cpp,
1287                                                pessimal_latency_ns);
1288                 enabled = crtc;
1289         } else {
1290                 planea_wm = fifo_size - wm_info->guard_size;
1291                 if (planea_wm > (long)wm_info->max_wm)
1292                         planea_wm = wm_info->max_wm;
1293         }
1294
1295         if (IS_GEN2(dev))
1296                 wm_info = &i830_bc_wm_info;
1297
1298         fifo_size = dev_priv->display.get_fifo_size(dev, 1);
1299         crtc = intel_get_crtc_for_plane(dev, 1);
1300         if (intel_crtc_active(crtc)) {
1301                 const struct drm_display_mode *adjusted_mode;
1302                 int cpp = crtc->primary->state->fb->bits_per_pixel / 8;
1303                 if (IS_GEN2(dev))
1304                         cpp = 4;
1305
1306                 adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1307                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1308                                                wm_info, fifo_size, cpp,
1309                                                pessimal_latency_ns);
1310                 if (enabled == NULL)
1311                         enabled = crtc;
1312                 else
1313                         enabled = NULL;
1314         } else {
1315                 planeb_wm = fifo_size - wm_info->guard_size;
1316                 if (planeb_wm > (long)wm_info->max_wm)
1317                         planeb_wm = wm_info->max_wm;
1318         }
1319
1320         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
1321
1322         if (IS_I915GM(dev) && enabled) {
1323                 struct drm_i915_gem_object *obj;
1324
1325                 obj = intel_fb_obj(enabled->primary->state->fb);
1326
1327                 /* self-refresh seems busted with untiled */
1328                 if (obj->tiling_mode == I915_TILING_NONE)
1329                         enabled = NULL;
1330         }
1331
1332         /*
1333          * Overlay gets an aggressive default since video jitter is bad.
1334          */
1335         cwm = 2;
1336
1337         /* Play safe and disable self-refresh before adjusting watermarks. */
1338         intel_set_memory_cxsr(dev_priv, false);
1339
1340         /* Calc sr entries for one plane configs */
1341         if (HAS_FW_BLC(dev) && enabled) {
1342                 /* self-refresh has much higher latency */
1343                 static const int sr_latency_ns = 6000;
1344                 const struct drm_display_mode *adjusted_mode =
1345                         &to_intel_crtc(enabled)->config->base.adjusted_mode;
1346                 int clock = adjusted_mode->crtc_clock;
1347                 int htotal = adjusted_mode->crtc_htotal;
1348                 int hdisplay = to_intel_crtc(enabled)->config->pipe_src_w;
1349                 int pixel_size = enabled->primary->state->fb->bits_per_pixel / 8;
1350                 unsigned long line_time_us;
1351                 int entries;
1352
1353                 line_time_us = max(htotal * 1000 / clock, 1);
1354
1355                 /* Use ns/us then divide to preserve precision */
1356                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
1357                         pixel_size * hdisplay;
1358                 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
1359                 DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
1360                 srwm = wm_info->fifo_size - entries;
1361                 if (srwm < 0)
1362                         srwm = 1;
1363
1364                 if (IS_I945G(dev) || IS_I945GM(dev))
1365                         I915_WRITE(FW_BLC_SELF,
1366                                    FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
1367                 else if (IS_I915GM(dev))
1368                         I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
1369         }
1370
1371         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
1372                       planea_wm, planeb_wm, cwm, srwm);
1373
1374         fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
1375         fwater_hi = (cwm & 0x1f);
1376
1377         /* Set request length to 8 cachelines per fetch */
1378         fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
1379         fwater_hi = fwater_hi | (1 << 8);
1380
1381         I915_WRITE(FW_BLC, fwater_lo);
1382         I915_WRITE(FW_BLC2, fwater_hi);
1383
1384         if (enabled)
1385                 intel_set_memory_cxsr(dev_priv, true);
1386 }
1387
1388 static void i845_update_wm(struct drm_crtc *unused_crtc)
1389 {
1390         struct drm_device *dev = unused_crtc->dev;
1391         struct drm_i915_private *dev_priv = dev->dev_private;
1392         struct drm_crtc *crtc;
1393         const struct drm_display_mode *adjusted_mode;
1394         uint32_t fwater_lo;
1395         int planea_wm;
1396
1397         crtc = single_enabled_crtc(dev);
1398         if (crtc == NULL)
1399                 return;
1400
1401         adjusted_mode = &to_intel_crtc(crtc)->config->base.adjusted_mode;
1402         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
1403                                        &i845_wm_info,
1404                                        dev_priv->display.get_fifo_size(dev, 0),
1405                                        4, pessimal_latency_ns);
1406         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
1407         fwater_lo |= (3<<8) | planea_wm;
1408
1409         DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
1410
1411         I915_WRITE(FW_BLC, fwater_lo);
1412 }
1413
1414 static uint32_t ilk_pipe_pixel_rate(struct drm_device *dev,
1415                                     struct drm_crtc *crtc)
1416 {
1417         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1418         uint32_t pixel_rate;
1419
1420         pixel_rate = intel_crtc->config->base.adjusted_mode.crtc_clock;
1421
1422         /* We only use IF-ID interlacing. If we ever use PF-ID we'll need to
1423          * adjust the pixel_rate here. */
1424
1425         if (intel_crtc->config->pch_pfit.enabled) {
1426                 uint64_t pipe_w, pipe_h, pfit_w, pfit_h;
1427                 uint32_t pfit_size = intel_crtc->config->pch_pfit.size;
1428
1429                 pipe_w = intel_crtc->config->pipe_src_w;
1430                 pipe_h = intel_crtc->config->pipe_src_h;
1431                 pfit_w = (pfit_size >> 16) & 0xFFFF;
1432                 pfit_h = pfit_size & 0xFFFF;
1433                 if (pipe_w < pfit_w)
1434                         pipe_w = pfit_w;
1435                 if (pipe_h < pfit_h)
1436                         pipe_h = pfit_h;
1437
1438                 pixel_rate = div_u64((uint64_t) pixel_rate * pipe_w * pipe_h,
1439                                      pfit_w * pfit_h);
1440         }
1441
1442         return pixel_rate;
1443 }
1444
1445 /* latency must be in 0.1us units. */
1446 static uint32_t ilk_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
1447                                uint32_t latency)
1448 {
1449         uint64_t ret;
1450
1451         if (WARN(latency == 0, "Latency value missing\n"))
1452                 return UINT_MAX;
1453
1454         ret = (uint64_t) pixel_rate * bytes_per_pixel * latency;
1455         ret = DIV_ROUND_UP_ULL(ret, 64 * 10000) + 2;
1456
1457         return ret;
1458 }
1459
1460 /* latency must be in 0.1us units. */
1461 static uint32_t ilk_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
1462                                uint32_t horiz_pixels, uint8_t bytes_per_pixel,
1463                                uint32_t latency)
1464 {
1465         uint32_t ret;
1466
1467         if (WARN(latency == 0, "Latency value missing\n"))
1468                 return UINT_MAX;
1469
1470         ret = (latency * pixel_rate) / (pipe_htotal * 10000);
1471         ret = (ret + 1) * horiz_pixels * bytes_per_pixel;
1472         ret = DIV_ROUND_UP(ret, 64) + 2;
1473         return ret;
1474 }
1475
1476 static uint32_t ilk_wm_fbc(uint32_t pri_val, uint32_t horiz_pixels,
1477                            uint8_t bytes_per_pixel)
1478 {
1479         return DIV_ROUND_UP(pri_val * 64, horiz_pixels * bytes_per_pixel) + 2;
1480 }
1481
1482 struct skl_pipe_wm_parameters {
1483         bool active;
1484         uint32_t pipe_htotal;
1485         uint32_t pixel_rate; /* in KHz */
1486         struct intel_plane_wm_parameters plane[I915_MAX_PLANES];
1487         struct intel_plane_wm_parameters cursor;
1488 };
1489
1490 struct ilk_pipe_wm_parameters {
1491         bool active;
1492         uint32_t pipe_htotal;
1493         uint32_t pixel_rate;
1494         struct intel_plane_wm_parameters pri;
1495         struct intel_plane_wm_parameters spr;
1496         struct intel_plane_wm_parameters cur;
1497 };
1498
1499 struct ilk_wm_maximums {
1500         uint16_t pri;
1501         uint16_t spr;
1502         uint16_t cur;
1503         uint16_t fbc;
1504 };
1505
1506 /* used in computing the new watermarks state */
1507 struct intel_wm_config {
1508         unsigned int num_pipes_active;
1509         bool sprites_enabled;
1510         bool sprites_scaled;
1511 };
1512
1513 /*
1514  * For both WM_PIPE and WM_LP.
1515  * mem_value must be in 0.1us units.
1516  */
1517 static uint32_t ilk_compute_pri_wm(const struct ilk_pipe_wm_parameters *params,
1518                                    uint32_t mem_value,
1519                                    bool is_lp)
1520 {
1521         uint32_t method1, method2;
1522
1523         if (!params->active || !params->pri.enabled)
1524                 return 0;
1525
1526         method1 = ilk_wm_method1(params->pixel_rate,
1527                                  params->pri.bytes_per_pixel,
1528                                  mem_value);
1529
1530         if (!is_lp)
1531                 return method1;
1532
1533         method2 = ilk_wm_method2(params->pixel_rate,
1534                                  params->pipe_htotal,
1535                                  params->pri.horiz_pixels,
1536                                  params->pri.bytes_per_pixel,
1537                                  mem_value);
1538
1539         return min(method1, method2);
1540 }
1541
1542 /*
1543  * For both WM_PIPE and WM_LP.
1544  * mem_value must be in 0.1us units.
1545  */
1546 static uint32_t ilk_compute_spr_wm(const struct ilk_pipe_wm_parameters *params,
1547                                    uint32_t mem_value)
1548 {
1549         uint32_t method1, method2;
1550
1551         if (!params->active || !params->spr.enabled)
1552                 return 0;
1553
1554         method1 = ilk_wm_method1(params->pixel_rate,
1555                                  params->spr.bytes_per_pixel,
1556                                  mem_value);
1557         method2 = ilk_wm_method2(params->pixel_rate,
1558                                  params->pipe_htotal,
1559                                  params->spr.horiz_pixels,
1560                                  params->spr.bytes_per_pixel,
1561                                  mem_value);
1562         return min(method1, method2);
1563 }
1564
1565 /*
1566  * For both WM_PIPE and WM_LP.
1567  * mem_value must be in 0.1us units.
1568  */
1569 static uint32_t ilk_compute_cur_wm(const struct ilk_pipe_wm_parameters *params,
1570                                    uint32_t mem_value)
1571 {
1572         if (!params->active || !params->cur.enabled)
1573                 return 0;
1574
1575         return ilk_wm_method2(params->pixel_rate,
1576                               params->pipe_htotal,
1577                               params->cur.horiz_pixels,
1578                               params->cur.bytes_per_pixel,
1579                               mem_value);
1580 }
1581
1582 /* Only for WM_LP. */
1583 static uint32_t ilk_compute_fbc_wm(const struct ilk_pipe_wm_parameters *params,
1584                                    uint32_t pri_val)
1585 {
1586         if (!params->active || !params->pri.enabled)
1587                 return 0;
1588
1589         return ilk_wm_fbc(pri_val,
1590                           params->pri.horiz_pixels,
1591                           params->pri.bytes_per_pixel);
1592 }
1593
1594 static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
1595 {
1596         if (INTEL_INFO(dev)->gen >= 8)
1597                 return 3072;
1598         else if (INTEL_INFO(dev)->gen >= 7)
1599                 return 768;
1600         else
1601                 return 512;
1602 }
1603
1604 static unsigned int ilk_plane_wm_reg_max(const struct drm_device *dev,
1605                                          int level, bool is_sprite)
1606 {
1607         if (INTEL_INFO(dev)->gen >= 8)
1608                 /* BDW primary/sprite plane watermarks */
1609                 return level == 0 ? 255 : 2047;
1610         else if (INTEL_INFO(dev)->gen >= 7)
1611                 /* IVB/HSW primary/sprite plane watermarks */
1612                 return level == 0 ? 127 : 1023;
1613         else if (!is_sprite)
1614                 /* ILK/SNB primary plane watermarks */
1615                 return level == 0 ? 127 : 511;
1616         else
1617                 /* ILK/SNB sprite plane watermarks */
1618                 return level == 0 ? 63 : 255;
1619 }
1620
1621 static unsigned int ilk_cursor_wm_reg_max(const struct drm_device *dev,
1622                                           int level)
1623 {
1624         if (INTEL_INFO(dev)->gen >= 7)
1625                 return level == 0 ? 63 : 255;
1626         else
1627                 return level == 0 ? 31 : 63;
1628 }
1629
1630 static unsigned int ilk_fbc_wm_reg_max(const struct drm_device *dev)
1631 {
1632         if (INTEL_INFO(dev)->gen >= 8)
1633                 return 31;
1634         else
1635                 return 15;
1636 }
1637
1638 /* Calculate the maximum primary/sprite plane watermark */
1639 static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
1640                                      int level,
1641                                      const struct intel_wm_config *config,
1642                                      enum intel_ddb_partitioning ddb_partitioning,
1643                                      bool is_sprite)
1644 {
1645         unsigned int fifo_size = ilk_display_fifo_size(dev);
1646
1647         /* if sprites aren't enabled, sprites get nothing */
1648         if (is_sprite && !config->sprites_enabled)
1649                 return 0;
1650
1651         /* HSW allows LP1+ watermarks even with multiple pipes */
1652         if (level == 0 || config->num_pipes_active > 1) {
1653                 fifo_size /= INTEL_INFO(dev)->num_pipes;
1654
1655                 /*
1656                  * For some reason the non self refresh
1657                  * FIFO size is only half of the self
1658                  * refresh FIFO size on ILK/SNB.
1659                  */
1660                 if (INTEL_INFO(dev)->gen <= 6)
1661                         fifo_size /= 2;
1662         }
1663
1664         if (config->sprites_enabled) {
1665                 /* level 0 is always calculated with 1:1 split */
1666                 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
1667                         if (is_sprite)
1668                                 fifo_size *= 5;
1669                         fifo_size /= 6;
1670                 } else {
1671                         fifo_size /= 2;
1672                 }
1673         }
1674
1675         /* clamp to max that the registers can hold */
1676         return min(fifo_size, ilk_plane_wm_reg_max(dev, level, is_sprite));
1677 }
1678
1679 /* Calculate the maximum cursor plane watermark */
1680 static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
1681                                       int level,
1682                                       const struct intel_wm_config *config)
1683 {
1684         /* HSW LP1+ watermarks w/ multiple pipes */
1685         if (level > 0 && config->num_pipes_active > 1)
1686                 return 64;
1687
1688         /* otherwise just report max that registers can hold */
1689         return ilk_cursor_wm_reg_max(dev, level);
1690 }
1691
1692 static void ilk_compute_wm_maximums(const struct drm_device *dev,
1693                                     int level,
1694                                     const struct intel_wm_config *config,
1695                                     enum intel_ddb_partitioning ddb_partitioning,
1696                                     struct ilk_wm_maximums *max)
1697 {
1698         max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
1699         max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
1700         max->cur = ilk_cursor_wm_max(dev, level, config);
1701         max->fbc = ilk_fbc_wm_reg_max(dev);
1702 }
1703
1704 static void ilk_compute_wm_reg_maximums(struct drm_device *dev,
1705                                         int level,
1706                                         struct ilk_wm_maximums *max)
1707 {
1708         max->pri = ilk_plane_wm_reg_max(dev, level, false);
1709         max->spr = ilk_plane_wm_reg_max(dev, level, true);
1710         max->cur = ilk_cursor_wm_reg_max(dev, level);
1711         max->fbc = ilk_fbc_wm_reg_max(dev);
1712 }
1713
1714 static bool ilk_validate_wm_level(int level,
1715                                   const struct ilk_wm_maximums *max,
1716                                   struct intel_wm_level *result)
1717 {
1718         bool ret;
1719
1720         /* already determined to be invalid? */
1721         if (!result->enable)
1722                 return false;
1723
1724         result->enable = result->pri_val <= max->pri &&
1725                          result->spr_val <= max->spr &&
1726                          result->cur_val <= max->cur;
1727
1728         ret = result->enable;
1729
1730         /*
1731          * HACK until we can pre-compute everything,
1732          * and thus fail gracefully if LP0 watermarks
1733          * are exceeded...
1734          */
1735         if (level == 0 && !result->enable) {
1736                 if (result->pri_val > max->pri)
1737                         DRM_DEBUG_KMS("Primary WM%d too large %u (max %u)\n",
1738                                       level, result->pri_val, max->pri);
1739                 if (result->spr_val > max->spr)
1740                         DRM_DEBUG_KMS("Sprite WM%d too large %u (max %u)\n",
1741                                       level, result->spr_val, max->spr);
1742                 if (result->cur_val > max->cur)
1743                         DRM_DEBUG_KMS("Cursor WM%d too large %u (max %u)\n",
1744                                       level, result->cur_val, max->cur);
1745
1746                 result->pri_val = min_t(uint32_t, result->pri_val, max->pri);
1747                 result->spr_val = min_t(uint32_t, result->spr_val, max->spr);
1748                 result->cur_val = min_t(uint32_t, result->cur_val, max->cur);
1749                 result->enable = true;
1750         }
1751
1752         return ret;
1753 }
1754
1755 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
1756                                  int level,
1757                                  const struct ilk_pipe_wm_parameters *p,
1758                                  struct intel_wm_level *result)
1759 {
1760         uint16_t pri_latency = dev_priv->wm.pri_latency[level];
1761         uint16_t spr_latency = dev_priv->wm.spr_latency[level];
1762         uint16_t cur_latency = dev_priv->wm.cur_latency[level];
1763
1764         /* WM1+ latency values stored in 0.5us units */
1765         if (level > 0) {
1766                 pri_latency *= 5;
1767                 spr_latency *= 5;
1768                 cur_latency *= 5;
1769         }
1770
1771         result->pri_val = ilk_compute_pri_wm(p, pri_latency, level);
1772         result->spr_val = ilk_compute_spr_wm(p, spr_latency);
1773         result->cur_val = ilk_compute_cur_wm(p, cur_latency);
1774         result->fbc_val = ilk_compute_fbc_wm(p, result->pri_val);
1775         result->enable = true;
1776 }
1777
1778 static uint32_t
1779 hsw_compute_linetime_wm(struct drm_device *dev, struct drm_crtc *crtc)
1780 {
1781         struct drm_i915_private *dev_priv = dev->dev_private;
1782         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1783         struct drm_display_mode *mode = &intel_crtc->config->base.adjusted_mode;
1784         u32 linetime, ips_linetime;
1785
1786         if (!intel_crtc->active)
1787                 return 0;
1788
1789         /* The WM are computed with base on how long it takes to fill a single
1790          * row at the given clock rate, multiplied by 8.
1791          * */
1792         linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
1793                                      mode->crtc_clock);
1794         ips_linetime = DIV_ROUND_CLOSEST(mode->crtc_htotal * 1000 * 8,
1795                                          intel_ddi_get_cdclk_freq(dev_priv));
1796
1797         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
1798                PIPE_WM_LINETIME_TIME(linetime);
1799 }
1800
1801 static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
1802 {
1803         struct drm_i915_private *dev_priv = dev->dev_private;
1804
1805         if (IS_GEN9(dev)) {
1806                 uint32_t val;
1807                 int ret, i;
1808                 int level, max_level = ilk_wm_max_level(dev);
1809
1810                 /* read the first set of memory latencies[0:3] */
1811                 val = 0; /* data0 to be programmed to 0 for first set */
1812                 mutex_lock(&dev_priv->rps.hw_lock);
1813                 ret = sandybridge_pcode_read(dev_priv,
1814                                              GEN9_PCODE_READ_MEM_LATENCY,
1815                                              &val);
1816                 mutex_unlock(&dev_priv->rps.hw_lock);
1817
1818                 if (ret) {
1819                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
1820                         return;
1821                 }
1822
1823                 wm[0] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
1824                 wm[1] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
1825                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1826                 wm[2] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
1827                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1828                 wm[3] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
1829                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1830
1831                 /* read the second set of memory latencies[4:7] */
1832                 val = 1; /* data0 to be programmed to 1 for second set */
1833                 mutex_lock(&dev_priv->rps.hw_lock);
1834                 ret = sandybridge_pcode_read(dev_priv,
1835                                              GEN9_PCODE_READ_MEM_LATENCY,
1836                                              &val);
1837                 mutex_unlock(&dev_priv->rps.hw_lock);
1838                 if (ret) {
1839                         DRM_ERROR("SKL Mailbox read error = %d\n", ret);
1840                         return;
1841                 }
1842
1843                 wm[4] = val & GEN9_MEM_LATENCY_LEVEL_MASK;
1844                 wm[5] = (val >> GEN9_MEM_LATENCY_LEVEL_1_5_SHIFT) &
1845                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1846                 wm[6] = (val >> GEN9_MEM_LATENCY_LEVEL_2_6_SHIFT) &
1847                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1848                 wm[7] = (val >> GEN9_MEM_LATENCY_LEVEL_3_7_SHIFT) &
1849                                 GEN9_MEM_LATENCY_LEVEL_MASK;
1850
1851                 /*
1852                  * WaWmMemoryReadLatency:skl
1853                  *
1854                  * punit doesn't take into account the read latency so we need
1855                  * to add 2us to the various latency levels we retrieve from
1856                  * the punit.
1857                  *   - W0 is a bit special in that it's the only level that
1858                  *   can't be disabled if we want to have display working, so
1859                  *   we always add 2us there.
1860                  *   - For levels >=1, punit returns 0us latency when they are
1861                  *   disabled, so we respect that and don't add 2us then
1862                  *
1863                  * Additionally, if a level n (n > 1) has a 0us latency, all
1864                  * levels m (m >= n) need to be disabled. We make sure to
1865                  * sanitize the values out of the punit to satisfy this
1866                  * requirement.
1867                  */
1868                 wm[0] += 2;
1869                 for (level = 1; level <= max_level; level++)
1870                         if (wm[level] != 0)
1871                                 wm[level] += 2;
1872                         else {
1873                                 for (i = level + 1; i <= max_level; i++)
1874                                         wm[i] = 0;
1875
1876                                 break;
1877                         }
1878         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
1879                 uint64_t sskpd = I915_READ64(MCH_SSKPD);
1880
1881                 wm[0] = (sskpd >> 56) & 0xFF;
1882                 if (wm[0] == 0)
1883                         wm[0] = sskpd & 0xF;
1884                 wm[1] = (sskpd >> 4) & 0xFF;
1885                 wm[2] = (sskpd >> 12) & 0xFF;
1886                 wm[3] = (sskpd >> 20) & 0x1FF;
1887                 wm[4] = (sskpd >> 32) & 0x1FF;
1888         } else if (INTEL_INFO(dev)->gen >= 6) {
1889                 uint32_t sskpd = I915_READ(MCH_SSKPD);
1890
1891                 wm[0] = (sskpd >> SSKPD_WM0_SHIFT) & SSKPD_WM_MASK;
1892                 wm[1] = (sskpd >> SSKPD_WM1_SHIFT) & SSKPD_WM_MASK;
1893                 wm[2] = (sskpd >> SSKPD_WM2_SHIFT) & SSKPD_WM_MASK;
1894                 wm[3] = (sskpd >> SSKPD_WM3_SHIFT) & SSKPD_WM_MASK;
1895         } else if (INTEL_INFO(dev)->gen >= 5) {
1896                 uint32_t mltr = I915_READ(MLTR_ILK);
1897
1898                 /* ILK primary LP0 latency is 700 ns */
1899                 wm[0] = 7;
1900                 wm[1] = (mltr >> MLTR_WM1_SHIFT) & ILK_SRLT_MASK;
1901                 wm[2] = (mltr >> MLTR_WM2_SHIFT) & ILK_SRLT_MASK;
1902         }
1903 }
1904
1905 static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
1906 {
1907         /* ILK sprite LP0 latency is 1300 ns */
1908         if (INTEL_INFO(dev)->gen == 5)
1909                 wm[0] = 13;
1910 }
1911
1912 static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
1913 {
1914         /* ILK cursor LP0 latency is 1300 ns */
1915         if (INTEL_INFO(dev)->gen == 5)
1916                 wm[0] = 13;
1917
1918         /* WaDoubleCursorLP3Latency:ivb */
1919         if (IS_IVYBRIDGE(dev))
1920                 wm[3] *= 2;
1921 }
1922
1923 int ilk_wm_max_level(const struct drm_device *dev)
1924 {
1925         /* how many WM levels are we expecting */
1926         if (IS_GEN9(dev))
1927                 return 7;
1928         else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
1929                 return 4;
1930         else if (INTEL_INFO(dev)->gen >= 6)
1931                 return 3;
1932         else
1933                 return 2;
1934 }
1935
1936 static void intel_print_wm_latency(struct drm_device *dev,
1937                                    const char *name,
1938                                    const uint16_t wm[8])
1939 {
1940         int level, max_level = ilk_wm_max_level(dev);
1941
1942         for (level = 0; level <= max_level; level++) {
1943                 unsigned int latency = wm[level];
1944
1945                 if (latency == 0) {
1946                         DRM_ERROR("%s WM%d latency not provided\n",
1947                                   name, level);
1948                         continue;
1949                 }
1950
1951                 /*
1952                  * - latencies are in us on gen9.
1953                  * - before then, WM1+ latency values are in 0.5us units
1954                  */
1955                 if (IS_GEN9(dev))
1956                         latency *= 10;
1957                 else if (level > 0)
1958                         latency *= 5;
1959
1960                 DRM_DEBUG_KMS("%s WM%d latency %u (%u.%u usec)\n",
1961                               name, level, wm[level],
1962                               latency / 10, latency % 10);
1963         }
1964 }
1965
1966 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
1967                                     uint16_t wm[5], uint16_t min)
1968 {
1969         int level, max_level = ilk_wm_max_level(dev_priv->dev);
1970
1971         if (wm[0] >= min)
1972                 return false;
1973
1974         wm[0] = max(wm[0], min);
1975         for (level = 1; level <= max_level; level++)
1976                 wm[level] = max_t(uint16_t, wm[level], DIV_ROUND_UP(min, 5));
1977
1978         return true;
1979 }
1980
1981 static void snb_wm_latency_quirk(struct drm_device *dev)
1982 {
1983         struct drm_i915_private *dev_priv = dev->dev_private;
1984         bool changed;
1985
1986         /*
1987          * The BIOS provided WM memory latency values are often
1988          * inadequate for high resolution displays. Adjust them.
1989          */
1990         changed = ilk_increase_wm_latency(dev_priv, dev_priv->wm.pri_latency, 12) |
1991                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.spr_latency, 12) |
1992                 ilk_increase_wm_latency(dev_priv, dev_priv->wm.cur_latency, 12);
1993
1994         if (!changed)
1995                 return;
1996
1997         DRM_DEBUG_KMS("WM latency values increased to avoid potential underruns\n");
1998         intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
1999         intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2000         intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2001 }
2002
2003 static void ilk_setup_wm_latency(struct drm_device *dev)
2004 {
2005         struct drm_i915_private *dev_priv = dev->dev_private;
2006
2007         intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
2008
2009         memcpy(dev_priv->wm.spr_latency, dev_priv->wm.pri_latency,
2010                sizeof(dev_priv->wm.pri_latency));
2011         memcpy(dev_priv->wm.cur_latency, dev_priv->wm.pri_latency,
2012                sizeof(dev_priv->wm.pri_latency));
2013
2014         intel_fixup_spr_wm_latency(dev, dev_priv->wm.spr_latency);
2015         intel_fixup_cur_wm_latency(dev, dev_priv->wm.cur_latency);
2016
2017         intel_print_wm_latency(dev, "Primary", dev_priv->wm.pri_latency);
2018         intel_print_wm_latency(dev, "Sprite", dev_priv->wm.spr_latency);
2019         intel_print_wm_latency(dev, "Cursor", dev_priv->wm.cur_latency);
2020
2021         if (IS_GEN6(dev))
2022                 snb_wm_latency_quirk(dev);
2023 }
2024
2025 static void skl_setup_wm_latency(struct drm_device *dev)
2026 {
2027         struct drm_i915_private *dev_priv = dev->dev_private;
2028
2029         intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
2030         intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
2031 }
2032
2033 static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
2034                                       struct ilk_pipe_wm_parameters *p)
2035 {
2036         struct drm_device *dev = crtc->dev;
2037         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2038         enum pipe pipe = intel_crtc->pipe;
2039         struct drm_plane *plane;
2040
2041         if (!intel_crtc->active)
2042                 return;
2043
2044         p->active = true;
2045         p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
2046         p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
2047
2048         if (crtc->primary->state->fb) {
2049                 p->pri.enabled = true;
2050                 p->pri.bytes_per_pixel =
2051                         crtc->primary->state->fb->bits_per_pixel / 8;
2052         } else {
2053                 p->pri.enabled = false;
2054                 p->pri.bytes_per_pixel = 0;
2055         }
2056
2057         if (crtc->cursor->state->fb) {
2058                 p->cur.enabled = true;
2059                 p->cur.bytes_per_pixel = 4;
2060         } else {
2061                 p->cur.enabled = false;
2062                 p->cur.bytes_per_pixel = 0;
2063         }
2064         p->pri.horiz_pixels = intel_crtc->config->pipe_src_w;
2065         p->cur.horiz_pixels = intel_crtc->base.cursor->state->crtc_w;
2066
2067         drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
2068                 struct intel_plane *intel_plane = to_intel_plane(plane);
2069
2070                 if (intel_plane->pipe == pipe) {
2071                         p->spr = intel_plane->wm;
2072                         break;
2073                 }
2074         }
2075 }
2076
2077 static void ilk_compute_wm_config(struct drm_device *dev,
2078                                   struct intel_wm_config *config)
2079 {
2080         struct intel_crtc *intel_crtc;
2081
2082         /* Compute the currently _active_ config */
2083         for_each_intel_crtc(dev, intel_crtc) {
2084                 const struct intel_pipe_wm *wm = &intel_crtc->wm.active;
2085
2086                 if (!wm->pipe_enabled)
2087                         continue;
2088
2089                 config->sprites_enabled |= wm->sprites_enabled;
2090                 config->sprites_scaled |= wm->sprites_scaled;
2091                 config->num_pipes_active++;
2092         }
2093 }
2094
2095 /* Compute new watermarks for the pipe */
2096 static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
2097                                   const struct ilk_pipe_wm_parameters *params,
2098                                   struct intel_pipe_wm *pipe_wm)
2099 {
2100         struct drm_device *dev = crtc->dev;
2101         const struct drm_i915_private *dev_priv = dev->dev_private;
2102         int level, max_level = ilk_wm_max_level(dev);
2103         /* LP0 watermark maximums depend on this pipe alone */
2104         struct intel_wm_config config = {
2105                 .num_pipes_active = 1,
2106                 .sprites_enabled = params->spr.enabled,
2107                 .sprites_scaled = params->spr.scaled,
2108         };
2109         struct ilk_wm_maximums max;
2110
2111         pipe_wm->pipe_enabled = params->active;
2112         pipe_wm->sprites_enabled = params->spr.enabled;
2113         pipe_wm->sprites_scaled = params->spr.scaled;
2114
2115         /* ILK/SNB: LP2+ watermarks only w/o sprites */
2116         if (INTEL_INFO(dev)->gen <= 6 && params->spr.enabled)
2117                 max_level = 1;
2118
2119         /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2120         if (params->spr.scaled)
2121                 max_level = 0;
2122
2123         ilk_compute_wm_level(dev_priv, 0, params, &pipe_wm->wm[0]);
2124
2125         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2126                 pipe_wm->linetime = hsw_compute_linetime_wm(dev, crtc);
2127
2128         /* LP0 watermarks always use 1/2 DDB partitioning */
2129         ilk_compute_wm_maximums(dev, 0, &config, INTEL_DDB_PART_1_2, &max);
2130
2131         /* At least LP0 must be valid */
2132         if (!ilk_validate_wm_level(0, &max, &pipe_wm->wm[0]))
2133                 return false;
2134
2135         ilk_compute_wm_reg_maximums(dev, 1, &max);
2136
2137         for (level = 1; level <= max_level; level++) {
2138                 struct intel_wm_level wm = {};
2139
2140                 ilk_compute_wm_level(dev_priv, level, params, &wm);
2141
2142                 /*
2143                  * Disable any watermark level that exceeds the
2144                  * register maximums since such watermarks are
2145                  * always invalid.
2146                  */
2147                 if (!ilk_validate_wm_level(level, &max, &wm))
2148                         break;
2149
2150                 pipe_wm->wm[level] = wm;
2151         }
2152
2153         return true;
2154 }
2155
2156 /*
2157  * Merge the watermarks from all active pipes for a specific level.
2158  */
2159 static void ilk_merge_wm_level(struct drm_device *dev,
2160                                int level,
2161                                struct intel_wm_level *ret_wm)
2162 {
2163         const struct intel_crtc *intel_crtc;
2164
2165         ret_wm->enable = true;
2166
2167         for_each_intel_crtc(dev, intel_crtc) {
2168                 const struct intel_pipe_wm *active = &intel_crtc->wm.active;
2169                 const struct intel_wm_level *wm = &active->wm[level];
2170
2171                 if (!active->pipe_enabled)
2172                         continue;
2173
2174                 /*
2175                  * The watermark values may have been used in the past,
2176                  * so we must maintain them in the registers for some
2177                  * time even if the level is now disabled.
2178                  */
2179                 if (!wm->enable)
2180                         ret_wm->enable = false;
2181
2182                 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2183                 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2184                 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2185                 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2186         }
2187 }
2188
2189 /*
2190  * Merge all low power watermarks for all active pipes.
2191  */
2192 static void ilk_wm_merge(struct drm_device *dev,
2193                          const struct intel_wm_config *config,
2194                          const struct ilk_wm_maximums *max,
2195                          struct intel_pipe_wm *merged)
2196 {
2197         int level, max_level = ilk_wm_max_level(dev);
2198         int last_enabled_level = max_level;
2199
2200         /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2201         if ((INTEL_INFO(dev)->gen <= 6 || IS_IVYBRIDGE(dev)) &&
2202             config->num_pipes_active > 1)
2203                 return;
2204
2205         /* ILK: FBC WM must be disabled always */
2206         merged->fbc_wm_enabled = INTEL_INFO(dev)->gen >= 6;
2207
2208         /* merge each WM1+ level */
2209         for (level = 1; level <= max_level; level++) {
2210                 struct intel_wm_level *wm = &merged->wm[level];
2211
2212                 ilk_merge_wm_level(dev, level, wm);
2213
2214                 if (level > last_enabled_level)
2215                         wm->enable = false;
2216                 else if (!ilk_validate_wm_level(level, max, wm))
2217                         /* make sure all following levels get disabled */
2218                         last_enabled_level = level - 1;
2219
2220                 /*
2221                  * The spec says it is preferred to disable
2222                  * FBC WMs instead of disabling a WM level.
2223                  */
2224                 if (wm->fbc_val > max->fbc) {
2225                         if (wm->enable)
2226                                 merged->fbc_wm_enabled = false;
2227                         wm->fbc_val = 0;
2228                 }
2229         }
2230
2231         /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
2232         /*
2233          * FIXME this is racy. FBC might get enabled later.
2234          * What we should check here is whether FBC can be
2235          * enabled sometime later.
2236          */
2237         if (IS_GEN5(dev) && !merged->fbc_wm_enabled && intel_fbc_enabled(dev)) {
2238                 for (level = 2; level <= max_level; level++) {
2239                         struct intel_wm_level *wm = &merged->wm[level];
2240
2241                         wm->enable = false;
2242                 }
2243         }
2244 }
2245
2246 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
2247 {
2248         /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
2249         return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
2250 }
2251
2252 /* The value we need to program into the WM_LPx latency field */
2253 static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
2254 {
2255         struct drm_i915_private *dev_priv = dev->dev_private;
2256
2257         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2258                 return 2 * level;
2259         else
2260                 return dev_priv->wm.pri_latency[level];
2261 }
2262
2263 static void ilk_compute_wm_results(struct drm_device *dev,
2264                                    const struct intel_pipe_wm *merged,
2265                                    enum intel_ddb_partitioning partitioning,
2266                                    struct ilk_wm_values *results)
2267 {
2268         struct intel_crtc *intel_crtc;
2269         int level, wm_lp;
2270
2271         results->enable_fbc_wm = merged->fbc_wm_enabled;
2272         results->partitioning = partitioning;
2273
2274         /* LP1+ register values */
2275         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2276                 const struct intel_wm_level *r;
2277
2278                 level = ilk_wm_lp_to_level(wm_lp, merged);
2279
2280                 r = &merged->wm[level];
2281
2282                 /*
2283                  * Maintain the watermark values even if the level is
2284                  * disabled. Doing otherwise could cause underruns.
2285                  */
2286                 results->wm_lp[wm_lp - 1] =
2287                         (ilk_wm_lp_latency(dev, level) << WM1_LP_LATENCY_SHIFT) |
2288                         (r->pri_val << WM1_LP_SR_SHIFT) |
2289                         r->cur_val;
2290
2291                 if (r->enable)
2292                         results->wm_lp[wm_lp - 1] |= WM1_LP_SR_EN;
2293
2294                 if (INTEL_INFO(dev)->gen >= 8)
2295                         results->wm_lp[wm_lp - 1] |=
2296                                 r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
2297                 else
2298                         results->wm_lp[wm_lp - 1] |=
2299                                 r->fbc_val << WM1_LP_FBC_SHIFT;
2300
2301                 /*
2302                  * Always set WM1S_LP_EN when spr_val != 0, even if the
2303                  * level is disabled. Doing otherwise could cause underruns.
2304                  */
2305                 if (INTEL_INFO(dev)->gen <= 6 && r->spr_val) {
2306                         WARN_ON(wm_lp != 1);
2307                         results->wm_lp_spr[wm_lp - 1] = WM1S_LP_EN | r->spr_val;
2308                 } else
2309                         results->wm_lp_spr[wm_lp - 1] = r->spr_val;
2310         }
2311
2312         /* LP0 register values */
2313         for_each_intel_crtc(dev, intel_crtc) {
2314                 enum pipe pipe = intel_crtc->pipe;
2315                 const struct intel_wm_level *r =
2316                         &intel_crtc->wm.active.wm[0];
2317
2318                 if (WARN_ON(!r->enable))
2319                         continue;
2320
2321                 results->wm_linetime[pipe] = intel_crtc->wm.active.linetime;
2322
2323                 results->wm_pipe[pipe] =
2324                         (r->pri_val << WM0_PIPE_PLANE_SHIFT) |
2325                         (r->spr_val << WM0_PIPE_SPRITE_SHIFT) |
2326                         r->cur_val;
2327         }
2328 }
2329
2330 /* Find the result with the highest level enabled. Check for enable_fbc_wm in
2331  * case both are at the same level. Prefer r1 in case they're the same. */
2332 static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
2333                                                   struct intel_pipe_wm *r1,
2334                                                   struct intel_pipe_wm *r2)
2335 {
2336         int level, max_level = ilk_wm_max_level(dev);
2337         int level1 = 0, level2 = 0;
2338
2339         for (level = 1; level <= max_level; level++) {
2340                 if (r1->wm[level].enable)
2341                         level1 = level;
2342                 if (r2->wm[level].enable)
2343                         level2 = level;
2344         }
2345
2346         if (level1 == level2) {
2347                 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
2348                         return r2;
2349                 else
2350                         return r1;
2351         } else if (level1 > level2) {
2352                 return r1;
2353         } else {
2354                 return r2;
2355         }
2356 }
2357
2358 /* dirty bits used to track which watermarks need changes */
2359 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
2360 #define WM_DIRTY_LINETIME(pipe) (1 << (8 + (pipe)))
2361 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
2362 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
2363 #define WM_DIRTY_FBC (1 << 24)
2364 #define WM_DIRTY_DDB (1 << 25)
2365
2366 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
2367                                          const struct ilk_wm_values *old,
2368                                          const struct ilk_wm_values *new)
2369 {
2370         unsigned int dirty = 0;
2371         enum pipe pipe;
2372         int wm_lp;
2373
2374         for_each_pipe(dev_priv, pipe) {
2375                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
2376                         dirty |= WM_DIRTY_LINETIME(pipe);
2377                         /* Must disable LP1+ watermarks too */
2378                         dirty |= WM_DIRTY_LP_ALL;
2379                 }
2380
2381                 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
2382                         dirty |= WM_DIRTY_PIPE(pipe);
2383                         /* Must disable LP1+ watermarks too */
2384                         dirty |= WM_DIRTY_LP_ALL;
2385                 }
2386         }
2387
2388         if (old->enable_fbc_wm != new->enable_fbc_wm) {
2389                 dirty |= WM_DIRTY_FBC;
2390                 /* Must disable LP1+ watermarks too */
2391                 dirty |= WM_DIRTY_LP_ALL;
2392         }
2393
2394         if (old->partitioning != new->partitioning) {
2395                 dirty |= WM_DIRTY_DDB;
2396                 /* Must disable LP1+ watermarks too */
2397                 dirty |= WM_DIRTY_LP_ALL;
2398         }
2399
2400         /* LP1+ watermarks already deemed dirty, no need to continue */
2401         if (dirty & WM_DIRTY_LP_ALL)
2402                 return dirty;
2403
2404         /* Find the lowest numbered LP1+ watermark in need of an update... */
2405         for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
2406                 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
2407                     old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
2408                         break;
2409         }
2410
2411         /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
2412         for (; wm_lp <= 3; wm_lp++)
2413                 dirty |= WM_DIRTY_LP(wm_lp);
2414
2415         return dirty;
2416 }
2417
2418 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
2419                                unsigned int dirty)
2420 {
2421         struct ilk_wm_values *previous = &dev_priv->wm.hw;
2422         bool changed = false;
2423
2424         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM1_LP_SR_EN) {
2425                 previous->wm_lp[2] &= ~WM1_LP_SR_EN;
2426                 I915_WRITE(WM3_LP_ILK, previous->wm_lp[2]);
2427                 changed = true;
2428         }
2429         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM1_LP_SR_EN) {
2430                 previous->wm_lp[1] &= ~WM1_LP_SR_EN;
2431                 I915_WRITE(WM2_LP_ILK, previous->wm_lp[1]);
2432                 changed = true;
2433         }
2434         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM1_LP_SR_EN) {
2435                 previous->wm_lp[0] &= ~WM1_LP_SR_EN;
2436                 I915_WRITE(WM1_LP_ILK, previous->wm_lp[0]);
2437                 changed = true;
2438         }
2439
2440         /*
2441          * Don't touch WM1S_LP_EN here.
2442          * Doing so could cause underruns.
2443          */
2444
2445         return changed;
2446 }
2447
2448 /*
2449  * The spec says we shouldn't write when we don't need, because every write
2450  * causes WMs to be re-evaluated, expending some power.
2451  */
2452 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
2453                                 struct ilk_wm_values *results)
2454 {
2455         struct drm_device *dev = dev_priv->dev;
2456         struct ilk_wm_values *previous = &dev_priv->wm.hw;
2457         unsigned int dirty;
2458         uint32_t val;
2459
2460         dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
2461         if (!dirty)
2462                 return;
2463
2464         _ilk_disable_lp_wm(dev_priv, dirty);
2465
2466         if (dirty & WM_DIRTY_PIPE(PIPE_A))
2467                 I915_WRITE(WM0_PIPEA_ILK, results->wm_pipe[0]);
2468         if (dirty & WM_DIRTY_PIPE(PIPE_B))
2469                 I915_WRITE(WM0_PIPEB_ILK, results->wm_pipe[1]);
2470         if (dirty & WM_DIRTY_PIPE(PIPE_C))
2471                 I915_WRITE(WM0_PIPEC_IVB, results->wm_pipe[2]);
2472
2473         if (dirty & WM_DIRTY_LINETIME(PIPE_A))
2474                 I915_WRITE(PIPE_WM_LINETIME(PIPE_A), results->wm_linetime[0]);
2475         if (dirty & WM_DIRTY_LINETIME(PIPE_B))
2476                 I915_WRITE(PIPE_WM_LINETIME(PIPE_B), results->wm_linetime[1]);
2477         if (dirty & WM_DIRTY_LINETIME(PIPE_C))
2478                 I915_WRITE(PIPE_WM_LINETIME(PIPE_C), results->wm_linetime[2]);
2479
2480         if (dirty & WM_DIRTY_DDB) {
2481                 if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2482                         val = I915_READ(WM_MISC);
2483                         if (results->partitioning == INTEL_DDB_PART_1_2)
2484                                 val &= ~WM_MISC_DATA_PARTITION_5_6;
2485                         else
2486                                 val |= WM_MISC_DATA_PARTITION_5_6;
2487                         I915_WRITE(WM_MISC, val);
2488                 } else {
2489                         val = I915_READ(DISP_ARB_CTL2);
2490                         if (results->partitioning == INTEL_DDB_PART_1_2)
2491                                 val &= ~DISP_DATA_PARTITION_5_6;
2492                         else
2493                                 val |= DISP_DATA_PARTITION_5_6;
2494                         I915_WRITE(DISP_ARB_CTL2, val);
2495                 }
2496         }
2497
2498         if (dirty & WM_DIRTY_FBC) {
2499                 val = I915_READ(DISP_ARB_CTL);
2500                 if (results->enable_fbc_wm)
2501                         val &= ~DISP_FBC_WM_DIS;
2502                 else
2503                         val |= DISP_FBC_WM_DIS;
2504                 I915_WRITE(DISP_ARB_CTL, val);
2505         }
2506
2507         if (dirty & WM_DIRTY_LP(1) &&
2508             previous->wm_lp_spr[0] != results->wm_lp_spr[0])
2509                 I915_WRITE(WM1S_LP_ILK, results->wm_lp_spr[0]);
2510
2511         if (INTEL_INFO(dev)->gen >= 7) {
2512                 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
2513                         I915_WRITE(WM2S_LP_IVB, results->wm_lp_spr[1]);
2514                 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
2515                         I915_WRITE(WM3S_LP_IVB, results->wm_lp_spr[2]);
2516         }
2517
2518         if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
2519                 I915_WRITE(WM1_LP_ILK, results->wm_lp[0]);
2520         if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
2521                 I915_WRITE(WM2_LP_ILK, results->wm_lp[1]);
2522         if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
2523                 I915_WRITE(WM3_LP_ILK, results->wm_lp[2]);
2524
2525         dev_priv->wm.hw = *results;
2526 }
2527
2528 static bool ilk_disable_lp_wm(struct drm_device *dev)
2529 {
2530         struct drm_i915_private *dev_priv = dev->dev_private;
2531
2532         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
2533 }
2534
2535 /*
2536  * On gen9, we need to allocate Display Data Buffer (DDB) portions to the
2537  * different active planes.
2538  */
2539
2540 #define SKL_DDB_SIZE            896     /* in blocks */
2541
2542 static void
2543 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
2544                                    struct drm_crtc *for_crtc,
2545                                    const struct intel_wm_config *config,
2546                                    const struct skl_pipe_wm_parameters *params,
2547                                    struct skl_ddb_entry *alloc /* out */)
2548 {
2549         struct drm_crtc *crtc;
2550         unsigned int pipe_size, ddb_size;
2551         int nth_active_pipe;
2552
2553         if (!params->active) {
2554                 alloc->start = 0;
2555                 alloc->end = 0;
2556                 return;
2557         }
2558
2559         ddb_size = SKL_DDB_SIZE;
2560
2561         ddb_size -= 4; /* 4 blocks for bypass path allocation */
2562
2563         nth_active_pipe = 0;
2564         for_each_crtc(dev, crtc) {
2565                 if (!to_intel_crtc(crtc)->active)
2566                         continue;
2567
2568                 if (crtc == for_crtc)
2569                         break;
2570
2571                 nth_active_pipe++;
2572         }
2573
2574         pipe_size = ddb_size / config->num_pipes_active;
2575         alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
2576         alloc->end = alloc->start + pipe_size;
2577 }
2578
2579 static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
2580 {
2581         if (config->num_pipes_active == 1)
2582                 return 32;
2583
2584         return 8;
2585 }
2586
2587 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
2588 {
2589         entry->start = reg & 0x3ff;
2590         entry->end = (reg >> 16) & 0x3ff;
2591         if (entry->end)
2592                 entry->end += 1;
2593 }
2594
2595 void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
2596                           struct skl_ddb_allocation *ddb /* out */)
2597 {
2598         enum pipe pipe;
2599         int plane;
2600         u32 val;
2601
2602         for_each_pipe(dev_priv, pipe) {
2603                 for_each_plane(dev_priv, pipe, plane) {
2604                         val = I915_READ(PLANE_BUF_CFG(pipe, plane));
2605                         skl_ddb_entry_init_from_hw(&ddb->plane[pipe][plane],
2606                                                    val);
2607                 }
2608
2609                 val = I915_READ(CUR_BUF_CFG(pipe));
2610                 skl_ddb_entry_init_from_hw(&ddb->cursor[pipe], val);
2611         }
2612 }
2613
2614 static unsigned int
2615 skl_plane_relative_data_rate(const struct intel_plane_wm_parameters *p)
2616 {
2617         return p->horiz_pixels * p->vert_pixels * p->bytes_per_pixel;
2618 }
2619
2620 /*
2621  * We don't overflow 32 bits. Worst case is 3 planes enabled, each fetching
2622  * a 8192x4096@32bpp framebuffer:
2623  *   3 * 4096 * 8192  * 4 < 2^32
2624  */
2625 static unsigned int
2626 skl_get_total_relative_data_rate(struct intel_crtc *intel_crtc,
2627                                  const struct skl_pipe_wm_parameters *params)
2628 {
2629         unsigned int total_data_rate = 0;
2630         int plane;
2631
2632         for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
2633                 const struct intel_plane_wm_parameters *p;
2634
2635                 p = &params->plane[plane];
2636                 if (!p->enabled)
2637                         continue;
2638
2639                 total_data_rate += skl_plane_relative_data_rate(p);
2640         }
2641
2642         return total_data_rate;
2643 }
2644
2645 static void
2646 skl_allocate_pipe_ddb(struct drm_crtc *crtc,
2647                       const struct intel_wm_config *config,
2648                       const struct skl_pipe_wm_parameters *params,
2649                       struct skl_ddb_allocation *ddb /* out */)
2650 {
2651         struct drm_device *dev = crtc->dev;
2652         struct drm_i915_private *dev_priv = dev->dev_private;
2653         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2654         enum pipe pipe = intel_crtc->pipe;
2655         struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
2656         uint16_t alloc_size, start, cursor_blocks;
2657         uint16_t minimum[I915_MAX_PLANES];
2658         unsigned int total_data_rate;
2659         int plane;
2660
2661         skl_ddb_get_pipe_allocation_limits(dev, crtc, config, params, alloc);
2662         alloc_size = skl_ddb_entry_size(alloc);
2663         if (alloc_size == 0) {
2664                 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
2665                 memset(&ddb->cursor[pipe], 0, sizeof(ddb->cursor[pipe]));
2666                 return;
2667         }
2668
2669         cursor_blocks = skl_cursor_allocation(config);
2670         ddb->cursor[pipe].start = alloc->end - cursor_blocks;
2671         ddb->cursor[pipe].end = alloc->end;
2672
2673         alloc_size -= cursor_blocks;
2674         alloc->end -= cursor_blocks;
2675
2676         /* 1. Allocate the mininum required blocks for each active plane */
2677         for_each_plane(dev_priv, pipe, plane) {
2678                 const struct intel_plane_wm_parameters *p;
2679
2680                 p = &params->plane[plane];
2681                 if (!p->enabled)
2682                         continue;
2683
2684                 minimum[plane] = 8;
2685                 alloc_size -= minimum[plane];
2686         }
2687
2688         /*
2689          * 2. Distribute the remaining space in proportion to the amount of
2690          * data each plane needs to fetch from memory.
2691          *
2692          * FIXME: we may not allocate every single block here.
2693          */
2694         total_data_rate = skl_get_total_relative_data_rate(intel_crtc, params);
2695
2696         start = alloc->start;
2697         for (plane = 0; plane < intel_num_planes(intel_crtc); plane++) {
2698                 const struct intel_plane_wm_parameters *p;
2699                 unsigned int data_rate;
2700                 uint16_t plane_blocks;
2701
2702                 p = &params->plane[plane];
2703                 if (!p->enabled)
2704                         continue;
2705
2706                 data_rate = skl_plane_relative_data_rate(p);
2707
2708                 /*
2709                  * promote the expression to 64 bits to avoid overflowing, the
2710                  * result is < available as data_rate / total_data_rate < 1
2711                  */
2712                 plane_blocks = minimum[plane];
2713                 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
2714                                         total_data_rate);
2715
2716                 ddb->plane[pipe][plane].start = start;
2717                 ddb->plane[pipe][plane].end = start + plane_blocks;
2718
2719                 start += plane_blocks;
2720         }
2721
2722 }
2723
2724 static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
2725 {
2726         /* TODO: Take into account the scalers once we support them */
2727         return config->base.adjusted_mode.crtc_clock;
2728 }
2729
2730 /*
2731  * The max latency should be 257 (max the punit can code is 255 and we add 2us
2732  * for the read latency) and bytes_per_pixel should always be <= 8, so that
2733  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
2734  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
2735 */
2736 static uint32_t skl_wm_method1(uint32_t pixel_rate, uint8_t bytes_per_pixel,
2737                                uint32_t latency)
2738 {
2739         uint32_t wm_intermediate_val, ret;
2740
2741         if (latency == 0)
2742                 return UINT_MAX;
2743
2744         wm_intermediate_val = latency * pixel_rate * bytes_per_pixel / 512;
2745         ret = DIV_ROUND_UP(wm_intermediate_val, 1000);
2746
2747         return ret;
2748 }
2749
2750 static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
2751                                uint32_t horiz_pixels, uint8_t bytes_per_pixel,
2752                                uint64_t tiling, uint32_t latency)
2753 {
2754         uint32_t ret;
2755         uint32_t plane_bytes_per_line, plane_blocks_per_line;
2756         uint32_t wm_intermediate_val;
2757
2758         if (latency == 0)
2759                 return UINT_MAX;
2760
2761         plane_bytes_per_line = horiz_pixels * bytes_per_pixel;
2762
2763         if (tiling == I915_FORMAT_MOD_Y_TILED ||
2764             tiling == I915_FORMAT_MOD_Yf_TILED) {
2765                 plane_bytes_per_line *= 4;
2766                 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
2767                 plane_blocks_per_line /= 4;
2768         } else {
2769                 plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
2770         }
2771
2772         wm_intermediate_val = latency * pixel_rate;
2773         ret = DIV_ROUND_UP(wm_intermediate_val, pipe_htotal * 1000) *
2774                                 plane_blocks_per_line;
2775
2776         return ret;
2777 }
2778
2779 static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
2780                                        const struct intel_crtc *intel_crtc)
2781 {
2782         struct drm_device *dev = intel_crtc->base.dev;
2783         struct drm_i915_private *dev_priv = dev->dev_private;
2784         const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
2785         enum pipe pipe = intel_crtc->pipe;
2786
2787         if (memcmp(new_ddb->plane[pipe], cur_ddb->plane[pipe],
2788                    sizeof(new_ddb->plane[pipe])))
2789                 return true;
2790
2791         if (memcmp(&new_ddb->cursor[pipe], &cur_ddb->cursor[pipe],
2792                     sizeof(new_ddb->cursor[pipe])))
2793                 return true;
2794
2795         return false;
2796 }
2797
2798 static void skl_compute_wm_global_parameters(struct drm_device *dev,
2799                                              struct intel_wm_config *config)
2800 {
2801         struct drm_crtc *crtc;
2802         struct drm_plane *plane;
2803
2804         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
2805                 config->num_pipes_active += to_intel_crtc(crtc)->active;
2806
2807         /* FIXME: I don't think we need those two global parameters on SKL */
2808         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
2809                 struct intel_plane *intel_plane = to_intel_plane(plane);
2810
2811                 config->sprites_enabled |= intel_plane->wm.enabled;
2812                 config->sprites_scaled |= intel_plane->wm.scaled;
2813         }
2814 }
2815
2816 static void skl_compute_wm_pipe_parameters(struct drm_crtc *crtc,
2817                                            struct skl_pipe_wm_parameters *p)
2818 {
2819         struct drm_device *dev = crtc->dev;
2820         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2821         enum pipe pipe = intel_crtc->pipe;
2822         struct drm_plane *plane;
2823         struct drm_framebuffer *fb;
2824         int i = 1; /* Index for sprite planes start */
2825
2826         p->active = intel_crtc->active;
2827         if (p->active) {
2828                 p->pipe_htotal = intel_crtc->config->base.adjusted_mode.crtc_htotal;
2829                 p->pixel_rate = skl_pipe_pixel_rate(intel_crtc->config);
2830
2831                 fb = crtc->primary->state->fb;
2832                 if (fb) {
2833                         p->plane[0].enabled = true;
2834                         p->plane[0].bytes_per_pixel = fb->bits_per_pixel / 8;
2835                         p->plane[0].tiling = fb->modifier[0];
2836                 } else {
2837                         p->plane[0].enabled = false;
2838                         p->plane[0].bytes_per_pixel = 0;
2839                         p->plane[0].tiling = DRM_FORMAT_MOD_NONE;
2840                 }
2841                 p->plane[0].horiz_pixels = intel_crtc->config->pipe_src_w;
2842                 p->plane[0].vert_pixels = intel_crtc->config->pipe_src_h;
2843
2844                 fb = crtc->cursor->state->fb;
2845                 if (fb) {
2846                         p->cursor.enabled = true;
2847                         p->cursor.bytes_per_pixel = fb->bits_per_pixel / 8;
2848                         p->cursor.horiz_pixels = crtc->cursor->state->crtc_w;
2849                         p->cursor.vert_pixels = crtc->cursor->state->crtc_h;
2850                 } else {
2851                         p->cursor.enabled = false;
2852                         p->cursor.bytes_per_pixel = 0;
2853                         p->cursor.horiz_pixels = 64;
2854                         p->cursor.vert_pixels = 64;
2855                 }
2856         }
2857
2858         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
2859                 struct intel_plane *intel_plane = to_intel_plane(plane);
2860
2861                 if (intel_plane->pipe == pipe &&
2862                         plane->type == DRM_PLANE_TYPE_OVERLAY)
2863                         p->plane[i++] = intel_plane->wm;
2864         }
2865 }
2866
2867 static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
2868                                  struct skl_pipe_wm_parameters *p,
2869                                  struct intel_plane_wm_parameters *p_params,
2870                                  uint16_t ddb_allocation,
2871                                  int level,
2872                                  uint16_t *out_blocks, /* out */
2873                                  uint8_t *out_lines /* out */)
2874 {
2875         uint32_t latency = dev_priv->wm.skl_latency[level];
2876         uint32_t method1, method2;
2877         uint32_t plane_bytes_per_line, plane_blocks_per_line;
2878         uint32_t res_blocks, res_lines;
2879         uint32_t selected_result;
2880
2881         if (latency == 0 || !p->active || !p_params->enabled)
2882                 return false;
2883
2884         method1 = skl_wm_method1(p->pixel_rate,
2885                                  p_params->bytes_per_pixel,
2886                                  latency);
2887         method2 = skl_wm_method2(p->pixel_rate,
2888                                  p->pipe_htotal,
2889                                  p_params->horiz_pixels,
2890                                  p_params->bytes_per_pixel,
2891                                  p_params->tiling,
2892                                  latency);
2893
2894         plane_bytes_per_line = p_params->horiz_pixels *
2895                                         p_params->bytes_per_pixel;
2896         plane_blocks_per_line = DIV_ROUND_UP(plane_bytes_per_line, 512);
2897
2898         if (p_params->tiling == I915_FORMAT_MOD_Y_TILED ||
2899             p_params->tiling == I915_FORMAT_MOD_Yf_TILED) {
2900                 uint32_t y_tile_minimum = plane_blocks_per_line * 4;
2901                 selected_result = max(method2, y_tile_minimum);
2902         } else {
2903                 if ((ddb_allocation / plane_blocks_per_line) >= 1)
2904                         selected_result = min(method1, method2);
2905                 else
2906                         selected_result = method1;
2907         }
2908
2909         res_blocks = selected_result + 1;
2910         res_lines = DIV_ROUND_UP(selected_result, plane_blocks_per_line);
2911
2912         if (level >= 1 && level <= 7) {
2913                 if (p_params->tiling == I915_FORMAT_MOD_Y_TILED ||
2914                     p_params->tiling == I915_FORMAT_MOD_Yf_TILED)
2915                         res_lines += 4;
2916                 else
2917                         res_blocks++;
2918         }
2919
2920         if (res_blocks >= ddb_allocation || res_lines > 31)
2921                 return false;
2922
2923         *out_blocks = res_blocks;
2924         *out_lines = res_lines;
2925
2926         return true;
2927 }
2928
2929 static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
2930                                  struct skl_ddb_allocation *ddb,
2931                                  struct skl_pipe_wm_parameters *p,
2932                                  enum pipe pipe,
2933                                  int level,
2934                                  int num_planes,
2935                                  struct skl_wm_level *result)
2936 {
2937         uint16_t ddb_blocks;
2938         int i;
2939
2940         for (i = 0; i < num_planes; i++) {
2941                 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
2942
2943                 result->plane_en[i] = skl_compute_plane_wm(dev_priv,
2944                                                 p, &p->plane[i],
2945                                                 ddb_blocks,
2946                                                 level,
2947                                                 &result->plane_res_b[i],
2948                                                 &result->plane_res_l[i]);
2949         }
2950
2951         ddb_blocks = skl_ddb_entry_size(&ddb->cursor[pipe]);
2952         result->cursor_en = skl_compute_plane_wm(dev_priv, p, &p->cursor,
2953                                                  ddb_blocks, level,
2954                                                  &result->cursor_res_b,
2955                                                  &result->cursor_res_l);
2956 }
2957
2958 static uint32_t
2959 skl_compute_linetime_wm(struct drm_crtc *crtc, struct skl_pipe_wm_parameters *p)
2960 {
2961         if (!to_intel_crtc(crtc)->active)
2962                 return 0;
2963
2964         return DIV_ROUND_UP(8 * p->pipe_htotal * 1000, p->pixel_rate);
2965
2966 }
2967
2968 static void skl_compute_transition_wm(struct drm_crtc *crtc,
2969                                       struct skl_pipe_wm_parameters *params,
2970                                       struct skl_wm_level *trans_wm /* out */)
2971 {
2972         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2973         int i;
2974
2975         if (!params->active)
2976                 return;
2977
2978         /* Until we know more, just disable transition WMs */
2979         for (i = 0; i < intel_num_planes(intel_crtc); i++)
2980                 trans_wm->plane_en[i] = false;
2981         trans_wm->cursor_en = false;
2982 }
2983
2984 static void skl_compute_pipe_wm(struct drm_crtc *crtc,
2985                                 struct skl_ddb_allocation *ddb,
2986                                 struct skl_pipe_wm_parameters *params,
2987                                 struct skl_pipe_wm *pipe_wm)
2988 {
2989         struct drm_device *dev = crtc->dev;
2990         const struct drm_i915_private *dev_priv = dev->dev_private;
2991         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2992         int level, max_level = ilk_wm_max_level(dev);
2993
2994         for (level = 0; level <= max_level; level++) {
2995                 skl_compute_wm_level(dev_priv, ddb, params, intel_crtc->pipe,
2996                                      level, intel_num_planes(intel_crtc),
2997                                      &pipe_wm->wm[level]);
2998         }
2999         pipe_wm->linetime = skl_compute_linetime_wm(crtc, params);
3000
3001         skl_compute_transition_wm(crtc, params, &pipe_wm->trans_wm);
3002 }
3003
3004 static void skl_compute_wm_results(struct drm_device *dev,
3005                                    struct skl_pipe_wm_parameters *p,
3006                                    struct skl_pipe_wm *p_wm,
3007                                    struct skl_wm_values *r,
3008                                    struct intel_crtc *intel_crtc)
3009 {
3010         int level, max_level = ilk_wm_max_level(dev);
3011         enum pipe pipe = intel_crtc->pipe;
3012         uint32_t temp;
3013         int i;
3014
3015         for (level = 0; level <= max_level; level++) {
3016                 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3017                         temp = 0;
3018
3019                         temp |= p_wm->wm[level].plane_res_l[i] <<
3020                                         PLANE_WM_LINES_SHIFT;
3021                         temp |= p_wm->wm[level].plane_res_b[i];
3022                         if (p_wm->wm[level].plane_en[i])
3023                                 temp |= PLANE_WM_EN;
3024
3025                         r->plane[pipe][i][level] = temp;
3026                 }
3027
3028                 temp = 0;
3029
3030                 temp |= p_wm->wm[level].cursor_res_l << PLANE_WM_LINES_SHIFT;
3031                 temp |= p_wm->wm[level].cursor_res_b;
3032
3033                 if (p_wm->wm[level].cursor_en)
3034                         temp |= PLANE_WM_EN;
3035
3036                 r->cursor[pipe][level] = temp;
3037
3038         }
3039
3040         /* transition WMs */
3041         for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3042                 temp = 0;
3043                 temp |= p_wm->trans_wm.plane_res_l[i] << PLANE_WM_LINES_SHIFT;
3044                 temp |= p_wm->trans_wm.plane_res_b[i];
3045                 if (p_wm->trans_wm.plane_en[i])
3046                         temp |= PLANE_WM_EN;
3047
3048                 r->plane_trans[pipe][i] = temp;
3049         }
3050
3051         temp = 0;
3052         temp |= p_wm->trans_wm.cursor_res_l << PLANE_WM_LINES_SHIFT;
3053         temp |= p_wm->trans_wm.cursor_res_b;
3054         if (p_wm->trans_wm.cursor_en)
3055                 temp |= PLANE_WM_EN;
3056
3057         r->cursor_trans[pipe] = temp;
3058
3059         r->wm_linetime[pipe] = p_wm->linetime;
3060 }
3061
3062 static void skl_ddb_entry_write(struct drm_i915_private *dev_priv, uint32_t reg,
3063                                 const struct skl_ddb_entry *entry)
3064 {
3065         if (entry->end)
3066                 I915_WRITE(reg, (entry->end - 1) << 16 | entry->start);
3067         else
3068                 I915_WRITE(reg, 0);
3069 }
3070
3071 static void skl_write_wm_values(struct drm_i915_private *dev_priv,
3072                                 const struct skl_wm_values *new)
3073 {
3074         struct drm_device *dev = dev_priv->dev;
3075         struct intel_crtc *crtc;
3076
3077         list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
3078                 int i, level, max_level = ilk_wm_max_level(dev);
3079                 enum pipe pipe = crtc->pipe;
3080
3081                 if (!new->dirty[pipe])
3082                         continue;
3083
3084                 I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
3085
3086                 for (level = 0; level <= max_level; level++) {
3087                         for (i = 0; i < intel_num_planes(crtc); i++)
3088                                 I915_WRITE(PLANE_WM(pipe, i, level),
3089                                            new->plane[pipe][i][level]);
3090                         I915_WRITE(CUR_WM(pipe, level),
3091                                    new->cursor[pipe][level]);
3092                 }
3093                 for (i = 0; i < intel_num_planes(crtc); i++)
3094                         I915_WRITE(PLANE_WM_TRANS(pipe, i),
3095                                    new->plane_trans[pipe][i]);
3096                 I915_WRITE(CUR_WM_TRANS(pipe), new->cursor_trans[pipe]);
3097
3098                 for (i = 0; i < intel_num_planes(crtc); i++)
3099                         skl_ddb_entry_write(dev_priv,
3100                                             PLANE_BUF_CFG(pipe, i),
3101                                             &new->ddb.plane[pipe][i]);
3102
3103                 skl_ddb_entry_write(dev_priv, CUR_BUF_CFG(pipe),
3104                                     &new->ddb.cursor[pipe]);
3105         }
3106 }
3107
3108 /*
3109  * When setting up a new DDB allocation arrangement, we need to correctly
3110  * sequence the times at which the new allocations for the pipes are taken into
3111  * account or we'll have pipes fetching from space previously allocated to
3112  * another pipe.
3113  *
3114  * Roughly the sequence looks like:
3115  *  1. re-allocate the pipe(s) with the allocation being reduced and not
3116  *     overlapping with a previous light-up pipe (another way to put it is:
3117  *     pipes with their new allocation strickly included into their old ones).
3118  *  2. re-allocate the other pipes that get their allocation reduced
3119  *  3. allocate the pipes having their allocation increased
3120  *
3121  * Steps 1. and 2. are here to take care of the following case:
3122  * - Initially DDB looks like this:
3123  *     |   B    |   C    |
3124  * - enable pipe A.
3125  * - pipe B has a reduced DDB allocation that overlaps with the old pipe C
3126  *   allocation
3127  *     |  A  |  B  |  C  |
3128  *
3129  * We need to sequence the re-allocation: C, B, A (and not B, C, A).
3130  */
3131
3132 static void
3133 skl_wm_flush_pipe(struct drm_i915_private *dev_priv, enum pipe pipe, int pass)
3134 {
3135         int plane;
3136
3137         DRM_DEBUG_KMS("flush pipe %c (pass %d)\n", pipe_name(pipe), pass);
3138
3139         for_each_plane(dev_priv, pipe, plane) {
3140                 I915_WRITE(PLANE_SURF(pipe, plane),
3141                            I915_READ(PLANE_SURF(pipe, plane)));
3142         }
3143         I915_WRITE(CURBASE(pipe), I915_READ(CURBASE(pipe)));
3144 }
3145
3146 static bool
3147 skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
3148                             const struct skl_ddb_allocation *new,
3149                             enum pipe pipe)
3150 {
3151         uint16_t old_size, new_size;
3152
3153         old_size = skl_ddb_entry_size(&old->pipe[pipe]);
3154         new_size = skl_ddb_entry_size(&new->pipe[pipe]);
3155
3156         return old_size != new_size &&
3157                new->pipe[pipe].start >= old->pipe[pipe].start &&
3158                new->pipe[pipe].end <= old->pipe[pipe].end;
3159 }
3160
3161 static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
3162                                 struct skl_wm_values *new_values)
3163 {
3164         struct drm_device *dev = dev_priv->dev;
3165         struct skl_ddb_allocation *cur_ddb, *new_ddb;
3166         bool reallocated[I915_MAX_PIPES] = {false, false, false};
3167         struct intel_crtc *crtc;
3168         enum pipe pipe;
3169
3170         new_ddb = &new_values->ddb;
3171         cur_ddb = &dev_priv->wm.skl_hw.ddb;
3172
3173         /*
3174          * First pass: flush the pipes with the new allocation contained into
3175          * the old space.
3176          *
3177          * We'll wait for the vblank on those pipes to ensure we can safely
3178          * re-allocate the freed space without this pipe fetching from it.
3179          */
3180         for_each_intel_crtc(dev, crtc) {
3181                 if (!crtc->active)
3182                         continue;
3183
3184                 pipe = crtc->pipe;
3185
3186                 if (!skl_ddb_allocation_included(cur_ddb, new_ddb, pipe))
3187                         continue;
3188
3189                 skl_wm_flush_pipe(dev_priv, pipe, 1);
3190                 intel_wait_for_vblank(dev, pipe);
3191
3192                 reallocated[pipe] = true;
3193         }
3194
3195
3196         /*
3197          * Second pass: flush the pipes that are having their allocation
3198          * reduced, but overlapping with a previous allocation.
3199          *
3200          * Here as well we need to wait for the vblank to make sure the freed
3201          * space is not used anymore.
3202          */
3203         for_each_intel_crtc(dev, crtc) {
3204                 if (!crtc->active)
3205                         continue;
3206
3207                 pipe = crtc->pipe;
3208
3209                 if (reallocated[pipe])
3210                         continue;
3211
3212                 if (skl_ddb_entry_size(&new_ddb->pipe[pipe]) <
3213                     skl_ddb_entry_size(&cur_ddb->pipe[pipe])) {
3214                         skl_wm_flush_pipe(dev_priv, pipe, 2);
3215                         intel_wait_for_vblank(dev, pipe);
3216                         reallocated[pipe] = true;
3217                 }
3218         }
3219
3220         /*
3221          * Third pass: flush the pipes that got more space allocated.
3222          *
3223          * We don't need to actively wait for the update here, next vblank
3224          * will just get more DDB space with the correct WM values.
3225          */
3226         for_each_intel_crtc(dev, crtc) {
3227                 if (!crtc->active)
3228                         continue;
3229
3230                 pipe = crtc->pipe;
3231
3232                 /*
3233                  * At this point, only the pipes more space than before are
3234                  * left to re-allocate.
3235                  */
3236                 if (reallocated[pipe])
3237                         continue;
3238
3239                 skl_wm_flush_pipe(dev_priv, pipe, 3);
3240         }
3241 }
3242
3243 static bool skl_update_pipe_wm(struct drm_crtc *crtc,
3244                                struct skl_pipe_wm_parameters *params,
3245                                struct intel_wm_config *config,
3246                                struct skl_ddb_allocation *ddb, /* out */
3247                                struct skl_pipe_wm *pipe_wm /* out */)
3248 {
3249         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3250
3251         skl_compute_wm_pipe_parameters(crtc, params);
3252         skl_allocate_pipe_ddb(crtc, config, params, ddb);
3253         skl_compute_pipe_wm(crtc, ddb, params, pipe_wm);
3254
3255         if (!memcmp(&intel_crtc->wm.skl_active, pipe_wm, sizeof(*pipe_wm)))
3256                 return false;
3257
3258         intel_crtc->wm.skl_active = *pipe_wm;
3259         return true;
3260 }
3261
3262 static void skl_update_other_pipe_wm(struct drm_device *dev,
3263                                      struct drm_crtc *crtc,
3264                                      struct intel_wm_config *config,
3265                                      struct skl_wm_values *r)
3266 {
3267         struct intel_crtc *intel_crtc;
3268         struct intel_crtc *this_crtc = to_intel_crtc(crtc);
3269
3270         /*
3271          * If the WM update hasn't changed the allocation for this_crtc (the
3272          * crtc we are currently computing the new WM values for), other
3273          * enabled crtcs will keep the same allocation and we don't need to
3274          * recompute anything for them.
3275          */
3276         if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
3277                 return;
3278
3279         /*
3280          * Otherwise, because of this_crtc being freshly enabled/disabled, the
3281          * other active pipes need new DDB allocation and WM values.
3282          */
3283         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
3284                                 base.head) {
3285                 struct skl_pipe_wm_parameters params = {};
3286                 struct skl_pipe_wm pipe_wm = {};
3287                 bool wm_changed;
3288
3289                 if (this_crtc->pipe == intel_crtc->pipe)
3290                         continue;
3291
3292                 if (!intel_crtc->active)
3293                         continue;
3294
3295                 wm_changed = skl_update_pipe_wm(&intel_crtc->base,
3296                                                 &params, config,
3297                                                 &r->ddb, &pipe_wm);
3298
3299                 /*
3300                  * If we end up re-computing the other pipe WM values, it's
3301                  * because it was really needed, so we expect the WM values to
3302                  * be different.
3303                  */
3304                 WARN_ON(!wm_changed);
3305
3306                 skl_compute_wm_results(dev, &params, &pipe_wm, r, intel_crtc);
3307                 r->dirty[intel_crtc->pipe] = true;
3308         }
3309 }
3310
3311 static void skl_update_wm(struct drm_crtc *crtc)
3312 {
3313         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3314         struct drm_device *dev = crtc->dev;
3315         struct drm_i915_private *dev_priv = dev->dev_private;
3316         struct skl_pipe_wm_parameters params = {};
3317         struct skl_wm_values *results = &dev_priv->wm.skl_results;
3318         struct skl_pipe_wm pipe_wm = {};
3319         struct intel_wm_config config = {};
3320
3321         memset(results, 0, sizeof(*results));
3322
3323         skl_compute_wm_global_parameters(dev, &config);
3324
3325         if (!skl_update_pipe_wm(crtc, &params, &config,
3326                                 &results->ddb, &pipe_wm))
3327                 return;
3328
3329         skl_compute_wm_results(dev, &params, &pipe_wm, results, intel_crtc);
3330         results->dirty[intel_crtc->pipe] = true;
3331
3332         skl_update_other_pipe_wm(dev, crtc, &config, results);
3333         skl_write_wm_values(dev_priv, results);
3334         skl_flush_wm_values(dev_priv, results);
3335
3336         /* store the new configuration */
3337         dev_priv->wm.skl_hw = *results;
3338 }
3339
3340 static void
3341 skl_update_sprite_wm(struct drm_plane *plane, struct drm_crtc *crtc,
3342                      uint32_t sprite_width, uint32_t sprite_height,
3343                      int pixel_size, bool enabled, bool scaled)
3344 {
3345         struct intel_plane *intel_plane = to_intel_plane(plane);
3346         struct drm_framebuffer *fb = plane->state->fb;
3347
3348         intel_plane->wm.enabled = enabled;
3349         intel_plane->wm.scaled = scaled;
3350         intel_plane->wm.horiz_pixels = sprite_width;
3351         intel_plane->wm.vert_pixels = sprite_height;
3352         intel_plane->wm.bytes_per_pixel = pixel_size;
3353         intel_plane->wm.tiling = DRM_FORMAT_MOD_NONE;
3354         /*
3355          * Framebuffer can be NULL on plane disable, but it does not
3356          * matter for watermarks if we assume no tiling in that case.
3357          */
3358         if (fb)
3359                 intel_plane->wm.tiling = fb->modifier[0];
3360
3361         skl_update_wm(crtc);
3362 }
3363
3364 static void ilk_update_wm(struct drm_crtc *crtc)
3365 {
3366         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3367         struct drm_device *dev = crtc->dev;
3368         struct drm_i915_private *dev_priv = dev->dev_private;
3369         struct ilk_wm_maximums max;
3370         struct ilk_pipe_wm_parameters params = {};
3371         struct ilk_wm_values results = {};
3372         enum intel_ddb_partitioning partitioning;
3373         struct intel_pipe_wm pipe_wm = {};
3374         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3375         struct intel_wm_config config = {};
3376
3377         ilk_compute_wm_parameters(crtc, &params);
3378
3379         intel_compute_pipe_wm(crtc, &params, &pipe_wm);
3380
3381         if (!memcmp(&intel_crtc->wm.active, &pipe_wm, sizeof(pipe_wm)))
3382                 return;
3383
3384         intel_crtc->wm.active = pipe_wm;
3385
3386         ilk_compute_wm_config(dev, &config);
3387
3388         ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_1_2, &max);
3389         ilk_wm_merge(dev, &config, &max, &lp_wm_1_2);
3390
3391         /* 5/6 split only in single pipe config on IVB+ */
3392         if (INTEL_INFO(dev)->gen >= 7 &&
3393             config.num_pipes_active == 1 && config.sprites_enabled) {
3394                 ilk_compute_wm_maximums(dev, 1, &config, INTEL_DDB_PART_5_6, &max);
3395                 ilk_wm_merge(dev, &config, &max, &lp_wm_5_6);
3396
3397                 best_lp_wm = ilk_find_best_result(dev, &lp_wm_1_2, &lp_wm_5_6);
3398         } else {
3399                 best_lp_wm = &lp_wm_1_2;
3400         }
3401
3402         partitioning = (best_lp_wm == &lp_wm_1_2) ?
3403                        INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3404
3405         ilk_compute_wm_results(dev, best_lp_wm, partitioning, &results);
3406
3407         ilk_write_wm_values(dev_priv, &results);
3408 }
3409
3410 static void
3411 ilk_update_sprite_wm(struct drm_plane *plane,
3412                      struct drm_crtc *crtc,
3413                      uint32_t sprite_width, uint32_t sprite_height,
3414                      int pixel_size, bool enabled, bool scaled)
3415 {
3416         struct drm_device *dev = plane->dev;
3417         struct intel_plane *intel_plane = to_intel_plane(plane);
3418
3419         intel_plane->wm.enabled = enabled;
3420         intel_plane->wm.scaled = scaled;
3421         intel_plane->wm.horiz_pixels = sprite_width;
3422         intel_plane->wm.vert_pixels = sprite_width;
3423         intel_plane->wm.bytes_per_pixel = pixel_size;
3424
3425         /*
3426          * IVB workaround: must disable low power watermarks for at least
3427          * one frame before enabling scaling.  LP watermarks can be re-enabled
3428          * when scaling is disabled.
3429          *
3430          * WaCxSRDisabledForSpriteScaling:ivb
3431          */
3432         if (IS_IVYBRIDGE(dev) && scaled && ilk_disable_lp_wm(dev))
3433                 intel_wait_for_vblank(dev, intel_plane->pipe);
3434
3435         ilk_update_wm(crtc);
3436 }
3437
3438 static void skl_pipe_wm_active_state(uint32_t val,
3439                                      struct skl_pipe_wm *active,
3440                                      bool is_transwm,
3441                                      bool is_cursor,
3442                                      int i,
3443                                      int level)
3444 {
3445         bool is_enabled = (val & PLANE_WM_EN) != 0;
3446
3447         if (!is_transwm) {
3448                 if (!is_cursor) {
3449                         active->wm[level].plane_en[i] = is_enabled;
3450                         active->wm[level].plane_res_b[i] =
3451                                         val & PLANE_WM_BLOCKS_MASK;
3452                         active->wm[level].plane_res_l[i] =
3453                                         (val >> PLANE_WM_LINES_SHIFT) &
3454                                                 PLANE_WM_LINES_MASK;
3455                 } else {
3456                         active->wm[level].cursor_en = is_enabled;
3457                         active->wm[level].cursor_res_b =
3458                                         val & PLANE_WM_BLOCKS_MASK;
3459                         active->wm[level].cursor_res_l =
3460                                         (val >> PLANE_WM_LINES_SHIFT) &
3461                                                 PLANE_WM_LINES_MASK;
3462                 }
3463         } else {
3464                 if (!is_cursor) {
3465                         active->trans_wm.plane_en[i] = is_enabled;
3466                         active->trans_wm.plane_res_b[i] =
3467                                         val & PLANE_WM_BLOCKS_MASK;
3468                         active->trans_wm.plane_res_l[i] =
3469                                         (val >> PLANE_WM_LINES_SHIFT) &
3470                                                 PLANE_WM_LINES_MASK;
3471                 } else {
3472                         active->trans_wm.cursor_en = is_enabled;
3473                         active->trans_wm.cursor_res_b =
3474                                         val & PLANE_WM_BLOCKS_MASK;
3475                         active->trans_wm.cursor_res_l =
3476                                         (val >> PLANE_WM_LINES_SHIFT) &
3477                                                 PLANE_WM_LINES_MASK;
3478                 }
3479         }
3480 }
3481
3482 static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3483 {
3484         struct drm_device *dev = crtc->dev;
3485         struct drm_i915_private *dev_priv = dev->dev_private;
3486         struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
3487         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3488         struct skl_pipe_wm *active = &intel_crtc->wm.skl_active;
3489         enum pipe pipe = intel_crtc->pipe;
3490         int level, i, max_level;
3491         uint32_t temp;
3492
3493         max_level = ilk_wm_max_level(dev);
3494
3495         hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3496
3497         for (level = 0; level <= max_level; level++) {
3498                 for (i = 0; i < intel_num_planes(intel_crtc); i++)
3499                         hw->plane[pipe][i][level] =
3500                                         I915_READ(PLANE_WM(pipe, i, level));
3501                 hw->cursor[pipe][level] = I915_READ(CUR_WM(pipe, level));
3502         }
3503
3504         for (i = 0; i < intel_num_planes(intel_crtc); i++)
3505                 hw->plane_trans[pipe][i] = I915_READ(PLANE_WM_TRANS(pipe, i));
3506         hw->cursor_trans[pipe] = I915_READ(CUR_WM_TRANS(pipe));
3507
3508         if (!intel_crtc->active)
3509                 return;
3510
3511         hw->dirty[pipe] = true;
3512
3513         active->linetime = hw->wm_linetime[pipe];
3514
3515         for (level = 0; level <= max_level; level++) {
3516                 for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3517                         temp = hw->plane[pipe][i][level];
3518                         skl_pipe_wm_active_state(temp, active, false,
3519                                                 false, i, level);
3520                 }
3521                 temp = hw->cursor[pipe][level];
3522                 skl_pipe_wm_active_state(temp, active, false, true, i, level);
3523         }
3524
3525         for (i = 0; i < intel_num_planes(intel_crtc); i++) {
3526                 temp = hw->plane_trans[pipe][i];
3527                 skl_pipe_wm_active_state(temp, active, true, false, i, 0);
3528         }
3529
3530         temp = hw->cursor_trans[pipe];
3531         skl_pipe_wm_active_state(temp, active, true, true, i, 0);
3532 }
3533
3534 void skl_wm_get_hw_state(struct drm_device *dev)
3535 {
3536         struct drm_i915_private *dev_priv = dev->dev_private;
3537         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
3538         struct drm_crtc *crtc;
3539
3540         skl_ddb_get_hw_state(dev_priv, ddb);
3541         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
3542                 skl_pipe_wm_get_hw_state(crtc);
3543 }
3544
3545 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
3546 {
3547         struct drm_device *dev = crtc->dev;
3548         struct drm_i915_private *dev_priv = dev->dev_private;
3549         struct ilk_wm_values *hw = &dev_priv->wm.hw;
3550         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3551         struct intel_pipe_wm *active = &intel_crtc->wm.active;
3552         enum pipe pipe = intel_crtc->pipe;
3553         static const unsigned int wm0_pipe_reg[] = {
3554                 [PIPE_A] = WM0_PIPEA_ILK,
3555                 [PIPE_B] = WM0_PIPEB_ILK,
3556                 [PIPE_C] = WM0_PIPEC_IVB,
3557         };
3558
3559         hw->wm_pipe[pipe] = I915_READ(wm0_pipe_reg[pipe]);
3560         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
3561                 hw->wm_linetime[pipe] = I915_READ(PIPE_WM_LINETIME(pipe));
3562
3563         active->pipe_enabled = intel_crtc->active;
3564
3565         if (active->pipe_enabled) {
3566                 u32 tmp = hw->wm_pipe[pipe];
3567
3568                 /*
3569                  * For active pipes LP0 watermark is marked as
3570                  * enabled, and LP1+ watermaks as disabled since
3571                  * we can't really reverse compute them in case
3572                  * multiple pipes are active.
3573                  */
3574                 active->wm[0].enable = true;
3575                 active->wm[0].pri_val = (tmp & WM0_PIPE_PLANE_MASK) >> WM0_PIPE_PLANE_SHIFT;
3576                 active->wm[0].spr_val = (tmp & WM0_PIPE_SPRITE_MASK) >> WM0_PIPE_SPRITE_SHIFT;
3577                 active->wm[0].cur_val = tmp & WM0_PIPE_CURSOR_MASK;
3578                 active->linetime = hw->wm_linetime[pipe];
3579         } else {
3580                 int level, max_level = ilk_wm_max_level(dev);
3581
3582                 /*
3583                  * For inactive pipes, all watermark levels
3584                  * should be marked as enabled but zeroed,
3585                  * which is what we'd compute them to.
3586                  */
3587                 for (level = 0; level <= max_level; level++)
3588                         active->wm[level].enable = true;
3589         }
3590 }
3591
3592 void ilk_wm_get_hw_state(struct drm_device *dev)
3593 {
3594         struct drm_i915_private *dev_priv = dev->dev_private;
3595         struct ilk_wm_values *hw = &dev_priv->wm.hw;
3596         struct drm_crtc *crtc;
3597
3598         for_each_crtc(dev, crtc)
3599                 ilk_pipe_wm_get_hw_state(crtc);
3600
3601         hw->wm_lp[0] = I915_READ(WM1_LP_ILK);
3602         hw->wm_lp[1] = I915_READ(WM2_LP_ILK);
3603         hw->wm_lp[2] = I915_READ(WM3_LP_ILK);
3604
3605         hw->wm_lp_spr[0] = I915_READ(WM1S_LP_ILK);
3606         if (INTEL_INFO(dev)->gen >= 7) {
3607                 hw->wm_lp_spr[1] = I915_READ(WM2S_LP_IVB);
3608                 hw->wm_lp_spr[2] = I915_READ(WM3S_LP_IVB);
3609         }
3610
3611         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
3612                 hw->partitioning = (I915_READ(WM_MISC) & WM_MISC_DATA_PARTITION_5_6) ?
3613                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3614         else if (IS_IVYBRIDGE(dev))
3615                 hw->partitioning = (I915_READ(DISP_ARB_CTL2) & DISP_DATA_PARTITION_5_6) ?
3616                         INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3617
3618         hw->enable_fbc_wm =
3619                 !(I915_READ(DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3620 }
3621
3622 /**
3623  * intel_update_watermarks - update FIFO watermark values based on current modes
3624  *
3625  * Calculate watermark values for the various WM regs based on current mode
3626  * and plane configuration.
3627  *
3628  * There are several cases to deal with here:
3629  *   - normal (i.e. non-self-refresh)
3630  *   - self-refresh (SR) mode
3631  *   - lines are large relative to FIFO size (buffer can hold up to 2)
3632  *   - lines are small relative to FIFO size (buffer can hold more than 2
3633  *     lines), so need to account for TLB latency
3634  *
3635  *   The normal calculation is:
3636  *     watermark = dotclock * bytes per pixel * latency
3637  *   where latency is platform & configuration dependent (we assume pessimal
3638  *   values here).
3639  *
3640  *   The SR calculation is:
3641  *     watermark = (trunc(latency/line time)+1) * surface width *
3642  *       bytes per pixel
3643  *   where
3644  *     line time = htotal / dotclock
3645  *     surface width = hdisplay for normal plane and 64 for cursor
3646  *   and latency is assumed to be high, as above.
3647  *
3648  * The final value programmed to the register should always be rounded up,
3649  * and include an extra 2 entries to account for clock crossings.
3650  *
3651  * We don't use the sprite, so we can ignore that.  And on Crestline we have
3652  * to set the non-SR watermarks to 8.
3653  */
3654 void intel_update_watermarks(struct drm_crtc *crtc)
3655 {
3656         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
3657
3658         if (dev_priv->display.update_wm)
3659                 dev_priv->display.update_wm(crtc);
3660 }
3661
3662 void intel_update_sprite_watermarks(struct drm_plane *plane,
3663                                     struct drm_crtc *crtc,
3664                                     uint32_t sprite_width,
3665                                     uint32_t sprite_height,
3666                                     int pixel_size,
3667                                     bool enabled, bool scaled)
3668 {
3669         struct drm_i915_private *dev_priv = plane->dev->dev_private;
3670
3671         if (dev_priv->display.update_sprite_wm)
3672                 dev_priv->display.update_sprite_wm(plane, crtc,
3673                                                    sprite_width, sprite_height,
3674                                                    pixel_size, enabled, scaled);
3675 }
3676
3677 /**
3678  * Lock protecting IPS related data structures
3679  */
3680 DEFINE_SPINLOCK(mchdev_lock);
3681
3682 /* Global for IPS driver to get at the current i915 device. Protected by
3683  * mchdev_lock. */
3684 static struct drm_i915_private *i915_mch_dev;
3685
3686 bool ironlake_set_drps(struct drm_device *dev, u8 val)
3687 {
3688         struct drm_i915_private *dev_priv = dev->dev_private;
3689         u16 rgvswctl;
3690
3691         assert_spin_locked(&mchdev_lock);
3692
3693         rgvswctl = I915_READ16(MEMSWCTL);
3694         if (rgvswctl & MEMCTL_CMD_STS) {
3695                 DRM_DEBUG("gpu busy, RCS change rejected\n");
3696                 return false; /* still busy with another command */
3697         }
3698
3699         rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
3700                 (val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
3701         I915_WRITE16(MEMSWCTL, rgvswctl);
3702         POSTING_READ16(MEMSWCTL);
3703
3704         rgvswctl |= MEMCTL_CMD_STS;
3705         I915_WRITE16(MEMSWCTL, rgvswctl);
3706
3707         return true;
3708 }
3709
3710 static void ironlake_enable_drps(struct drm_device *dev)
3711 {
3712         struct drm_i915_private *dev_priv = dev->dev_private;
3713         u32 rgvmodectl = I915_READ(MEMMODECTL);
3714         u8 fmax, fmin, fstart, vstart;
3715
3716         spin_lock_irq(&mchdev_lock);
3717
3718         /* Enable temp reporting */
3719         I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
3720         I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
3721
3722         /* 100ms RC evaluation intervals */
3723         I915_WRITE(RCUPEI, 100000);
3724         I915_WRITE(RCDNEI, 100000);
3725
3726         /* Set max/min thresholds to 90ms and 80ms respectively */
3727         I915_WRITE(RCBMAXAVG, 90000);
3728         I915_WRITE(RCBMINAVG, 80000);
3729
3730         I915_WRITE(MEMIHYST, 1);
3731
3732         /* Set up min, max, and cur for interrupt handling */
3733         fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
3734         fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
3735         fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
3736                 MEMMODE_FSTART_SHIFT;
3737
3738         vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
3739                 PXVFREQ_PX_SHIFT;
3740
3741         dev_priv->ips.fmax = fmax; /* IPS callback will increase this */
3742         dev_priv->ips.fstart = fstart;
3743
3744         dev_priv->ips.max_delay = fstart;
3745         dev_priv->ips.min_delay = fmin;
3746         dev_priv->ips.cur_delay = fstart;
3747
3748         DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
3749                          fmax, fmin, fstart);
3750
3751         I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
3752
3753         /*
3754          * Interrupts will be enabled in ironlake_irq_postinstall
3755          */
3756
3757         I915_WRITE(VIDSTART, vstart);
3758         POSTING_READ(VIDSTART);
3759
3760         rgvmodectl |= MEMMODE_SWMODE_EN;
3761         I915_WRITE(MEMMODECTL, rgvmodectl);
3762
3763         if (wait_for_atomic((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
3764                 DRM_ERROR("stuck trying to change perf mode\n");
3765         mdelay(1);
3766
3767         ironlake_set_drps(dev, fstart);
3768
3769         dev_priv->ips.last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
3770                 I915_READ(0x112e0);
3771         dev_priv->ips.last_time1 = jiffies_to_msecs(jiffies);
3772         dev_priv->ips.last_count2 = I915_READ(0x112f4);
3773         dev_priv->ips.last_time2 = ktime_get_raw_ns();
3774
3775         spin_unlock_irq(&mchdev_lock);
3776 }
3777
3778 static void ironlake_disable_drps(struct drm_device *dev)
3779 {
3780         struct drm_i915_private *dev_priv = dev->dev_private;
3781         u16 rgvswctl;
3782
3783         spin_lock_irq(&mchdev_lock);
3784
3785         rgvswctl = I915_READ16(MEMSWCTL);
3786
3787         /* Ack interrupts, disable EFC interrupt */
3788         I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
3789         I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
3790         I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
3791         I915_WRITE(DEIIR, DE_PCU_EVENT);
3792         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
3793
3794         /* Go back to the starting frequency */
3795         ironlake_set_drps(dev, dev_priv->ips.fstart);
3796         mdelay(1);
3797         rgvswctl |= MEMCTL_CMD_STS;
3798         I915_WRITE(MEMSWCTL, rgvswctl);
3799         mdelay(1);
3800
3801         spin_unlock_irq(&mchdev_lock);
3802 }
3803
3804 /* There's a funny hw issue where the hw returns all 0 when reading from
3805  * GEN6_RP_INTERRUPT_LIMITS. Hence we always need to compute the desired value
3806  * ourselves, instead of doing a rmw cycle (which might result in us clearing
3807  * all limits and the gpu stuck at whatever frequency it is at atm).
3808  */
3809 static u32 intel_rps_limits(struct drm_i915_private *dev_priv, u8 val)
3810 {
3811         u32 limits;
3812
3813         /* Only set the down limit when we've reached the lowest level to avoid
3814          * getting more interrupts, otherwise leave this clear. This prevents a
3815          * race in the hw when coming out of rc6: There's a tiny window where
3816          * the hw runs at the minimal clock before selecting the desired
3817          * frequency, if the down threshold expires in that window we will not
3818          * receive a down interrupt. */
3819         if (IS_GEN9(dev_priv->dev)) {
3820                 limits = (dev_priv->rps.max_freq_softlimit) << 23;
3821                 if (val <= dev_priv->rps.min_freq_softlimit)
3822                         limits |= (dev_priv->rps.min_freq_softlimit) << 14;
3823         } else {
3824                 limits = dev_priv->rps.max_freq_softlimit << 24;
3825                 if (val <= dev_priv->rps.min_freq_softlimit)
3826                         limits |= dev_priv->rps.min_freq_softlimit << 16;
3827         }
3828
3829         return limits;
3830 }
3831
3832 static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
3833 {
3834         int new_power;
3835         u32 threshold_up = 0, threshold_down = 0; /* in % */
3836         u32 ei_up = 0, ei_down = 0;
3837
3838         new_power = dev_priv->rps.power;
3839         switch (dev_priv->rps.power) {
3840         case LOW_POWER:
3841                 if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
3842                         new_power = BETWEEN;
3843                 break;
3844
3845         case BETWEEN:
3846                 if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
3847                         new_power = LOW_POWER;
3848                 else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
3849                         new_power = HIGH_POWER;
3850                 break;
3851
3852         case HIGH_POWER:
3853                 if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
3854                         new_power = BETWEEN;
3855                 break;
3856         }
3857         /* Max/min bins are special */
3858         if (val == dev_priv->rps.min_freq_softlimit)
3859                 new_power = LOW_POWER;
3860         if (val == dev_priv->rps.max_freq_softlimit)
3861                 new_power = HIGH_POWER;
3862         if (new_power == dev_priv->rps.power)
3863                 return;
3864
3865         /* Note the units here are not exactly 1us, but 1280ns. */
3866         switch (new_power) {
3867         case LOW_POWER:
3868                 /* Upclock if more than 95% busy over 16ms */
3869                 ei_up = 16000;
3870                 threshold_up = 95;
3871
3872                 /* Downclock if less than 85% busy over 32ms */
3873                 ei_down = 32000;
3874                 threshold_down = 85;
3875                 break;
3876
3877         case BETWEEN:
3878                 /* Upclock if more than 90% busy over 13ms */
3879                 ei_up = 13000;
3880                 threshold_up = 90;
3881
3882                 /* Downclock if less than 75% busy over 32ms */
3883                 ei_down = 32000;
3884                 threshold_down = 75;
3885                 break;
3886
3887         case HIGH_POWER:
3888                 /* Upclock if more than 85% busy over 10ms */
3889                 ei_up = 10000;
3890                 threshold_up = 85;
3891
3892                 /* Downclock if less than 60% busy over 32ms */
3893                 ei_down = 32000;
3894                 threshold_down = 60;
3895                 break;
3896         }
3897
3898         I915_WRITE(GEN6_RP_UP_EI,
3899                 GT_INTERVAL_FROM_US(dev_priv, ei_up));
3900         I915_WRITE(GEN6_RP_UP_THRESHOLD,
3901                 GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100)));
3902
3903         I915_WRITE(GEN6_RP_DOWN_EI,
3904                 GT_INTERVAL_FROM_US(dev_priv, ei_down));
3905         I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
3906                 GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100)));
3907
3908          I915_WRITE(GEN6_RP_CONTROL,
3909                     GEN6_RP_MEDIA_TURBO |
3910                     GEN6_RP_MEDIA_HW_NORMAL_MODE |
3911                     GEN6_RP_MEDIA_IS_GFX |
3912                     GEN6_RP_ENABLE |
3913                     GEN6_RP_UP_BUSY_AVG |
3914                     GEN6_RP_DOWN_IDLE_AVG);
3915
3916         dev_priv->rps.power = new_power;
3917         dev_priv->rps.last_adj = 0;
3918 }
3919
3920 static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
3921 {
3922         u32 mask = 0;
3923
3924         if (val > dev_priv->rps.min_freq_softlimit)
3925                 mask |= GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
3926         if (val < dev_priv->rps.max_freq_softlimit)
3927                 mask |= GEN6_PM_RP_UP_THRESHOLD;
3928
3929         mask |= dev_priv->pm_rps_events & (GEN6_PM_RP_DOWN_EI_EXPIRED | GEN6_PM_RP_UP_EI_EXPIRED);
3930         mask &= dev_priv->pm_rps_events;
3931
3932         return gen6_sanitize_rps_pm_mask(dev_priv, ~mask);
3933 }
3934
3935 /* gen6_set_rps is called to update the frequency request, but should also be
3936  * called when the range (min_delay and max_delay) is modified so that we can
3937  * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
3938 static void gen6_set_rps(struct drm_device *dev, u8 val)
3939 {
3940         struct drm_i915_private *dev_priv = dev->dev_private;
3941
3942         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3943         WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3944         WARN_ON(val < dev_priv->rps.min_freq_softlimit);
3945
3946         /* min/max delay may still have been modified so be sure to
3947          * write the limits value.
3948          */
3949         if (val != dev_priv->rps.cur_freq) {
3950                 gen6_set_rps_thresholds(dev_priv, val);
3951
3952                 if (IS_GEN9(dev))
3953                         I915_WRITE(GEN6_RPNSWREQ,
3954                                    GEN9_FREQUENCY(val));
3955                 else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
3956                         I915_WRITE(GEN6_RPNSWREQ,
3957                                    HSW_FREQUENCY(val));
3958                 else
3959                         I915_WRITE(GEN6_RPNSWREQ,
3960                                    GEN6_FREQUENCY(val) |
3961                                    GEN6_OFFSET(0) |
3962                                    GEN6_AGGRESSIVE_TURBO);
3963         }
3964
3965         /* Make sure we continue to get interrupts
3966          * until we hit the minimum or maximum frequencies.
3967          */
3968         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, intel_rps_limits(dev_priv, val));
3969         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3970
3971         POSTING_READ(GEN6_RPNSWREQ);
3972
3973         dev_priv->rps.cur_freq = val;
3974         trace_intel_gpu_freq_change(val * 50);
3975 }
3976
3977 static void valleyview_set_rps(struct drm_device *dev, u8 val)
3978 {
3979         struct drm_i915_private *dev_priv = dev->dev_private;
3980
3981         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
3982         WARN_ON(val > dev_priv->rps.max_freq_softlimit);
3983         WARN_ON(val < dev_priv->rps.min_freq_softlimit);
3984
3985         if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
3986                       "Odd GPU freq value\n"))
3987                 val &= ~1;
3988
3989         if (val != dev_priv->rps.cur_freq)
3990                 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
3991
3992         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
3993
3994         dev_priv->rps.cur_freq = val;
3995         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
3996 }
3997
3998 /* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down
3999  *
4000  * * If Gfx is Idle, then
4001  * 1. Mask Turbo interrupts
4002  * 2. Bring up Gfx clock
4003  * 3. Change the freq to Rpn and wait till P-Unit updates freq
4004  * 4. Clear the Force GFX CLK ON bit so that Gfx can down
4005  * 5. Unmask Turbo interrupts
4006 */
4007 static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
4008 {
4009         struct drm_device *dev = dev_priv->dev;
4010
4011         /* CHV and latest VLV don't need to force the gfx clock */
4012         if (IS_CHERRYVIEW(dev) || dev->pdev->revision >= 0xd) {
4013                 valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4014                 return;
4015         }
4016
4017         /*
4018          * When we are idle.  Drop to min voltage state.
4019          */
4020
4021         if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit)
4022                 return;
4023
4024         /* Mask turbo interrupt so that they will not come in between */
4025         I915_WRITE(GEN6_PMINTRMSK,
4026                    gen6_sanitize_rps_pm_mask(dev_priv, ~0));
4027
4028         vlv_force_gfx_clock(dev_priv, true);
4029
4030         dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit;
4031
4032         vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ,
4033                                         dev_priv->rps.min_freq_softlimit);
4034
4035         if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS))
4036                                 & GENFREQSTATUS) == 0, 100))
4037                 DRM_ERROR("timed out waiting for Punit\n");
4038
4039         vlv_force_gfx_clock(dev_priv, false);
4040
4041         I915_WRITE(GEN6_PMINTRMSK,
4042                    gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
4043 }
4044
4045 void gen6_rps_idle(struct drm_i915_private *dev_priv)
4046 {
4047         struct drm_device *dev = dev_priv->dev;
4048
4049         mutex_lock(&dev_priv->rps.hw_lock);
4050         if (dev_priv->rps.enabled) {
4051                 if (IS_VALLEYVIEW(dev))
4052                         vlv_set_rps_idle(dev_priv);
4053                 else
4054                         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4055                 dev_priv->rps.last_adj = 0;
4056         }
4057         mutex_unlock(&dev_priv->rps.hw_lock);
4058 }
4059
4060 void gen6_rps_boost(struct drm_i915_private *dev_priv)
4061 {
4062         mutex_lock(&dev_priv->rps.hw_lock);
4063         if (dev_priv->rps.enabled) {
4064                 intel_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
4065                 dev_priv->rps.last_adj = 0;
4066         }
4067         mutex_unlock(&dev_priv->rps.hw_lock);
4068 }
4069
4070 void intel_set_rps(struct drm_device *dev, u8 val)
4071 {
4072         if (IS_VALLEYVIEW(dev))
4073                 valleyview_set_rps(dev, val);
4074         else
4075                 gen6_set_rps(dev, val);
4076 }
4077
4078 static void gen9_disable_rps(struct drm_device *dev)
4079 {
4080         struct drm_i915_private *dev_priv = dev->dev_private;
4081
4082         I915_WRITE(GEN6_RC_CONTROL, 0);
4083         I915_WRITE(GEN9_PG_ENABLE, 0);
4084 }
4085
4086 static void gen6_disable_rps(struct drm_device *dev)
4087 {
4088         struct drm_i915_private *dev_priv = dev->dev_private;
4089
4090         I915_WRITE(GEN6_RC_CONTROL, 0);
4091         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
4092 }
4093
4094 static void cherryview_disable_rps(struct drm_device *dev)
4095 {
4096         struct drm_i915_private *dev_priv = dev->dev_private;
4097
4098         I915_WRITE(GEN6_RC_CONTROL, 0);
4099 }
4100
4101 static void valleyview_disable_rps(struct drm_device *dev)
4102 {
4103         struct drm_i915_private *dev_priv = dev->dev_private;
4104
4105         /* we're doing forcewake before Disabling RC6,
4106          * This what the BIOS expects when going into suspend */
4107         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4108
4109         I915_WRITE(GEN6_RC_CONTROL, 0);
4110
4111         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4112 }
4113
4114 static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
4115 {
4116         if (IS_VALLEYVIEW(dev)) {
4117                 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
4118                         mode = GEN6_RC_CTL_RC6_ENABLE;
4119                 else
4120                         mode = 0;
4121         }
4122         if (HAS_RC6p(dev))
4123                 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
4124                               (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
4125                               (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
4126                               (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
4127
4128         else
4129                 DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
4130                               (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
4131 }
4132
4133 static int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
4134 {
4135         /* No RC6 before Ironlake */
4136         if (INTEL_INFO(dev)->gen < 5)
4137                 return 0;
4138
4139         /* RC6 is only on Ironlake mobile not on desktop */
4140         if (INTEL_INFO(dev)->gen == 5 && !IS_IRONLAKE_M(dev))
4141                 return 0;
4142
4143         /* Respect the kernel parameter if it is set */
4144         if (enable_rc6 >= 0) {
4145                 int mask;
4146
4147                 if (HAS_RC6p(dev))
4148                         mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
4149                                INTEL_RC6pp_ENABLE;
4150                 else
4151                         mask = INTEL_RC6_ENABLE;
4152
4153                 if ((enable_rc6 & mask) != enable_rc6)
4154                         DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
4155                                       enable_rc6 & mask, enable_rc6, mask);
4156
4157                 return enable_rc6 & mask;
4158         }
4159
4160         /* Disable RC6 on Ironlake */
4161         if (INTEL_INFO(dev)->gen == 5)
4162                 return 0;
4163
4164         if (IS_IVYBRIDGE(dev))
4165                 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
4166
4167         return INTEL_RC6_ENABLE;
4168 }
4169
4170 int intel_enable_rc6(const struct drm_device *dev)
4171 {
4172         return i915.enable_rc6;
4173 }
4174
4175 static void gen6_init_rps_frequencies(struct drm_device *dev)
4176 {
4177         struct drm_i915_private *dev_priv = dev->dev_private;
4178         uint32_t rp_state_cap;
4179         u32 ddcc_status = 0;
4180         int ret;
4181
4182         rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
4183         /* All of these values are in units of 50MHz */
4184         dev_priv->rps.cur_freq          = 0;
4185         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
4186         dev_priv->rps.rp0_freq          = (rp_state_cap >>  0) & 0xff;
4187         dev_priv->rps.rp1_freq          = (rp_state_cap >>  8) & 0xff;
4188         dev_priv->rps.min_freq          = (rp_state_cap >> 16) & 0xff;
4189         if (IS_SKYLAKE(dev)) {
4190                 /* Store the frequency values in 16.66 MHZ units, which is
4191                    the natural hardware unit for SKL */
4192                 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
4193                 dev_priv->rps.rp1_freq *= GEN9_FREQ_SCALER;
4194                 dev_priv->rps.min_freq *= GEN9_FREQ_SCALER;
4195         }
4196         /* hw_max = RP0 until we check for overclocking */
4197         dev_priv->rps.max_freq          = dev_priv->rps.rp0_freq;
4198
4199         dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
4200         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
4201                 ret = sandybridge_pcode_read(dev_priv,
4202                                         HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
4203                                         &ddcc_status);
4204                 if (0 == ret)
4205                         dev_priv->rps.efficient_freq =
4206                                 clamp_t(u8,
4207                                         ((ddcc_status >> 8) & 0xff),
4208                                         dev_priv->rps.min_freq,
4209                                         dev_priv->rps.max_freq);
4210         }
4211
4212         /* Preserve min/max settings in case of re-init */
4213         if (dev_priv->rps.max_freq_softlimit == 0)
4214                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4215
4216         if (dev_priv->rps.min_freq_softlimit == 0) {
4217                 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
4218                         dev_priv->rps.min_freq_softlimit =
4219                                 /* max(RPe, 450 MHz) */
4220                                 max(dev_priv->rps.efficient_freq, (u8) 9);
4221                 else
4222                         dev_priv->rps.min_freq_softlimit =
4223                                 dev_priv->rps.min_freq;
4224         }
4225 }
4226
4227 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
4228 static void gen9_enable_rps(struct drm_device *dev)
4229 {
4230         struct drm_i915_private *dev_priv = dev->dev_private;
4231
4232         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4233
4234         gen6_init_rps_frequencies(dev);
4235
4236         /* Program defaults and thresholds for RPS*/
4237         I915_WRITE(GEN6_RC_VIDEO_FREQ,
4238                 GEN9_FREQUENCY(dev_priv->rps.rp1_freq));
4239
4240         /* 1 second timeout*/
4241         I915_WRITE(GEN6_RP_DOWN_TIMEOUT,
4242                 GT_INTERVAL_FROM_US(dev_priv, 1000000));
4243
4244         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 0xa);
4245
4246         /* Leaning on the below call to gen6_set_rps to program/setup the
4247          * Up/Down EI & threshold registers, as well as the RP_CONTROL,
4248          * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
4249         dev_priv->rps.power = HIGH_POWER; /* force a reset */
4250         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4251
4252         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4253 }
4254
4255 static void gen9_enable_rc6(struct drm_device *dev)
4256 {
4257         struct drm_i915_private *dev_priv = dev->dev_private;
4258         struct intel_engine_cs *ring;
4259         uint32_t rc6_mask = 0;
4260         int unused;
4261
4262         /* 1a: Software RC state - RC0 */
4263         I915_WRITE(GEN6_RC_STATE, 0);
4264
4265         /* 1b: Get forcewake during program sequence. Although the driver
4266          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
4267         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4268
4269         /* 2a: Disable RC states. */
4270         I915_WRITE(GEN6_RC_CONTROL, 0);
4271
4272         /* 2b: Program RC6 thresholds.*/
4273         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
4274         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4275         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4276         for_each_ring(ring, dev_priv, unused)
4277                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4278         I915_WRITE(GEN6_RC_SLEEP, 0);
4279         I915_WRITE(GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
4280
4281         /* 2c: Program Coarse Power Gating Policies. */
4282         I915_WRITE(GEN9_MEDIA_PG_IDLE_HYSTERESIS, 25);
4283         I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
4284
4285         /* 3a: Enable RC6 */
4286         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4287                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4288         DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
4289                         "on" : "off");
4290         I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4291                                    GEN6_RC_CTL_EI_MODE(1) |
4292                                    rc6_mask);
4293
4294         /* 3b: Enable Coarse Power Gating only when RC6 is enabled */
4295         I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? 3 : 0);
4296
4297         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4298
4299 }
4300
4301 static void gen8_enable_rps(struct drm_device *dev)
4302 {
4303         struct drm_i915_private *dev_priv = dev->dev_private;
4304         struct intel_engine_cs *ring;
4305         uint32_t rc6_mask = 0;
4306         int unused;
4307
4308         /* 1a: Software RC state - RC0 */
4309         I915_WRITE(GEN6_RC_STATE, 0);
4310
4311         /* 1c & 1d: Get forcewake during program sequence. Although the driver
4312          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
4313         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4314
4315         /* 2a: Disable RC states. */
4316         I915_WRITE(GEN6_RC_CONTROL, 0);
4317
4318         /* Initialize rps frequencies */
4319         gen6_init_rps_frequencies(dev);
4320
4321         /* 2b: Program RC6 thresholds.*/
4322         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
4323         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4324         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4325         for_each_ring(ring, dev_priv, unused)
4326                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4327         I915_WRITE(GEN6_RC_SLEEP, 0);
4328         if (IS_BROADWELL(dev))
4329                 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
4330         else
4331                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
4332
4333         /* 3: Enable RC6 */
4334         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
4335                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
4336         intel_print_rc6_info(dev, rc6_mask);
4337         if (IS_BROADWELL(dev))
4338                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4339                                 GEN7_RC_CTL_TO_MODE |
4340                                 rc6_mask);
4341         else
4342                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
4343                                 GEN6_RC_CTL_EI_MODE(1) |
4344                                 rc6_mask);
4345
4346         /* 4 Program defaults and thresholds for RPS*/
4347         I915_WRITE(GEN6_RPNSWREQ,
4348                    HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4349         I915_WRITE(GEN6_RC_VIDEO_FREQ,
4350                    HSW_FREQUENCY(dev_priv->rps.rp1_freq));
4351         /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
4352         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
4353
4354         /* Docs recommend 900MHz, and 300 MHz respectively */
4355         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
4356                    dev_priv->rps.max_freq_softlimit << 24 |
4357                    dev_priv->rps.min_freq_softlimit << 16);
4358
4359         I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
4360         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
4361         I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
4362         I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
4363
4364         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4365
4366         /* 5: Enable RPS */
4367         I915_WRITE(GEN6_RP_CONTROL,
4368                    GEN6_RP_MEDIA_TURBO |
4369                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
4370                    GEN6_RP_MEDIA_IS_GFX |
4371                    GEN6_RP_ENABLE |
4372                    GEN6_RP_UP_BUSY_AVG |
4373                    GEN6_RP_DOWN_IDLE_AVG);
4374
4375         /* 6: Ring frequency + overclocking (our driver does this later */
4376
4377         dev_priv->rps.power = HIGH_POWER; /* force a reset */
4378         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4379
4380         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4381 }
4382
4383 static void gen6_enable_rps(struct drm_device *dev)
4384 {
4385         struct drm_i915_private *dev_priv = dev->dev_private;
4386         struct intel_engine_cs *ring;
4387         u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
4388         u32 gtfifodbg;
4389         int rc6_mode;
4390         int i, ret;
4391
4392         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4393
4394         /* Here begins a magic sequence of register writes to enable
4395          * auto-downclocking.
4396          *
4397          * Perhaps there might be some value in exposing these to
4398          * userspace...
4399          */
4400         I915_WRITE(GEN6_RC_STATE, 0);
4401
4402         /* Clear the DBG now so we don't confuse earlier errors */
4403         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
4404                 DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
4405                 I915_WRITE(GTFIFODBG, gtfifodbg);
4406         }
4407
4408         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4409
4410         /* Initialize rps frequencies */
4411         gen6_init_rps_frequencies(dev);
4412
4413         /* disable the counters and set deterministic thresholds */
4414         I915_WRITE(GEN6_RC_CONTROL, 0);
4415
4416         I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
4417         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
4418         I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
4419         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
4420         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
4421
4422         for_each_ring(ring, dev_priv, i)
4423                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4424
4425         I915_WRITE(GEN6_RC_SLEEP, 0);
4426         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
4427         if (IS_IVYBRIDGE(dev))
4428                 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
4429         else
4430                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
4431         I915_WRITE(GEN6_RC6p_THRESHOLD, 150000);
4432         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
4433
4434         /* Check if we are enabling RC6 */
4435         rc6_mode = intel_enable_rc6(dev_priv->dev);
4436         if (rc6_mode & INTEL_RC6_ENABLE)
4437                 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
4438
4439         /* We don't use those on Haswell */
4440         if (!IS_HASWELL(dev)) {
4441                 if (rc6_mode & INTEL_RC6p_ENABLE)
4442                         rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
4443
4444                 if (rc6_mode & INTEL_RC6pp_ENABLE)
4445                         rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
4446         }
4447
4448         intel_print_rc6_info(dev, rc6_mask);
4449
4450         I915_WRITE(GEN6_RC_CONTROL,
4451                    rc6_mask |
4452                    GEN6_RC_CTL_EI_MODE(1) |
4453                    GEN6_RC_CTL_HW_ENABLE);
4454
4455         /* Power down if completely idle for over 50ms */
4456         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 50000);
4457         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4458
4459         ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
4460         if (ret)
4461                 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
4462
4463         ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
4464         if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
4465                 DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
4466                                  (dev_priv->rps.max_freq_softlimit & 0xff) * 50,
4467                                  (pcu_mbox & 0xff) * 50);
4468                 dev_priv->rps.max_freq = pcu_mbox & 0xff;
4469         }
4470
4471         dev_priv->rps.power = HIGH_POWER; /* force a reset */
4472         gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
4473
4474         rc6vids = 0;
4475         ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
4476         if (IS_GEN6(dev) && ret) {
4477                 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
4478         } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
4479                 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
4480                           GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
4481                 rc6vids &= 0xffff00;
4482                 rc6vids |= GEN6_ENCODE_RC6_VID(450);
4483                 ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
4484                 if (ret)
4485                         DRM_ERROR("Couldn't fix incorrect rc6 voltage\n");
4486         }
4487
4488         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4489 }
4490
4491 static void __gen6_update_ring_freq(struct drm_device *dev)
4492 {
4493         struct drm_i915_private *dev_priv = dev->dev_private;
4494         int min_freq = 15;
4495         unsigned int gpu_freq;
4496         unsigned int max_ia_freq, min_ring_freq;
4497         int scaling_factor = 180;
4498         struct cpufreq_policy *policy;
4499
4500         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4501
4502         policy = cpufreq_cpu_get(0);
4503         if (policy) {
4504                 max_ia_freq = policy->cpuinfo.max_freq;
4505                 cpufreq_cpu_put(policy);
4506         } else {
4507                 /*
4508                  * Default to measured freq if none found, PCU will ensure we
4509                  * don't go over
4510                  */
4511                 max_ia_freq = tsc_khz;
4512         }
4513
4514         /* Convert from kHz to MHz */
4515         max_ia_freq /= 1000;
4516
4517         min_ring_freq = I915_READ(DCLK) & 0xf;
4518         /* convert DDR frequency from units of 266.6MHz to bandwidth */
4519         min_ring_freq = mult_frac(min_ring_freq, 8, 3);
4520
4521         /*
4522          * For each potential GPU frequency, load a ring frequency we'd like
4523          * to use for memory access.  We do this by specifying the IA frequency
4524          * the PCU should use as a reference to determine the ring frequency.
4525          */
4526         for (gpu_freq = dev_priv->rps.max_freq; gpu_freq >= dev_priv->rps.min_freq;
4527              gpu_freq--) {
4528                 int diff = dev_priv->rps.max_freq - gpu_freq;
4529                 unsigned int ia_freq = 0, ring_freq = 0;
4530
4531                 if (INTEL_INFO(dev)->gen >= 8) {
4532                         /* max(2 * GT, DDR). NB: GT is 50MHz units */
4533                         ring_freq = max(min_ring_freq, gpu_freq);
4534                 } else if (IS_HASWELL(dev)) {
4535                         ring_freq = mult_frac(gpu_freq, 5, 4);
4536                         ring_freq = max(min_ring_freq, ring_freq);
4537                         /* leave ia_freq as the default, chosen by cpufreq */
4538                 } else {
4539                         /* On older processors, there is no separate ring
4540                          * clock domain, so in order to boost the bandwidth
4541                          * of the ring, we need to upclock the CPU (ia_freq).
4542                          *
4543                          * For GPU frequencies less than 750MHz,
4544                          * just use the lowest ring freq.
4545                          */
4546                         if (gpu_freq < min_freq)
4547                                 ia_freq = 800;
4548                         else
4549                                 ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
4550                         ia_freq = DIV_ROUND_CLOSEST(ia_freq, 100);
4551                 }
4552
4553                 sandybridge_pcode_write(dev_priv,
4554                                         GEN6_PCODE_WRITE_MIN_FREQ_TABLE,
4555                                         ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT |
4556                                         ring_freq << GEN6_PCODE_FREQ_RING_RATIO_SHIFT |
4557                                         gpu_freq);
4558         }
4559 }
4560
4561 void gen6_update_ring_freq(struct drm_device *dev)
4562 {
4563         struct drm_i915_private *dev_priv = dev->dev_private;
4564
4565         if (INTEL_INFO(dev)->gen < 6 || IS_VALLEYVIEW(dev))
4566                 return;
4567
4568         mutex_lock(&dev_priv->rps.hw_lock);
4569         __gen6_update_ring_freq(dev);
4570         mutex_unlock(&dev_priv->rps.hw_lock);
4571 }
4572
4573 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
4574 {
4575         struct drm_device *dev = dev_priv->dev;
4576         u32 val, rp0;
4577
4578         if (dev->pdev->revision >= 0x20) {
4579                 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
4580
4581                 switch (INTEL_INFO(dev)->eu_total) {
4582                 case 8:
4583                                 /* (2 * 4) config */
4584                                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
4585                                 break;
4586                 case 12:
4587                                 /* (2 * 6) config */
4588                                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT);
4589                                 break;
4590                 case 16:
4591                                 /* (2 * 8) config */
4592                 default:
4593                                 /* Setting (2 * 8) Min RP0 for any other combination */
4594                                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT);
4595                                 break;
4596                 }
4597                 rp0 = (rp0 & FB_GFX_FREQ_FUSE_MASK);
4598         } else {
4599                 /* For pre-production hardware */
4600                 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
4601                 rp0 = (val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
4602                        PUNIT_GPU_STATUS_MAX_FREQ_MASK;
4603         }
4604         return rp0;
4605 }
4606
4607 static int cherryview_rps_rpe_freq(struct drm_i915_private *dev_priv)
4608 {
4609         u32 val, rpe;
4610
4611         val = vlv_punit_read(dev_priv, PUNIT_GPU_DUTYCYCLE_REG);
4612         rpe = (val >> PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT) & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
4613
4614         return rpe;
4615 }
4616
4617 static int cherryview_rps_guar_freq(struct drm_i915_private *dev_priv)
4618 {
4619         struct drm_device *dev = dev_priv->dev;
4620         u32 val, rp1;
4621
4622         if (dev->pdev->revision >= 0x20) {
4623                 val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
4624                 rp1 = (val & FB_GFX_FREQ_FUSE_MASK);
4625         } else {
4626                 /* For pre-production hardware */
4627                 val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4628                 rp1 = ((val >> PUNIT_GPU_STATUS_MAX_FREQ_SHIFT) &
4629                        PUNIT_GPU_STATUS_MAX_FREQ_MASK);
4630         }
4631         return rp1;
4632 }
4633
4634 static int cherryview_rps_min_freq(struct drm_i915_private *dev_priv)
4635 {
4636         struct drm_device *dev = dev_priv->dev;
4637         u32 val, rpn;
4638
4639         if (dev->pdev->revision >= 0x20) {
4640                 val = vlv_punit_read(dev_priv, FB_GFX_FMIN_AT_VMIN_FUSE);
4641                 rpn = ((val >> FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT) &
4642                        FB_GFX_FREQ_FUSE_MASK);
4643         } else { /* For pre-production hardware */
4644                 val = vlv_punit_read(dev_priv, PUNIT_GPU_STATUS_REG);
4645                 rpn = ((val >> PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT) &
4646                        PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK);
4647         }
4648
4649         return rpn;
4650 }
4651
4652 static int valleyview_rps_guar_freq(struct drm_i915_private *dev_priv)
4653 {
4654         u32 val, rp1;
4655
4656         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
4657
4658         rp1 = (val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK) >> FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
4659
4660         return rp1;
4661 }
4662
4663 static int valleyview_rps_max_freq(struct drm_i915_private *dev_priv)
4664 {
4665         u32 val, rp0;
4666
4667         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FREQ_FUSE);
4668
4669         rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
4670         /* Clamp to max */
4671         rp0 = min_t(u32, rp0, 0xea);
4672
4673         return rp0;
4674 }
4675
4676 static int valleyview_rps_rpe_freq(struct drm_i915_private *dev_priv)
4677 {
4678         u32 val, rpe;
4679
4680         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
4681         rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
4682         val = vlv_nc_read(dev_priv, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
4683         rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
4684
4685         return rpe;
4686 }
4687
4688 static int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
4689 {
4690         return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
4691 }
4692
4693 /* Check that the pctx buffer wasn't move under us. */
4694 static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
4695 {
4696         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
4697
4698         WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
4699                              dev_priv->vlv_pctx->stolen->start);
4700 }
4701
4702
4703 /* Check that the pcbr address is not empty. */
4704 static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
4705 {
4706         unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
4707
4708         WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
4709 }
4710
4711 static void cherryview_setup_pctx(struct drm_device *dev)
4712 {
4713         struct drm_i915_private *dev_priv = dev->dev_private;
4714         unsigned long pctx_paddr, paddr;
4715         struct i915_gtt *gtt = &dev_priv->gtt;
4716         u32 pcbr;
4717         int pctx_size = 32*1024;
4718
4719         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
4720
4721         pcbr = I915_READ(VLV_PCBR);
4722         if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
4723                 DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
4724                 paddr = (dev_priv->mm.stolen_base +
4725                          (gtt->stolen_size - pctx_size));
4726
4727                 pctx_paddr = (paddr & (~4095));
4728                 I915_WRITE(VLV_PCBR, pctx_paddr);
4729         }
4730
4731         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4732 }
4733
4734 static void valleyview_setup_pctx(struct drm_device *dev)
4735 {
4736         struct drm_i915_private *dev_priv = dev->dev_private;
4737         struct drm_i915_gem_object *pctx;
4738         unsigned long pctx_paddr;
4739         u32 pcbr;
4740         int pctx_size = 24*1024;
4741
4742         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
4743
4744         pcbr = I915_READ(VLV_PCBR);
4745         if (pcbr) {
4746                 /* BIOS set it up already, grab the pre-alloc'd space */
4747                 int pcbr_offset;
4748
4749                 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
4750                 pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
4751                                                                       pcbr_offset,
4752                                                                       I915_GTT_OFFSET_NONE,
4753                                                                       pctx_size);
4754                 goto out;
4755         }
4756
4757         DRM_DEBUG_DRIVER("BIOS didn't set up PCBR, fixing up\n");
4758
4759         /*
4760          * From the Gunit register HAS:
4761          * The Gfx driver is expected to program this register and ensure
4762          * proper allocation within Gfx stolen memory.  For example, this
4763          * register should be programmed such than the PCBR range does not
4764          * overlap with other ranges, such as the frame buffer, protected
4765          * memory, or any other relevant ranges.
4766          */
4767         pctx = i915_gem_object_create_stolen(dev, pctx_size);
4768         if (!pctx) {
4769                 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
4770                 return;
4771         }
4772
4773         pctx_paddr = dev_priv->mm.stolen_base + pctx->stolen->start;
4774         I915_WRITE(VLV_PCBR, pctx_paddr);
4775
4776 out:
4777         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
4778         dev_priv->vlv_pctx = pctx;
4779 }
4780
4781 static void valleyview_cleanup_pctx(struct drm_device *dev)
4782 {
4783         struct drm_i915_private *dev_priv = dev->dev_private;
4784
4785         if (WARN_ON(!dev_priv->vlv_pctx))
4786                 return;
4787
4788         drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
4789         dev_priv->vlv_pctx = NULL;
4790 }
4791
4792 static void valleyview_init_gt_powersave(struct drm_device *dev)
4793 {
4794         struct drm_i915_private *dev_priv = dev->dev_private;
4795         u32 val;
4796
4797         valleyview_setup_pctx(dev);
4798
4799         mutex_lock(&dev_priv->rps.hw_lock);
4800
4801         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4802         switch ((val >> 6) & 3) {
4803         case 0:
4804         case 1:
4805                 dev_priv->mem_freq = 800;
4806                 break;
4807         case 2:
4808                 dev_priv->mem_freq = 1066;
4809                 break;
4810         case 3:
4811                 dev_priv->mem_freq = 1333;
4812                 break;
4813         }
4814         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
4815
4816         dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
4817         dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
4818         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
4819                          intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
4820                          dev_priv->rps.max_freq);
4821
4822         dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
4823         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
4824                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
4825                          dev_priv->rps.efficient_freq);
4826
4827         dev_priv->rps.rp1_freq = valleyview_rps_guar_freq(dev_priv);
4828         DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
4829                          intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
4830                          dev_priv->rps.rp1_freq);
4831
4832         dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
4833         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
4834                          intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
4835                          dev_priv->rps.min_freq);
4836
4837         /* Preserve min/max settings in case of re-init */
4838         if (dev_priv->rps.max_freq_softlimit == 0)
4839                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4840
4841         if (dev_priv->rps.min_freq_softlimit == 0)
4842                 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
4843
4844         mutex_unlock(&dev_priv->rps.hw_lock);
4845 }
4846
4847 static void cherryview_init_gt_powersave(struct drm_device *dev)
4848 {
4849         struct drm_i915_private *dev_priv = dev->dev_private;
4850         u32 val;
4851
4852         cherryview_setup_pctx(dev);
4853
4854         mutex_lock(&dev_priv->rps.hw_lock);
4855
4856         mutex_lock(&dev_priv->dpio_lock);
4857         val = vlv_cck_read(dev_priv, CCK_FUSE_REG);
4858         mutex_unlock(&dev_priv->dpio_lock);
4859
4860         switch ((val >> 2) & 0x7) {
4861         case 0:
4862         case 1:
4863                 dev_priv->rps.cz_freq = 200;
4864                 dev_priv->mem_freq = 1600;
4865                 break;
4866         case 2:
4867                 dev_priv->rps.cz_freq = 267;
4868                 dev_priv->mem_freq = 1600;
4869                 break;
4870         case 3:
4871                 dev_priv->rps.cz_freq = 333;
4872                 dev_priv->mem_freq = 2000;
4873                 break;
4874         case 4:
4875                 dev_priv->rps.cz_freq = 320;
4876                 dev_priv->mem_freq = 1600;
4877                 break;
4878         case 5:
4879                 dev_priv->rps.cz_freq = 400;
4880                 dev_priv->mem_freq = 1600;
4881                 break;
4882         }
4883         DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", dev_priv->mem_freq);
4884
4885         dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
4886         dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
4887         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
4888                          intel_gpu_freq(dev_priv, dev_priv->rps.max_freq),
4889                          dev_priv->rps.max_freq);
4890
4891         dev_priv->rps.efficient_freq = cherryview_rps_rpe_freq(dev_priv);
4892         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
4893                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
4894                          dev_priv->rps.efficient_freq);
4895
4896         dev_priv->rps.rp1_freq = cherryview_rps_guar_freq(dev_priv);
4897         DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n",
4898                          intel_gpu_freq(dev_priv, dev_priv->rps.rp1_freq),
4899                          dev_priv->rps.rp1_freq);
4900
4901         dev_priv->rps.min_freq = cherryview_rps_min_freq(dev_priv);
4902         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
4903                          intel_gpu_freq(dev_priv, dev_priv->rps.min_freq),
4904                          dev_priv->rps.min_freq);
4905
4906         WARN_ONCE((dev_priv->rps.max_freq |
4907                    dev_priv->rps.efficient_freq |
4908                    dev_priv->rps.rp1_freq |
4909                    dev_priv->rps.min_freq) & 1,
4910                   "Odd GPU freq values\n");
4911
4912         /* Preserve min/max settings in case of re-init */
4913         if (dev_priv->rps.max_freq_softlimit == 0)
4914                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
4915
4916         if (dev_priv->rps.min_freq_softlimit == 0)
4917                 dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
4918
4919         mutex_unlock(&dev_priv->rps.hw_lock);
4920 }
4921
4922 static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
4923 {
4924         valleyview_cleanup_pctx(dev);
4925 }
4926
4927 static void cherryview_enable_rps(struct drm_device *dev)
4928 {
4929         struct drm_i915_private *dev_priv = dev->dev_private;
4930         struct intel_engine_cs *ring;
4931         u32 gtfifodbg, val, rc6_mode = 0, pcbr;
4932         int i;
4933
4934         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
4935
4936         gtfifodbg = I915_READ(GTFIFODBG);
4937         if (gtfifodbg) {
4938                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
4939                                  gtfifodbg);
4940                 I915_WRITE(GTFIFODBG, gtfifodbg);
4941         }
4942
4943         cherryview_check_pctx(dev_priv);
4944
4945         /* 1a & 1b: Get forcewake during program sequence. Although the driver
4946          * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
4947         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4948
4949         /*  Disable RC states. */
4950         I915_WRITE(GEN6_RC_CONTROL, 0);
4951
4952         /* 2a: Program RC6 thresholds.*/
4953         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
4954         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
4955         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
4956
4957         for_each_ring(ring, dev_priv, i)
4958                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
4959         I915_WRITE(GEN6_RC_SLEEP, 0);
4960
4961         /* TO threshold set to 1750 us ( 0x557 * 1.28 us) */
4962         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
4963
4964         /* allows RC6 residency counter to work */
4965         I915_WRITE(VLV_COUNTER_CONTROL,
4966                    _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
4967                                       VLV_MEDIA_RC6_COUNT_EN |
4968                                       VLV_RENDER_RC6_COUNT_EN));
4969
4970         /* For now we assume BIOS is allocating and populating the PCBR  */
4971         pcbr = I915_READ(VLV_PCBR);
4972
4973         /* 3: Enable RC6 */
4974         if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
4975                                                 (pcbr >> VLV_PCBR_ADDR_SHIFT))
4976                 rc6_mode = GEN7_RC_CTL_TO_MODE;
4977
4978         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
4979
4980         /* 4 Program defaults and thresholds for RPS*/
4981         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
4982         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
4983         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
4984         I915_WRITE(GEN6_RP_UP_EI, 66000);
4985         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
4986
4987         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
4988
4989         /* 5: Enable RPS */
4990         I915_WRITE(GEN6_RP_CONTROL,
4991                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
4992                    GEN6_RP_MEDIA_IS_GFX |
4993                    GEN6_RP_ENABLE |
4994                    GEN6_RP_UP_BUSY_AVG |
4995                    GEN6_RP_DOWN_IDLE_AVG);
4996
4997         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
4998
4999         /* RPS code assumes GPLL is used */
5000         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
5001
5002         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
5003         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
5004
5005         dev_priv->rps.cur_freq = (val >> 8) & 0xff;
5006         DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
5007                          intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5008                          dev_priv->rps.cur_freq);
5009
5010         DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
5011                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5012                          dev_priv->rps.efficient_freq);
5013
5014         valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
5015
5016         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5017 }
5018
5019 static void valleyview_enable_rps(struct drm_device *dev)
5020 {
5021         struct drm_i915_private *dev_priv = dev->dev_private;
5022         struct intel_engine_cs *ring;
5023         u32 gtfifodbg, val, rc6_mode = 0;
5024         int i;
5025
5026         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
5027
5028         valleyview_check_pctx(dev_priv);
5029
5030         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
5031                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
5032                                  gtfifodbg);
5033                 I915_WRITE(GTFIFODBG, gtfifodbg);
5034         }
5035
5036         /* If VLV, Forcewake all wells, else re-direct to regular path */
5037         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5038
5039         /*  Disable RC states. */
5040         I915_WRITE(GEN6_RC_CONTROL, 0);
5041
5042         I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
5043         I915_WRITE(GEN6_RP_UP_THRESHOLD, 59400);
5044         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 245000);
5045         I915_WRITE(GEN6_RP_UP_EI, 66000);
5046         I915_WRITE(GEN6_RP_DOWN_EI, 350000);
5047
5048         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
5049
5050         I915_WRITE(GEN6_RP_CONTROL,
5051                    GEN6_RP_MEDIA_TURBO |
5052                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
5053                    GEN6_RP_MEDIA_IS_GFX |
5054                    GEN6_RP_ENABLE |
5055                    GEN6_RP_UP_BUSY_AVG |
5056                    GEN6_RP_DOWN_IDLE_CONT);
5057
5058         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
5059         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
5060         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
5061
5062         for_each_ring(ring, dev_priv, i)
5063                 I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
5064
5065         I915_WRITE(GEN6_RC6_THRESHOLD, 0x557);
5066
5067         /* allows RC6 residency counter to work */
5068         I915_WRITE(VLV_COUNTER_CONTROL,
5069                    _MASKED_BIT_ENABLE(VLV_MEDIA_RC0_COUNT_EN |
5070                                       VLV_RENDER_RC0_COUNT_EN |
5071                                       VLV_MEDIA_RC6_COUNT_EN |
5072                                       VLV_RENDER_RC6_COUNT_EN));
5073
5074         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
5075                 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
5076
5077         intel_print_rc6_info(dev, rc6_mode);
5078
5079         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
5080
5081         val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
5082
5083         /* RPS code assumes GPLL is used */
5084         WARN_ONCE((val & GPLLENABLE) == 0, "GPLL not enabled\n");
5085
5086         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & GPLLENABLE ? "yes" : "no");
5087         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
5088
5089         dev_priv->rps.cur_freq = (val >> 8) & 0xff;
5090         DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
5091                          intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
5092                          dev_priv->rps.cur_freq);
5093
5094         DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
5095                          intel_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
5096                          dev_priv->rps.efficient_freq);
5097
5098         valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
5099
5100         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5101 }
5102
5103 static unsigned long intel_pxfreq(u32 vidfreq)
5104 {
5105         unsigned long freq;
5106         int div = (vidfreq & 0x3f0000) >> 16;
5107         int post = (vidfreq & 0x3000) >> 12;
5108         int pre = (vidfreq & 0x7);
5109
5110         if (!pre)
5111                 return 0;
5112
5113         freq = ((div * 133333) / ((1<<post) * pre));
5114
5115         return freq;
5116 }
5117
5118 static const struct cparams {
5119         u16 i;
5120         u16 t;
5121         u16 m;
5122         u16 c;
5123 } cparams[] = {
5124         { 1, 1333, 301, 28664 },
5125         { 1, 1066, 294, 24460 },
5126         { 1, 800, 294, 25192 },
5127         { 0, 1333, 276, 27605 },
5128         { 0, 1066, 276, 27605 },
5129         { 0, 800, 231, 23784 },
5130 };
5131
5132 static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
5133 {
5134         u64 total_count, diff, ret;
5135         u32 count1, count2, count3, m = 0, c = 0;
5136         unsigned long now = jiffies_to_msecs(jiffies), diff1;
5137         int i;
5138
5139         assert_spin_locked(&mchdev_lock);
5140
5141         diff1 = now - dev_priv->ips.last_time1;
5142
5143         /* Prevent division-by-zero if we are asking too fast.
5144          * Also, we don't get interesting results if we are polling
5145          * faster than once in 10ms, so just return the saved value
5146          * in such cases.
5147          */
5148         if (diff1 <= 10)
5149                 return dev_priv->ips.chipset_power;
5150
5151         count1 = I915_READ(DMIEC);
5152         count2 = I915_READ(DDREC);
5153         count3 = I915_READ(CSIEC);
5154
5155         total_count = count1 + count2 + count3;
5156
5157         /* FIXME: handle per-counter overflow */
5158         if (total_count < dev_priv->ips.last_count1) {
5159                 diff = ~0UL - dev_priv->ips.last_count1;
5160                 diff += total_count;
5161         } else {
5162                 diff = total_count - dev_priv->ips.last_count1;
5163         }
5164
5165         for (i = 0; i < ARRAY_SIZE(cparams); i++) {
5166                 if (cparams[i].i == dev_priv->ips.c_m &&
5167                     cparams[i].t == dev_priv->ips.r_t) {
5168                         m = cparams[i].m;
5169                         c = cparams[i].c;
5170                         break;
5171                 }
5172         }
5173
5174         diff = div_u64(diff, diff1);
5175         ret = ((m * diff) + c);
5176         ret = div_u64(ret, 10);
5177
5178         dev_priv->ips.last_count1 = total_count;
5179         dev_priv->ips.last_time1 = now;
5180
5181         dev_priv->ips.chipset_power = ret;
5182
5183         return ret;
5184 }
5185
5186 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
5187 {
5188         struct drm_device *dev = dev_priv->dev;
5189         unsigned long val;
5190
5191         if (INTEL_INFO(dev)->gen != 5)
5192                 return 0;
5193
5194         spin_lock_irq(&mchdev_lock);
5195
5196         val = __i915_chipset_val(dev_priv);
5197
5198         spin_unlock_irq(&mchdev_lock);
5199
5200         return val;
5201 }
5202
5203 unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
5204 {
5205         unsigned long m, x, b;
5206         u32 tsfs;
5207
5208         tsfs = I915_READ(TSFS);
5209
5210         m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
5211         x = I915_READ8(TR1);
5212
5213         b = tsfs & TSFS_INTR_MASK;
5214
5215         return ((m * x) / 127) - b;
5216 }
5217
5218 static int _pxvid_to_vd(u8 pxvid)
5219 {
5220         if (pxvid == 0)
5221                 return 0;
5222
5223         if (pxvid >= 8 && pxvid < 31)
5224                 pxvid = 31;
5225
5226         return (pxvid + 2) * 125;
5227 }
5228
5229 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
5230 {
5231         struct drm_device *dev = dev_priv->dev;
5232         const int vd = _pxvid_to_vd(pxvid);
5233         const int vm = vd - 1125;
5234
5235         if (INTEL_INFO(dev)->is_mobile)
5236                 return vm > 0 ? vm : 0;
5237
5238         return vd;
5239 }
5240
5241 static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
5242 {
5243         u64 now, diff, diffms;
5244         u32 count;
5245
5246         assert_spin_locked(&mchdev_lock);
5247
5248         now = ktime_get_raw_ns();
5249         diffms = now - dev_priv->ips.last_time2;
5250         do_div(diffms, NSEC_PER_MSEC);
5251
5252         /* Don't divide by 0 */
5253         if (!diffms)
5254                 return;
5255
5256         count = I915_READ(GFXEC);
5257
5258         if (count < dev_priv->ips.last_count2) {
5259                 diff = ~0UL - dev_priv->ips.last_count2;
5260                 diff += count;
5261         } else {
5262                 diff = count - dev_priv->ips.last_count2;
5263         }
5264
5265         dev_priv->ips.last_count2 = count;
5266         dev_priv->ips.last_time2 = now;
5267
5268         /* More magic constants... */
5269         diff = diff * 1181;
5270         diff = div_u64(diff, diffms * 10);
5271         dev_priv->ips.gfx_power = diff;
5272 }
5273
5274 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
5275 {
5276         struct drm_device *dev = dev_priv->dev;
5277
5278         if (INTEL_INFO(dev)->gen != 5)
5279                 return;
5280
5281         spin_lock_irq(&mchdev_lock);
5282
5283         __i915_update_gfx_val(dev_priv);
5284
5285         spin_unlock_irq(&mchdev_lock);
5286 }
5287
5288 static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
5289 {
5290         unsigned long t, corr, state1, corr2, state2;
5291         u32 pxvid, ext_v;
5292
5293         assert_spin_locked(&mchdev_lock);
5294
5295         pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4));
5296         pxvid = (pxvid >> 24) & 0x7f;
5297         ext_v = pvid_to_extvid(dev_priv, pxvid);
5298
5299         state1 = ext_v;
5300
5301         t = i915_mch_val(dev_priv);
5302
5303         /* Revel in the empirically derived constants */
5304
5305         /* Correction factor in 1/100000 units */
5306         if (t > 80)
5307                 corr = ((t * 2349) + 135940);
5308         else if (t >= 50)
5309                 corr = ((t * 964) + 29317);
5310         else /* < 50 */
5311                 corr = ((t * 301) + 1004);
5312
5313         corr = corr * ((150142 * state1) / 10000 - 78642);
5314         corr /= 100000;
5315         corr2 = (corr * dev_priv->ips.corr);
5316
5317         state2 = (corr2 * state1) / 10000;
5318         state2 /= 100; /* convert to mW */
5319
5320         __i915_update_gfx_val(dev_priv);
5321
5322         return dev_priv->ips.gfx_power + state2;
5323 }
5324
5325 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
5326 {
5327         struct drm_device *dev = dev_priv->dev;
5328         unsigned long val;
5329
5330         if (INTEL_INFO(dev)->gen != 5)
5331                 return 0;
5332
5333         spin_lock_irq(&mchdev_lock);
5334
5335         val = __i915_gfx_val(dev_priv);
5336
5337         spin_unlock_irq(&mchdev_lock);
5338
5339         return val;
5340 }
5341
5342 /**
5343  * i915_read_mch_val - return value for IPS use
5344  *
5345  * Calculate and return a value for the IPS driver to use when deciding whether
5346  * we have thermal and power headroom to increase CPU or GPU power budget.
5347  */
5348 unsigned long i915_read_mch_val(void)
5349 {
5350         struct drm_i915_private *dev_priv;
5351         unsigned long chipset_val, graphics_val, ret = 0;
5352
5353         spin_lock_irq(&mchdev_lock);
5354         if (!i915_mch_dev)
5355                 goto out_unlock;
5356         dev_priv = i915_mch_dev;
5357
5358         chipset_val = __i915_chipset_val(dev_priv);
5359         graphics_val = __i915_gfx_val(dev_priv);
5360
5361         ret = chipset_val + graphics_val;
5362
5363 out_unlock:
5364         spin_unlock_irq(&mchdev_lock);
5365
5366         return ret;
5367 }
5368 EXPORT_SYMBOL_GPL(i915_read_mch_val);
5369
5370 /**
5371  * i915_gpu_raise - raise GPU frequency limit
5372  *
5373  * Raise the limit; IPS indicates we have thermal headroom.
5374  */
5375 bool i915_gpu_raise(void)
5376 {
5377         struct drm_i915_private *dev_priv;
5378         bool ret = true;
5379
5380         spin_lock_irq(&mchdev_lock);
5381         if (!i915_mch_dev) {
5382                 ret = false;
5383                 goto out_unlock;
5384         }
5385         dev_priv = i915_mch_dev;
5386
5387         if (dev_priv->ips.max_delay > dev_priv->ips.fmax)
5388                 dev_priv->ips.max_delay--;
5389
5390 out_unlock:
5391         spin_unlock_irq(&mchdev_lock);
5392
5393         return ret;
5394 }
5395 EXPORT_SYMBOL_GPL(i915_gpu_raise);
5396
5397 /**
5398  * i915_gpu_lower - lower GPU frequency limit
5399  *
5400  * IPS indicates we're close to a thermal limit, so throttle back the GPU
5401  * frequency maximum.
5402  */
5403 bool i915_gpu_lower(void)
5404 {
5405         struct drm_i915_private *dev_priv;
5406         bool ret = true;
5407
5408         spin_lock_irq(&mchdev_lock);
5409         if (!i915_mch_dev) {
5410                 ret = false;
5411                 goto out_unlock;
5412         }
5413         dev_priv = i915_mch_dev;
5414
5415         if (dev_priv->ips.max_delay < dev_priv->ips.min_delay)
5416                 dev_priv->ips.max_delay++;
5417
5418 out_unlock:
5419         spin_unlock_irq(&mchdev_lock);
5420
5421         return ret;
5422 }
5423 EXPORT_SYMBOL_GPL(i915_gpu_lower);
5424
5425 /**
5426  * i915_gpu_busy - indicate GPU business to IPS
5427  *
5428  * Tell the IPS driver whether or not the GPU is busy.
5429  */
5430 bool i915_gpu_busy(void)
5431 {
5432         struct drm_i915_private *dev_priv;
5433         struct intel_engine_cs *ring;
5434         bool ret = false;
5435         int i;
5436
5437         spin_lock_irq(&mchdev_lock);
5438         if (!i915_mch_dev)
5439                 goto out_unlock;
5440         dev_priv = i915_mch_dev;
5441
5442         for_each_ring(ring, dev_priv, i)
5443                 ret |= !list_empty(&ring->request_list);
5444
5445 out_unlock:
5446         spin_unlock_irq(&mchdev_lock);
5447
5448         return ret;
5449 }
5450 EXPORT_SYMBOL_GPL(i915_gpu_busy);
5451
5452 /**
5453  * i915_gpu_turbo_disable - disable graphics turbo
5454  *
5455  * Disable graphics turbo by resetting the max frequency and setting the
5456  * current frequency to the default.
5457  */
5458 bool i915_gpu_turbo_disable(void)
5459 {
5460         struct drm_i915_private *dev_priv;
5461         bool ret = true;
5462
5463         spin_lock_irq(&mchdev_lock);
5464         if (!i915_mch_dev) {
5465                 ret = false;
5466                 goto out_unlock;
5467         }
5468         dev_priv = i915_mch_dev;
5469
5470         dev_priv->ips.max_delay = dev_priv->ips.fstart;
5471
5472         if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
5473                 ret = false;
5474
5475 out_unlock:
5476         spin_unlock_irq(&mchdev_lock);
5477
5478         return ret;
5479 }
5480 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
5481
5482 /**
5483  * Tells the intel_ips driver that the i915 driver is now loaded, if
5484  * IPS got loaded first.
5485  *
5486  * This awkward dance is so that neither module has to depend on the
5487  * other in order for IPS to do the appropriate communication of
5488  * GPU turbo limits to i915.
5489  */
5490 static void
5491 ips_ping_for_i915_load(void)
5492 {
5493         void (*link)(void);
5494
5495         link = symbol_get(ips_link_to_i915_driver);
5496         if (link) {
5497                 link();
5498                 symbol_put(ips_link_to_i915_driver);
5499         }
5500 }
5501
5502 void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
5503 {
5504         /* We only register the i915 ips part with intel-ips once everything is
5505          * set up, to avoid intel-ips sneaking in and reading bogus values. */
5506         spin_lock_irq(&mchdev_lock);
5507         i915_mch_dev = dev_priv;
5508         spin_unlock_irq(&mchdev_lock);
5509
5510         ips_ping_for_i915_load();
5511 }
5512
5513 void intel_gpu_ips_teardown(void)
5514 {
5515         spin_lock_irq(&mchdev_lock);
5516         i915_mch_dev = NULL;
5517         spin_unlock_irq(&mchdev_lock);
5518 }
5519
5520 static void intel_init_emon(struct drm_device *dev)
5521 {
5522         struct drm_i915_private *dev_priv = dev->dev_private;
5523         u32 lcfuse;
5524         u8 pxw[16];
5525         int i;
5526
5527         /* Disable to program */
5528         I915_WRITE(ECR, 0);
5529         POSTING_READ(ECR);
5530
5531         /* Program energy weights for various events */
5532         I915_WRITE(SDEW, 0x15040d00);
5533         I915_WRITE(CSIEW0, 0x007f0000);
5534         I915_WRITE(CSIEW1, 0x1e220004);
5535         I915_WRITE(CSIEW2, 0x04000004);
5536
5537         for (i = 0; i < 5; i++)
5538                 I915_WRITE(PEW + (i * 4), 0);
5539         for (i = 0; i < 3; i++)
5540                 I915_WRITE(DEW + (i * 4), 0);
5541
5542         /* Program P-state weights to account for frequency power adjustment */
5543         for (i = 0; i < 16; i++) {
5544                 u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
5545                 unsigned long freq = intel_pxfreq(pxvidfreq);
5546                 unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
5547                         PXVFREQ_PX_SHIFT;
5548                 unsigned long val;
5549
5550                 val = vid * vid;
5551                 val *= (freq / 1000);
5552                 val *= 255;
5553                 val /= (127*127*900);
5554                 if (val > 0xff)
5555                         DRM_ERROR("bad pxval: %ld\n", val);
5556                 pxw[i] = val;
5557         }
5558         /* Render standby states get 0 weight */
5559         pxw[14] = 0;
5560         pxw[15] = 0;
5561
5562         for (i = 0; i < 4; i++) {
5563                 u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
5564                         (pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
5565                 I915_WRITE(PXW + (i * 4), val);
5566         }
5567
5568         /* Adjust magic regs to magic values (more experimental results) */
5569         I915_WRITE(OGW0, 0);
5570         I915_WRITE(OGW1, 0);
5571         I915_WRITE(EG0, 0x00007f00);
5572         I915_WRITE(EG1, 0x0000000e);
5573         I915_WRITE(EG2, 0x000e0000);
5574         I915_WRITE(EG3, 0x68000300);
5575         I915_WRITE(EG4, 0x42000000);
5576         I915_WRITE(EG5, 0x00140031);
5577         I915_WRITE(EG6, 0);
5578         I915_WRITE(EG7, 0);
5579
5580         for (i = 0; i < 8; i++)
5581                 I915_WRITE(PXWL + (i * 4), 0);
5582
5583         /* Enable PMON + select events */
5584         I915_WRITE(ECR, 0x80000019);
5585
5586         lcfuse = I915_READ(LCFUSE02);
5587
5588         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
5589 }
5590
5591 void intel_init_gt_powersave(struct drm_device *dev)
5592 {
5593         i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
5594
5595         if (IS_CHERRYVIEW(dev))
5596                 cherryview_init_gt_powersave(dev);
5597         else if (IS_VALLEYVIEW(dev))
5598                 valleyview_init_gt_powersave(dev);
5599 }
5600
5601 void intel_cleanup_gt_powersave(struct drm_device *dev)
5602 {
5603         if (IS_CHERRYVIEW(dev))
5604                 return;
5605         else if (IS_VALLEYVIEW(dev))
5606                 valleyview_cleanup_gt_powersave(dev);
5607 }
5608
5609 static void gen6_suspend_rps(struct drm_device *dev)
5610 {
5611         struct drm_i915_private *dev_priv = dev->dev_private;
5612
5613         flush_delayed_work(&dev_priv->rps.delayed_resume_work);
5614
5615         /*
5616          * TODO: disable RPS interrupts on GEN9+ too once RPS support
5617          * is added for it.
5618          */
5619         if (INTEL_INFO(dev)->gen < 9)
5620                 gen6_disable_rps_interrupts(dev);
5621 }
5622
5623 /**
5624  * intel_suspend_gt_powersave - suspend PM work and helper threads
5625  * @dev: drm device
5626  *
5627  * We don't want to disable RC6 or other features here, we just want
5628  * to make sure any work we've queued has finished and won't bother
5629  * us while we're suspended.
5630  */
5631 void intel_suspend_gt_powersave(struct drm_device *dev)
5632 {
5633         struct drm_i915_private *dev_priv = dev->dev_private;
5634
5635         if (INTEL_INFO(dev)->gen < 6)
5636                 return;
5637
5638         gen6_suspend_rps(dev);
5639
5640         /* Force GPU to min freq during suspend */
5641         gen6_rps_idle(dev_priv);
5642 }
5643
5644 void intel_disable_gt_powersave(struct drm_device *dev)
5645 {
5646         struct drm_i915_private *dev_priv = dev->dev_private;
5647
5648         if (IS_IRONLAKE_M(dev)) {
5649                 ironlake_disable_drps(dev);
5650         } else if (INTEL_INFO(dev)->gen >= 6) {
5651                 intel_suspend_gt_powersave(dev);
5652
5653                 mutex_lock(&dev_priv->rps.hw_lock);
5654                 if (INTEL_INFO(dev)->gen >= 9)
5655                         gen9_disable_rps(dev);
5656                 else if (IS_CHERRYVIEW(dev))
5657                         cherryview_disable_rps(dev);
5658                 else if (IS_VALLEYVIEW(dev))
5659                         valleyview_disable_rps(dev);
5660                 else
5661                         gen6_disable_rps(dev);
5662
5663                 dev_priv->rps.enabled = false;
5664                 mutex_unlock(&dev_priv->rps.hw_lock);
5665         }
5666 }
5667
5668 static void intel_gen6_powersave_work(struct work_struct *work)
5669 {
5670         struct drm_i915_private *dev_priv =
5671                 container_of(work, struct drm_i915_private,
5672                              rps.delayed_resume_work.work);
5673         struct drm_device *dev = dev_priv->dev;
5674
5675         mutex_lock(&dev_priv->rps.hw_lock);
5676
5677         /*
5678          * TODO: reset/enable RPS interrupts on GEN9+ too, once RPS support is
5679          * added for it.
5680          */
5681         if (INTEL_INFO(dev)->gen < 9)
5682                 gen6_reset_rps_interrupts(dev);
5683
5684         if (IS_CHERRYVIEW(dev)) {
5685                 cherryview_enable_rps(dev);
5686         } else if (IS_VALLEYVIEW(dev)) {
5687                 valleyview_enable_rps(dev);
5688         } else if (INTEL_INFO(dev)->gen >= 9) {
5689                 gen9_enable_rc6(dev);
5690                 gen9_enable_rps(dev);
5691                 __gen6_update_ring_freq(dev);
5692         } else if (IS_BROADWELL(dev)) {
5693                 gen8_enable_rps(dev);
5694                 __gen6_update_ring_freq(dev);
5695         } else {
5696                 gen6_enable_rps(dev);
5697                 __gen6_update_ring_freq(dev);
5698         }
5699         dev_priv->rps.enabled = true;
5700
5701         if (INTEL_INFO(dev)->gen < 9)
5702                 gen6_enable_rps_interrupts(dev);
5703
5704         mutex_unlock(&dev_priv->rps.hw_lock);
5705
5706         intel_runtime_pm_put(dev_priv);
5707 }
5708
5709 void intel_enable_gt_powersave(struct drm_device *dev)
5710 {
5711         struct drm_i915_private *dev_priv = dev->dev_private;
5712
5713         /* Powersaving is controlled by the host when inside a VM */
5714         if (intel_vgpu_active(dev))
5715                 return;
5716
5717         if (IS_IRONLAKE_M(dev)) {
5718                 mutex_lock(&dev->struct_mutex);
5719                 ironlake_enable_drps(dev);
5720                 intel_init_emon(dev);
5721                 mutex_unlock(&dev->struct_mutex);
5722         } else if (INTEL_INFO(dev)->gen >= 6) {
5723                 /*
5724                  * PCU communication is slow and this doesn't need to be
5725                  * done at any specific time, so do this out of our fast path
5726                  * to make resume and init faster.
5727                  *
5728                  * We depend on the HW RC6 power context save/restore
5729                  * mechanism when entering D3 through runtime PM suspend. So
5730                  * disable RPM until RPS/RC6 is properly setup. We can only
5731                  * get here via the driver load/system resume/runtime resume
5732                  * paths, so the _noresume version is enough (and in case of
5733                  * runtime resume it's necessary).
5734                  */
5735                 if (schedule_delayed_work(&dev_priv->rps.delayed_resume_work,
5736                                            round_jiffies_up_relative(HZ)))
5737                         intel_runtime_pm_get_noresume(dev_priv);
5738         }
5739 }
5740
5741 void intel_reset_gt_powersave(struct drm_device *dev)
5742 {
5743         struct drm_i915_private *dev_priv = dev->dev_private;
5744
5745         if (INTEL_INFO(dev)->gen < 6)
5746                 return;
5747
5748         gen6_suspend_rps(dev);
5749         dev_priv->rps.enabled = false;
5750 }
5751
5752 static void ibx_init_clock_gating(struct drm_device *dev)
5753 {
5754         struct drm_i915_private *dev_priv = dev->dev_private;
5755
5756         /*
5757          * On Ibex Peak and Cougar Point, we need to disable clock
5758          * gating for the panel power sequencer or it will fail to
5759          * start up when no ports are active.
5760          */
5761         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
5762 }
5763
5764 static void g4x_disable_trickle_feed(struct drm_device *dev)
5765 {
5766         struct drm_i915_private *dev_priv = dev->dev_private;
5767         int pipe;
5768
5769         for_each_pipe(dev_priv, pipe) {
5770                 I915_WRITE(DSPCNTR(pipe),
5771                            I915_READ(DSPCNTR(pipe)) |
5772                            DISPPLANE_TRICKLE_FEED_DISABLE);
5773                 intel_flush_primary_plane(dev_priv, pipe);
5774         }
5775 }
5776
5777 static void ilk_init_lp_watermarks(struct drm_device *dev)
5778 {
5779         struct drm_i915_private *dev_priv = dev->dev_private;
5780
5781         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
5782         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
5783         I915_WRITE(WM1_LP_ILK, I915_READ(WM1_LP_ILK) & ~WM1_LP_SR_EN);
5784
5785         /*
5786          * Don't touch WM1S_LP_EN here.
5787          * Doing so could cause underruns.
5788          */
5789 }
5790
5791 static void ironlake_init_clock_gating(struct drm_device *dev)
5792 {
5793         struct drm_i915_private *dev_priv = dev->dev_private;
5794         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
5795
5796         /*
5797          * Required for FBC
5798          * WaFbcDisableDpfcClockGating:ilk
5799          */
5800         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
5801                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
5802                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
5803
5804         I915_WRITE(PCH_3DCGDIS0,
5805                    MARIUNIT_CLOCK_GATE_DISABLE |
5806                    SVSMUNIT_CLOCK_GATE_DISABLE);
5807         I915_WRITE(PCH_3DCGDIS1,
5808                    VFMUNIT_CLOCK_GATE_DISABLE);
5809
5810         /*
5811          * According to the spec the following bits should be set in
5812          * order to enable memory self-refresh
5813          * The bit 22/21 of 0x42004
5814          * The bit 5 of 0x42020
5815          * The bit 15 of 0x45000
5816          */
5817         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5818                    (I915_READ(ILK_DISPLAY_CHICKEN2) |
5819                     ILK_DPARB_GATE | ILK_VSDPFD_FULL));
5820         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
5821         I915_WRITE(DISP_ARB_CTL,
5822                    (I915_READ(DISP_ARB_CTL) |
5823                     DISP_FBC_WM_DIS));
5824
5825         ilk_init_lp_watermarks(dev);
5826
5827         /*
5828          * Based on the document from hardware guys the following bits
5829          * should be set unconditionally in order to enable FBC.
5830          * The bit 22 of 0x42000
5831          * The bit 22 of 0x42004
5832          * The bit 7,8,9 of 0x42020.
5833          */
5834         if (IS_IRONLAKE_M(dev)) {
5835                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
5836                 I915_WRITE(ILK_DISPLAY_CHICKEN1,
5837                            I915_READ(ILK_DISPLAY_CHICKEN1) |
5838                            ILK_FBCQ_DIS);
5839                 I915_WRITE(ILK_DISPLAY_CHICKEN2,
5840                            I915_READ(ILK_DISPLAY_CHICKEN2) |
5841                            ILK_DPARB_GATE);
5842         }
5843
5844         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
5845
5846         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5847                    I915_READ(ILK_DISPLAY_CHICKEN2) |
5848                    ILK_ELPIN_409_SELECT);
5849         I915_WRITE(_3D_CHICKEN2,
5850                    _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
5851                    _3D_CHICKEN2_WM_READ_PIPELINED);
5852
5853         /* WaDisableRenderCachePipelinedFlush:ilk */
5854         I915_WRITE(CACHE_MODE_0,
5855                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
5856
5857         /* WaDisable_RenderCache_OperationalFlush:ilk */
5858         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5859
5860         g4x_disable_trickle_feed(dev);
5861
5862         ibx_init_clock_gating(dev);
5863 }
5864
5865 static void cpt_init_clock_gating(struct drm_device *dev)
5866 {
5867         struct drm_i915_private *dev_priv = dev->dev_private;
5868         int pipe;
5869         uint32_t val;
5870
5871         /*
5872          * On Ibex Peak and Cougar Point, we need to disable clock
5873          * gating for the panel power sequencer or it will fail to
5874          * start up when no ports are active.
5875          */
5876         I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
5877                    PCH_DPLUNIT_CLOCK_GATE_DISABLE |
5878                    PCH_CPUNIT_CLOCK_GATE_DISABLE);
5879         I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
5880                    DPLS_EDP_PPS_FIX_DIS);
5881         /* The below fixes the weird display corruption, a few pixels shifted
5882          * downward, on (only) LVDS of some HP laptops with IVY.
5883          */
5884         for_each_pipe(dev_priv, pipe) {
5885                 val = I915_READ(TRANS_CHICKEN2(pipe));
5886                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
5887                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
5888                 if (dev_priv->vbt.fdi_rx_polarity_inverted)
5889                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
5890                 val &= ~TRANS_CHICKEN2_FRAME_START_DELAY_MASK;
5891                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
5892                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
5893                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
5894         }
5895         /* WADP0ClockGatingDisable */
5896         for_each_pipe(dev_priv, pipe) {
5897                 I915_WRITE(TRANS_CHICKEN1(pipe),
5898                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
5899         }
5900 }
5901
5902 static void gen6_check_mch_setup(struct drm_device *dev)
5903 {
5904         struct drm_i915_private *dev_priv = dev->dev_private;
5905         uint32_t tmp;
5906
5907         tmp = I915_READ(MCH_SSKPD);
5908         if ((tmp & MCH_SSKPD_WM0_MASK) != MCH_SSKPD_WM0_VAL)
5909                 DRM_DEBUG_KMS("Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
5910                               tmp);
5911 }
5912
5913 static void gen6_init_clock_gating(struct drm_device *dev)
5914 {
5915         struct drm_i915_private *dev_priv = dev->dev_private;
5916         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
5917
5918         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
5919
5920         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5921                    I915_READ(ILK_DISPLAY_CHICKEN2) |
5922                    ILK_ELPIN_409_SELECT);
5923
5924         /* WaDisableHiZPlanesWhenMSAAEnabled:snb */
5925         I915_WRITE(_3D_CHICKEN,
5926                    _MASKED_BIT_ENABLE(_3D_CHICKEN_HIZ_PLANE_DISABLE_MSAA_4X_SNB));
5927
5928         /* WaDisable_RenderCache_OperationalFlush:snb */
5929         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
5930
5931         /*
5932          * BSpec recoomends 8x4 when MSAA is used,
5933          * however in practice 16x4 seems fastest.
5934          *
5935          * Note that PS/WM thread counts depend on the WIZ hashing
5936          * disable bit, which we don't touch here, but it's good
5937          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
5938          */
5939         I915_WRITE(GEN6_GT_MODE,
5940                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
5941
5942         ilk_init_lp_watermarks(dev);
5943
5944         I915_WRITE(CACHE_MODE_0,
5945                    _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
5946
5947         I915_WRITE(GEN6_UCGCTL1,
5948                    I915_READ(GEN6_UCGCTL1) |
5949                    GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
5950                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
5951
5952         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
5953          * gating disable must be set.  Failure to set it results in
5954          * flickering pixels due to Z write ordering failures after
5955          * some amount of runtime in the Mesa "fire" demo, and Unigine
5956          * Sanctuary and Tropics, and apparently anything else with
5957          * alpha test or pixel discard.
5958          *
5959          * According to the spec, bit 11 (RCCUNIT) must also be set,
5960          * but we didn't debug actual testcases to find it out.
5961          *
5962          * WaDisableRCCUnitClockGating:snb
5963          * WaDisableRCPBUnitClockGating:snb
5964          */
5965         I915_WRITE(GEN6_UCGCTL2,
5966                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
5967                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
5968
5969         /* WaStripsFansDisableFastClipPerformanceFix:snb */
5970         I915_WRITE(_3D_CHICKEN3,
5971                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
5972
5973         /*
5974          * Bspec says:
5975          * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
5976          * 3DSTATE_SF number of SF output attributes is more than 16."
5977          */
5978         I915_WRITE(_3D_CHICKEN3,
5979                    _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
5980
5981         /*
5982          * According to the spec the following bits should be
5983          * set in order to enable memory self-refresh and fbc:
5984          * The bit21 and bit22 of 0x42000
5985          * The bit21 and bit22 of 0x42004
5986          * The bit5 and bit7 of 0x42020
5987          * The bit14 of 0x70180
5988          * The bit14 of 0x71180
5989          *
5990          * WaFbcAsynchFlipDisableFbcQueue:snb
5991          */
5992         I915_WRITE(ILK_DISPLAY_CHICKEN1,
5993                    I915_READ(ILK_DISPLAY_CHICKEN1) |
5994                    ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
5995         I915_WRITE(ILK_DISPLAY_CHICKEN2,
5996                    I915_READ(ILK_DISPLAY_CHICKEN2) |
5997                    ILK_DPARB_GATE | ILK_VSDPFD_FULL);
5998         I915_WRITE(ILK_DSPCLK_GATE_D,
5999                    I915_READ(ILK_DSPCLK_GATE_D) |
6000                    ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
6001                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
6002
6003         g4x_disable_trickle_feed(dev);
6004
6005         cpt_init_clock_gating(dev);
6006
6007         gen6_check_mch_setup(dev);
6008 }
6009
6010 static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
6011 {
6012         uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
6013
6014         /*
6015          * WaVSThreadDispatchOverride:ivb,vlv
6016          *
6017          * This actually overrides the dispatch
6018          * mode for all thread types.
6019          */
6020         reg &= ~GEN7_FF_SCHED_MASK;
6021         reg |= GEN7_FF_TS_SCHED_HW;
6022         reg |= GEN7_FF_VS_SCHED_HW;
6023         reg |= GEN7_FF_DS_SCHED_HW;
6024
6025         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
6026 }
6027
6028 static void lpt_init_clock_gating(struct drm_device *dev)
6029 {
6030         struct drm_i915_private *dev_priv = dev->dev_private;
6031
6032         /*
6033          * TODO: this bit should only be enabled when really needed, then
6034          * disabled when not needed anymore in order to save power.
6035          */
6036         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE)
6037                 I915_WRITE(SOUTH_DSPCLK_GATE_D,
6038                            I915_READ(SOUTH_DSPCLK_GATE_D) |
6039                            PCH_LP_PARTITION_LEVEL_DISABLE);
6040
6041         /* WADPOClockGatingDisable:hsw */
6042         I915_WRITE(_TRANSA_CHICKEN1,
6043                    I915_READ(_TRANSA_CHICKEN1) |
6044                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
6045 }
6046
6047 static void lpt_suspend_hw(struct drm_device *dev)
6048 {
6049         struct drm_i915_private *dev_priv = dev->dev_private;
6050
6051         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
6052                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
6053
6054                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
6055                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
6056         }
6057 }
6058
6059 static void broadwell_init_clock_gating(struct drm_device *dev)
6060 {
6061         struct drm_i915_private *dev_priv = dev->dev_private;
6062         enum pipe pipe;
6063
6064         I915_WRITE(WM3_LP_ILK, 0);
6065         I915_WRITE(WM2_LP_ILK, 0);
6066         I915_WRITE(WM1_LP_ILK, 0);
6067
6068         /* WaSwitchSolVfFArbitrationPriority:bdw */
6069         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6070
6071         /* WaPsrDPAMaskVBlankInSRD:bdw */
6072         I915_WRITE(CHICKEN_PAR1_1,
6073                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
6074
6075         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
6076         for_each_pipe(dev_priv, pipe) {
6077                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
6078                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
6079                            BDW_DPRS_MASK_VBLANK_SRD);
6080         }
6081
6082         /* WaVSRefCountFullforceMissDisable:bdw */
6083         /* WaDSRefCountFullforceMissDisable:bdw */
6084         I915_WRITE(GEN7_FF_THREAD_MODE,
6085                    I915_READ(GEN7_FF_THREAD_MODE) &
6086                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6087
6088         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6089                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6090
6091         /* WaDisableSDEUnitClockGating:bdw */
6092         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6093                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6094
6095         lpt_init_clock_gating(dev);
6096 }
6097
6098 static void haswell_init_clock_gating(struct drm_device *dev)
6099 {
6100         struct drm_i915_private *dev_priv = dev->dev_private;
6101
6102         ilk_init_lp_watermarks(dev);
6103
6104         /* L3 caching of data atomics doesn't work -- disable it. */
6105         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
6106         I915_WRITE(HSW_ROW_CHICKEN3,
6107                    _MASKED_BIT_ENABLE(HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE));
6108
6109         /* This is required by WaCatErrorRejectionIssue:hsw */
6110         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6111                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6112                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6113
6114         /* WaVSRefCountFullforceMissDisable:hsw */
6115         I915_WRITE(GEN7_FF_THREAD_MODE,
6116                    I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
6117
6118         /* WaDisable_RenderCache_OperationalFlush:hsw */
6119         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6120
6121         /* enable HiZ Raw Stall Optimization */
6122         I915_WRITE(CACHE_MODE_0_GEN7,
6123                    _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6124
6125         /* WaDisable4x2SubspanOptimization:hsw */
6126         I915_WRITE(CACHE_MODE_1,
6127                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6128
6129         /*
6130          * BSpec recommends 8x4 when MSAA is used,
6131          * however in practice 16x4 seems fastest.
6132          *
6133          * Note that PS/WM thread counts depend on the WIZ hashing
6134          * disable bit, which we don't touch here, but it's good
6135          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6136          */
6137         I915_WRITE(GEN7_GT_MODE,
6138                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6139
6140         /* WaSampleCChickenBitEnable:hsw */
6141         I915_WRITE(HALF_SLICE_CHICKEN3,
6142                    _MASKED_BIT_ENABLE(HSW_SAMPLE_C_PERFORMANCE));
6143
6144         /* WaSwitchSolVfFArbitrationPriority:hsw */
6145         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
6146
6147         /* WaRsPkgCStateDisplayPMReq:hsw */
6148         I915_WRITE(CHICKEN_PAR1_1,
6149                    I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
6150
6151         lpt_init_clock_gating(dev);
6152 }
6153
6154 static void ivybridge_init_clock_gating(struct drm_device *dev)
6155 {
6156         struct drm_i915_private *dev_priv = dev->dev_private;
6157         uint32_t snpcr;
6158
6159         ilk_init_lp_watermarks(dev);
6160
6161         I915_WRITE(ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
6162
6163         /* WaDisableEarlyCull:ivb */
6164         I915_WRITE(_3D_CHICKEN3,
6165                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6166
6167         /* WaDisableBackToBackFlipFix:ivb */
6168         I915_WRITE(IVB_CHICKEN3,
6169                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6170                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
6171
6172         /* WaDisablePSDDualDispatchEnable:ivb */
6173         if (IS_IVB_GT1(dev))
6174                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6175                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6176
6177         /* WaDisable_RenderCache_OperationalFlush:ivb */
6178         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6179
6180         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
6181         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
6182                    GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
6183
6184         /* WaApplyL3ControlAndL3ChickenMode:ivb */
6185         I915_WRITE(GEN7_L3CNTLREG1,
6186                         GEN7_WA_FOR_GEN7_L3_CONTROL);
6187         I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
6188                    GEN7_WA_L3_CHICKEN_MODE);
6189         if (IS_IVB_GT1(dev))
6190                 I915_WRITE(GEN7_ROW_CHICKEN2,
6191                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6192         else {
6193                 /* must write both registers */
6194                 I915_WRITE(GEN7_ROW_CHICKEN2,
6195                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6196                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
6197                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6198         }
6199
6200         /* WaForceL3Serialization:ivb */
6201         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6202                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6203
6204         /*
6205          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
6206          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
6207          */
6208         I915_WRITE(GEN6_UCGCTL2,
6209                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6210
6211         /* This is required by WaCatErrorRejectionIssue:ivb */
6212         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6213                         I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6214                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6215
6216         g4x_disable_trickle_feed(dev);
6217
6218         gen7_setup_fixed_func_scheduler(dev_priv);
6219
6220         if (0) { /* causes HiZ corruption on ivb:gt1 */
6221                 /* enable HiZ Raw Stall Optimization */
6222                 I915_WRITE(CACHE_MODE_0_GEN7,
6223                            _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
6224         }
6225
6226         /* WaDisable4x2SubspanOptimization:ivb */
6227         I915_WRITE(CACHE_MODE_1,
6228                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6229
6230         /*
6231          * BSpec recommends 8x4 when MSAA is used,
6232          * however in practice 16x4 seems fastest.
6233          *
6234          * Note that PS/WM thread counts depend on the WIZ hashing
6235          * disable bit, which we don't touch here, but it's good
6236          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6237          */
6238         I915_WRITE(GEN7_GT_MODE,
6239                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6240
6241         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
6242         snpcr &= ~GEN6_MBC_SNPCR_MASK;
6243         snpcr |= GEN6_MBC_SNPCR_MED;
6244         I915_WRITE(GEN6_MBCUNIT_SNPCR, snpcr);
6245
6246         if (!HAS_PCH_NOP(dev))
6247                 cpt_init_clock_gating(dev);
6248
6249         gen6_check_mch_setup(dev);
6250 }
6251
6252 static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
6253 {
6254         I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
6255
6256         /*
6257          * Disable trickle feed and enable pnd deadline calculation
6258          */
6259         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
6260         I915_WRITE(CBR1_VLV, 0);
6261 }
6262
6263 static void valleyview_init_clock_gating(struct drm_device *dev)
6264 {
6265         struct drm_i915_private *dev_priv = dev->dev_private;
6266
6267         vlv_init_display_clock_gating(dev_priv);
6268
6269         /* WaDisableEarlyCull:vlv */
6270         I915_WRITE(_3D_CHICKEN3,
6271                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SF_DISABLE_OBJEND_CULL));
6272
6273         /* WaDisableBackToBackFlipFix:vlv */
6274         I915_WRITE(IVB_CHICKEN3,
6275                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
6276                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
6277
6278         /* WaPsdDispatchEnable:vlv */
6279         /* WaDisablePSDDualDispatchEnable:vlv */
6280         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
6281                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
6282                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
6283
6284         /* WaDisable_RenderCache_OperationalFlush:vlv */
6285         I915_WRITE(CACHE_MODE_0_GEN7, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6286
6287         /* WaForceL3Serialization:vlv */
6288         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
6289                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
6290
6291         /* WaDisableDopClockGating:vlv */
6292         I915_WRITE(GEN7_ROW_CHICKEN2,
6293                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
6294
6295         /* This is required by WaCatErrorRejectionIssue:vlv */
6296         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
6297                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
6298                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
6299
6300         gen7_setup_fixed_func_scheduler(dev_priv);
6301
6302         /*
6303          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
6304          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
6305          */
6306         I915_WRITE(GEN6_UCGCTL2,
6307                    GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
6308
6309         /* WaDisableL3Bank2xClockGate:vlv
6310          * Disabling L3 clock gating- MMIO 940c[25] = 1
6311          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
6312         I915_WRITE(GEN7_UCGCTL4,
6313                    I915_READ(GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
6314
6315         /*
6316          * BSpec says this must be set, even though
6317          * WaDisable4x2SubspanOptimization isn't listed for VLV.
6318          */
6319         I915_WRITE(CACHE_MODE_1,
6320                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
6321
6322         /*
6323          * BSpec recommends 8x4 when MSAA is used,
6324          * however in practice 16x4 seems fastest.
6325          *
6326          * Note that PS/WM thread counts depend on the WIZ hashing
6327          * disable bit, which we don't touch here, but it's good
6328          * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
6329          */
6330         I915_WRITE(GEN7_GT_MODE,
6331                    _MASKED_FIELD(GEN6_WIZ_HASHING_MASK, GEN6_WIZ_HASHING_16x4));
6332
6333         /*
6334          * WaIncreaseL3CreditsForVLVB0:vlv
6335          * This is the hardware default actually.
6336          */
6337         I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
6338
6339         /*
6340          * WaDisableVLVClockGating_VBIIssue:vlv
6341          * Disable clock gating on th GCFG unit to prevent a delay
6342          * in the reporting of vblank events.
6343          */
6344         I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
6345 }
6346
6347 static void cherryview_init_clock_gating(struct drm_device *dev)
6348 {
6349         struct drm_i915_private *dev_priv = dev->dev_private;
6350
6351         vlv_init_display_clock_gating(dev_priv);
6352
6353         /* WaVSRefCountFullforceMissDisable:chv */
6354         /* WaDSRefCountFullforceMissDisable:chv */
6355         I915_WRITE(GEN7_FF_THREAD_MODE,
6356                    I915_READ(GEN7_FF_THREAD_MODE) &
6357                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
6358
6359         /* WaDisableSemaphoreAndSyncFlipWait:chv */
6360         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
6361                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
6362
6363         /* WaDisableCSUnitClockGating:chv */
6364         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
6365                    GEN6_CSUNIT_CLOCK_GATE_DISABLE);
6366
6367         /* WaDisableSDEUnitClockGating:chv */
6368         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
6369                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
6370 }
6371
6372 static void g4x_init_clock_gating(struct drm_device *dev)
6373 {
6374         struct drm_i915_private *dev_priv = dev->dev_private;
6375         uint32_t dspclk_gate;
6376
6377         I915_WRITE(RENCLK_GATE_D1, 0);
6378         I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
6379                    GS_UNIT_CLOCK_GATE_DISABLE |
6380                    CL_UNIT_CLOCK_GATE_DISABLE);
6381         I915_WRITE(RAMCLK_GATE_D, 0);
6382         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
6383                 OVRUNIT_CLOCK_GATE_DISABLE |
6384                 OVCUNIT_CLOCK_GATE_DISABLE;
6385         if (IS_GM45(dev))
6386                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
6387         I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
6388
6389         /* WaDisableRenderCachePipelinedFlush */
6390         I915_WRITE(CACHE_MODE_0,
6391                    _MASKED_BIT_ENABLE(CM0_PIPELINED_RENDER_FLUSH_DISABLE));
6392
6393         /* WaDisable_RenderCache_OperationalFlush:g4x */
6394         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6395
6396         g4x_disable_trickle_feed(dev);
6397 }
6398
6399 static void crestline_init_clock_gating(struct drm_device *dev)
6400 {
6401         struct drm_i915_private *dev_priv = dev->dev_private;
6402
6403         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
6404         I915_WRITE(RENCLK_GATE_D2, 0);
6405         I915_WRITE(DSPCLK_GATE_D, 0);
6406         I915_WRITE(RAMCLK_GATE_D, 0);
6407         I915_WRITE16(DEUC, 0);
6408         I915_WRITE(MI_ARB_STATE,
6409                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
6410
6411         /* WaDisable_RenderCache_OperationalFlush:gen4 */
6412         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6413 }
6414
6415 static void broadwater_init_clock_gating(struct drm_device *dev)
6416 {
6417         struct drm_i915_private *dev_priv = dev->dev_private;
6418
6419         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
6420                    I965_RCC_CLOCK_GATE_DISABLE |
6421                    I965_RCPB_CLOCK_GATE_DISABLE |
6422                    I965_ISC_CLOCK_GATE_DISABLE |
6423                    I965_FBC_CLOCK_GATE_DISABLE);
6424         I915_WRITE(RENCLK_GATE_D2, 0);
6425         I915_WRITE(MI_ARB_STATE,
6426                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
6427
6428         /* WaDisable_RenderCache_OperationalFlush:gen4 */
6429         I915_WRITE(CACHE_MODE_0, _MASKED_BIT_DISABLE(RC_OP_FLUSH_ENABLE));
6430 }
6431
6432 static void gen3_init_clock_gating(struct drm_device *dev)
6433 {
6434         struct drm_i915_private *dev_priv = dev->dev_private;
6435         u32 dstate = I915_READ(D_STATE);
6436
6437         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
6438                 DSTATE_DOT_CLOCK_GATING;
6439         I915_WRITE(D_STATE, dstate);
6440
6441         if (IS_PINEVIEW(dev))
6442                 I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
6443
6444         /* IIR "flip pending" means done if this bit is set */
6445         I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
6446
6447         /* interrupts should cause a wake up from C3 */
6448         I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
6449
6450         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
6451         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
6452
6453         I915_WRITE(MI_ARB_STATE,
6454                    _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
6455 }
6456
6457 static void i85x_init_clock_gating(struct drm_device *dev)
6458 {
6459         struct drm_i915_private *dev_priv = dev->dev_private;
6460
6461         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
6462
6463         /* interrupts should cause a wake up from C3 */
6464         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
6465                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
6466
6467         I915_WRITE(MEM_MODE,
6468                    _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
6469 }
6470
6471 static void i830_init_clock_gating(struct drm_device *dev)
6472 {
6473         struct drm_i915_private *dev_priv = dev->dev_private;
6474
6475         I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
6476
6477         I915_WRITE(MEM_MODE,
6478                    _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
6479                    _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
6480 }
6481
6482 void intel_init_clock_gating(struct drm_device *dev)
6483 {
6484         struct drm_i915_private *dev_priv = dev->dev_private;
6485
6486         if (dev_priv->display.init_clock_gating)
6487                 dev_priv->display.init_clock_gating(dev);
6488 }
6489
6490 void intel_suspend_hw(struct drm_device *dev)
6491 {
6492         if (HAS_PCH_LPT(dev))
6493                 lpt_suspend_hw(dev);
6494 }
6495
6496 /* Set up chip specific power management-related functions */
6497 void intel_init_pm(struct drm_device *dev)
6498 {
6499         struct drm_i915_private *dev_priv = dev->dev_private;
6500
6501         intel_fbc_init(dev_priv);
6502
6503         /* For cxsr */
6504         if (IS_PINEVIEW(dev))
6505                 i915_pineview_get_mem_freq(dev);
6506         else if (IS_GEN5(dev))
6507                 i915_ironlake_get_mem_freq(dev);
6508
6509         /* For FIFO watermark updates */
6510         if (INTEL_INFO(dev)->gen >= 9) {
6511                 skl_setup_wm_latency(dev);
6512
6513                 dev_priv->display.init_clock_gating = skl_init_clock_gating;
6514                 dev_priv->display.update_wm = skl_update_wm;
6515                 dev_priv->display.update_sprite_wm = skl_update_sprite_wm;
6516         } else if (HAS_PCH_SPLIT(dev)) {
6517                 ilk_setup_wm_latency(dev);
6518
6519                 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
6520                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
6521                     (!IS_GEN5(dev) && dev_priv->wm.pri_latency[0] &&
6522                      dev_priv->wm.spr_latency[0] && dev_priv->wm.cur_latency[0])) {
6523                         dev_priv->display.update_wm = ilk_update_wm;
6524                         dev_priv->display.update_sprite_wm = ilk_update_sprite_wm;
6525                 } else {
6526                         DRM_DEBUG_KMS("Failed to read display plane latency. "
6527                                       "Disable CxSR\n");
6528                 }
6529
6530                 if (IS_GEN5(dev))
6531                         dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
6532                 else if (IS_GEN6(dev))
6533                         dev_priv->display.init_clock_gating = gen6_init_clock_gating;
6534                 else if (IS_IVYBRIDGE(dev))
6535                         dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
6536                 else if (IS_HASWELL(dev))
6537                         dev_priv->display.init_clock_gating = haswell_init_clock_gating;
6538                 else if (INTEL_INFO(dev)->gen == 8)
6539                         dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
6540         } else if (IS_CHERRYVIEW(dev)) {
6541                 dev_priv->display.update_wm = valleyview_update_wm;
6542                 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
6543                 dev_priv->display.init_clock_gating =
6544                         cherryview_init_clock_gating;
6545         } else if (IS_VALLEYVIEW(dev)) {
6546                 dev_priv->display.update_wm = valleyview_update_wm;
6547                 dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
6548                 dev_priv->display.init_clock_gating =
6549                         valleyview_init_clock_gating;
6550         } else if (IS_PINEVIEW(dev)) {
6551                 if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
6552                                             dev_priv->is_ddr3,
6553                                             dev_priv->fsb_freq,
6554                                             dev_priv->mem_freq)) {
6555                         DRM_INFO("failed to find known CxSR latency "
6556                                  "(found ddr%s fsb freq %d, mem freq %d), "
6557                                  "disabling CxSR\n",
6558                                  (dev_priv->is_ddr3 == 1) ? "3" : "2",
6559                                  dev_priv->fsb_freq, dev_priv->mem_freq);
6560                         /* Disable CxSR and never update its watermark again */
6561                         intel_set_memory_cxsr(dev_priv, false);
6562                         dev_priv->display.update_wm = NULL;
6563                 } else
6564                         dev_priv->display.update_wm = pineview_update_wm;
6565                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
6566         } else if (IS_G4X(dev)) {
6567                 dev_priv->display.update_wm = g4x_update_wm;
6568                 dev_priv->display.init_clock_gating = g4x_init_clock_gating;
6569         } else if (IS_GEN4(dev)) {
6570                 dev_priv->display.update_wm = i965_update_wm;
6571                 if (IS_CRESTLINE(dev))
6572                         dev_priv->display.init_clock_gating = crestline_init_clock_gating;
6573                 else if (IS_BROADWATER(dev))
6574                         dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
6575         } else if (IS_GEN3(dev)) {
6576                 dev_priv->display.update_wm = i9xx_update_wm;
6577                 dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
6578                 dev_priv->display.init_clock_gating = gen3_init_clock_gating;
6579         } else if (IS_GEN2(dev)) {
6580                 if (INTEL_INFO(dev)->num_pipes == 1) {
6581                         dev_priv->display.update_wm = i845_update_wm;
6582                         dev_priv->display.get_fifo_size = i845_get_fifo_size;
6583                 } else {
6584                         dev_priv->display.update_wm = i9xx_update_wm;
6585                         dev_priv->display.get_fifo_size = i830_get_fifo_size;
6586                 }
6587
6588                 if (IS_I85X(dev) || IS_I865G(dev))
6589                         dev_priv->display.init_clock_gating = i85x_init_clock_gating;
6590                 else
6591                         dev_priv->display.init_clock_gating = i830_init_clock_gating;
6592         } else {
6593                 DRM_ERROR("unexpected fall-through in intel_init_pm\n");
6594         }
6595 }
6596
6597 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val)
6598 {
6599         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6600
6601         if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6602                 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
6603                 return -EAGAIN;
6604         }
6605
6606         I915_WRITE(GEN6_PCODE_DATA, *val);
6607         I915_WRITE(GEN6_PCODE_DATA1, 0);
6608         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6609
6610         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6611                      500)) {
6612                 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
6613                 return -ETIMEDOUT;
6614         }
6615
6616         *val = I915_READ(GEN6_PCODE_DATA);
6617         I915_WRITE(GEN6_PCODE_DATA, 0);
6618
6619         return 0;
6620 }
6621
6622 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
6623 {
6624         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
6625
6626         if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
6627                 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
6628                 return -EAGAIN;
6629         }
6630
6631         I915_WRITE(GEN6_PCODE_DATA, val);
6632         I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
6633
6634         if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
6635                      500)) {
6636                 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
6637                 return -ETIMEDOUT;
6638         }
6639
6640         I915_WRITE(GEN6_PCODE_DATA, 0);
6641
6642         return 0;
6643 }
6644
6645 static int vlv_gpu_freq_div(unsigned int czclk_freq)
6646 {
6647         switch (czclk_freq) {
6648         case 200:
6649                 return 10;
6650         case 267:
6651                 return 12;
6652         case 320:
6653         case 333:
6654                 return 16;
6655         case 400:
6656                 return 20;
6657         default:
6658                 return -1;
6659         }
6660 }
6661
6662 static int byt_gpu_freq(struct drm_i915_private *dev_priv, int val)
6663 {
6664         int div, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
6665
6666         div = vlv_gpu_freq_div(czclk_freq);
6667         if (div < 0)
6668                 return div;
6669
6670         return DIV_ROUND_CLOSEST(czclk_freq * (val + 6 - 0xbd), div);
6671 }
6672
6673 static int byt_freq_opcode(struct drm_i915_private *dev_priv, int val)
6674 {
6675         int mul, czclk_freq = DIV_ROUND_CLOSEST(dev_priv->mem_freq, 4);
6676
6677         mul = vlv_gpu_freq_div(czclk_freq);
6678         if (mul < 0)
6679                 return mul;
6680
6681         return DIV_ROUND_CLOSEST(mul * val, czclk_freq) + 0xbd - 6;
6682 }
6683
6684 static int chv_gpu_freq(struct drm_i915_private *dev_priv, int val)
6685 {
6686         int div, czclk_freq = dev_priv->rps.cz_freq;
6687
6688         div = vlv_gpu_freq_div(czclk_freq) / 2;
6689         if (div < 0)
6690                 return div;
6691
6692         return DIV_ROUND_CLOSEST(czclk_freq * val, 2 * div) / 2;
6693 }
6694
6695 static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
6696 {
6697         int mul, czclk_freq = dev_priv->rps.cz_freq;
6698
6699         mul = vlv_gpu_freq_div(czclk_freq) / 2;
6700         if (mul < 0)
6701                 return mul;
6702
6703         /* CHV needs even values */
6704         return DIV_ROUND_CLOSEST(val * 2 * mul, czclk_freq) * 2;
6705 }
6706
6707 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val)
6708 {
6709         if (IS_GEN9(dev_priv->dev))
6710                 return (val * GT_FREQUENCY_MULTIPLIER) / GEN9_FREQ_SCALER;
6711         else if (IS_CHERRYVIEW(dev_priv->dev))
6712                 return chv_gpu_freq(dev_priv, val);
6713         else if (IS_VALLEYVIEW(dev_priv->dev))
6714                 return byt_gpu_freq(dev_priv, val);
6715         else
6716                 return val * GT_FREQUENCY_MULTIPLIER;
6717 }
6718
6719 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val)
6720 {
6721         if (IS_GEN9(dev_priv->dev))
6722                 return (val * GEN9_FREQ_SCALER) / GT_FREQUENCY_MULTIPLIER;
6723         else if (IS_CHERRYVIEW(dev_priv->dev))
6724                 return chv_freq_opcode(dev_priv, val);
6725         else if (IS_VALLEYVIEW(dev_priv->dev))
6726                 return byt_freq_opcode(dev_priv, val);
6727         else
6728                 return val / GT_FREQUENCY_MULTIPLIER;
6729 }
6730
6731 void intel_pm_setup(struct drm_device *dev)
6732 {
6733         struct drm_i915_private *dev_priv = dev->dev_private;
6734
6735         mutex_init(&dev_priv->rps.hw_lock);
6736
6737         INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
6738                           intel_gen6_powersave_work);
6739
6740         dev_priv->pm.suspended = false;
6741 }