drivers/gpu/drm/i915/intel_clock_gating.c

   1 /*
   2  * Copyright © 2012 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eugeni Dodonov <eugeni.dodonov@intel.com>
  25  *
  26  */
  27
  28 #include "display/intel_de.h"
  29 #include "display/intel_display.h"
  30 #include "display/intel_display_trace.h"
  31 #include "display/intel_fbc_regs.h"
  32 #include "display/skl_watermark.h"
  33
  34 #include "gt/intel_engine_regs.h"
  35 #include "gt/intel_gt.h"
  36 #include "gt/intel_gt_mcr.h"
  37 #include "gt/intel_gt_regs.h"
  38
  39 #include "i915_drv.h"
  40 #include "i915_reg.h"
  41 #include "intel_clock_gating.h"
  42 #include "intel_mchbar_regs.h"
  43 #include "vlv_sideband.h"
  44
  45 struct drm_i915_clock_gating_funcs {
  46         void (*init_clock_gating)(struct drm_i915_private *i915);
  47 };
  48
  49 static void gen9_init_clock_gating(struct drm_i915_private *i915)
  50 {
  51         if (HAS_LLC(i915)) {
  52                 /*
  53                  * WaCompressedResourceDisplayNewHashMode:skl,kbl
  54                  * Display WA #0390: skl,kbl
  55                  *
  56                  * Must match Sampler, Pixel Back End, and Media. See
  57                  * WaCompressedResourceSamplerPbeMediaNewHashMode.
  58                  */
  59                 intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
  60         }
  61
  62         /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
  63         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
  64
  65         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
  66         intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
  67
  68         /*
  69          * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
  70          * Display WA #0859: skl,bxt,kbl,glk,cfl
  71          */
  72         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
  73 }
  74
  75 static void bxt_init_clock_gating(struct drm_i915_private *i915)
  76 {
  77         gen9_init_clock_gating(i915);
  78
  79         /* WaDisableSDEUnitClockGating:bxt */
  80         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
  81
  82         /*
  83          * FIXME:
  84          * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
  85          */
  86         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
  87
  88         /*
  89          * Wa: Backlight PWM may stop in the asserted state, causing backlight
  90          * to stay fully on.
  91          */
  92         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
  93                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
  94                            PWM1_GATING_DIS | PWM2_GATING_DIS);
  95
  96         /*
  97          * Lower the display internal timeout.
  98          * This is needed to avoid any hard hangs when DSI port PLL
  99          * is off and a MMIO access is attempted by any privilege
 100          * application, using batch buffers or any other means.
 101          */
 102         intel_uncore_write(&i915->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
 103
 104         /*
 105          * WaFbcTurnOffFbcWatermark:bxt
 106          * Display WA #0562: bxt
 107          */
 108         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 109 }
 110
 111 static void glk_init_clock_gating(struct drm_i915_private *i915)
 112 {
 113         gen9_init_clock_gating(i915);
 114
 115         /*
 116          * WaDisablePWMClockGating:glk
 117          * Backlight PWM may stop in the asserted state, causing backlight
 118          * to stay fully on.
 119          */
 120         intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
 121                            intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
 122                            PWM1_GATING_DIS | PWM2_GATING_DIS);
 123 }
 124
 125 static void ibx_init_clock_gating(struct drm_i915_private *i915)
 126 {
 127         /*
 128          * On Ibex Peak and Cougar Point, we need to disable clock
 129          * gating for the panel power sequencer or it will fail to
 130          * start up when no ports are active.
 131          */
 132         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
 133 }
 134
 135 static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
 136 {
 137         enum pipe pipe;
 138
 139         for_each_pipe(dev_priv, pipe) {
 140                 intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
 141
 142                 intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
 143                 intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
 144         }
 145 }
 146
 147 static void ilk_init_clock_gating(struct drm_i915_private *i915)
 148 {
 149         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 150
 151         /*
 152          * Required for FBC
 153          * WaFbcDisableDpfcClockGating:ilk
 154          */
 155         dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
 156                    ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
 157                    ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
 158
 159         intel_uncore_write(&i915->uncore, PCH_3DCGDIS0,
 160                            MARIUNIT_CLOCK_GATE_DISABLE |
 161                            SVSMUNIT_CLOCK_GATE_DISABLE);
 162         intel_uncore_write(&i915->uncore, PCH_3DCGDIS1,
 163                            VFMUNIT_CLOCK_GATE_DISABLE);
 164
 165         /*
 166          * According to the spec the following bits should be set in
 167          * order to enable memory self-refresh
 168          * The bit 22/21 of 0x42004
 169          * The bit 5 of 0x42020
 170          * The bit 15 of 0x45000
 171          */
 172         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 173                            (intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 174                             ILK_DPARB_GATE | ILK_VSDPFD_FULL));
 175         dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
 176         intel_uncore_write(&i915->uncore, DISP_ARB_CTL,
 177                            (intel_uncore_read(&i915->uncore, DISP_ARB_CTL) |
 178                             DISP_FBC_WM_DIS));
 179
 180         /*
 181          * Based on the document from hardware guys the following bits
 182          * should be set unconditionally in order to enable FBC.
 183          * The bit 22 of 0x42000
 184          * The bit 22 of 0x42004
 185          * The bit 7,8,9 of 0x42020.
 186          */
 187         if (IS_IRONLAKE_M(i915)) {
 188                 /* WaFbcAsynchFlipDisableFbcQueue:ilk */
 189                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 190                 intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
 191         }
 192
 193         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 194
 195         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 196
 197         g4x_disable_trickle_feed(i915);
 198
 199         ibx_init_clock_gating(i915);
 200 }
 201
 202 static void cpt_init_clock_gating(struct drm_i915_private *i915)
 203 {
 204         enum pipe pipe;
 205         u32 val;
 206
 207         /*
 208          * On Ibex Peak and Cougar Point, we need to disable clock
 209          * gating for the panel power sequencer or it will fail to
 210          * start up when no ports are active.
 211          */
 212         intel_uncore_write(&i915->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
 213                            PCH_DPLUNIT_CLOCK_GATE_DISABLE |
 214                            PCH_CPUNIT_CLOCK_GATE_DISABLE);
 215         intel_uncore_rmw(&i915->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
 216         /* The below fixes the weird display corruption, a few pixels shifted
 217          * downward, on (only) LVDS of some HP laptops with IVY.
 218          */
 219         for_each_pipe(i915, pipe) {
 220                 val = intel_uncore_read(&i915->uncore, TRANS_CHICKEN2(pipe));
 221                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
 222                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 223                 if (i915->display.vbt.fdi_rx_polarity_inverted)
 224                         val |= TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
 225                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_COUNTER;
 226                 val &= ~TRANS_CHICKEN2_DISABLE_DEEP_COLOR_MODESWITCH;
 227                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN2(pipe), val);
 228         }
 229         /* WADP0ClockGatingDisable */
 230         for_each_pipe(i915, pipe) {
 231                 intel_uncore_write(&i915->uncore, TRANS_CHICKEN1(pipe),
 232                                    TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 233         }
 234 }
 235
 236 static void gen6_check_mch_setup(struct drm_i915_private *i915)
 237 {
 238         u32 tmp;
 239
 240         tmp = intel_uncore_read(&i915->uncore, MCH_SSKPD);
 241         if (REG_FIELD_GET(SSKPD_WM0_MASK_SNB, tmp) != 12)
 242                 drm_dbg_kms(&i915->drm,
 243                             "Wrong MCH_SSKPD value: 0x%08x This can cause underruns.\n",
 244                             tmp);
 245 }
 246
 247 static void gen6_init_clock_gating(struct drm_i915_private *i915)
 248 {
 249         u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 250
 251         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 252
 253         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 254
 255         intel_uncore_write(&i915->uncore, GEN6_UCGCTL1,
 256                            intel_uncore_read(&i915->uncore, GEN6_UCGCTL1) |
 257                            GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
 258                            GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 259
 260         /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
 261          * gating disable must be set.  Failure to set it results in
 262          * flickering pixels due to Z write ordering failures after
 263          * some amount of runtime in the Mesa "fire" demo, and Unigine
 264          * Sanctuary and Tropics, and apparently anything else with
 265          * alpha test or pixel discard.
 266          *
 267          * According to the spec, bit 11 (RCCUNIT) must also be set,
 268          * but we didn't debug actual testcases to find it out.
 269          *
 270          * WaDisableRCCUnitClockGating:snb
 271          * WaDisableRCPBUnitClockGating:snb
 272          */
 273         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 274                            GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
 275                            GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
 276
 277         /*
 278          * According to the spec the following bits should be
 279          * set in order to enable memory self-refresh and fbc:
 280          * The bit21 and bit22 of 0x42000
 281          * The bit21 and bit22 of 0x42004
 282          * The bit5 and bit7 of 0x42020
 283          * The bit14 of 0x70180
 284          * The bit14 of 0x71180
 285          *
 286          * WaFbcAsynchFlipDisableFbcQueue:snb
 287          */
 288         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN1,
 289                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN1) |
 290                            ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
 291         intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
 292                            intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
 293                            ILK_DPARB_GATE | ILK_VSDPFD_FULL);
 294         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D,
 295                            intel_uncore_read(&i915->uncore, ILK_DSPCLK_GATE_D) |
 296                            ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
 297                            ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
 298
 299         g4x_disable_trickle_feed(i915);
 300
 301         cpt_init_clock_gating(i915);
 302
 303         gen6_check_mch_setup(i915);
 304 }
 305
 306 static void lpt_init_clock_gating(struct drm_i915_private *i915)
 307 {
 308         /*
 309          * TODO: this bit should only be enabled when really needed, then
 310          * disabled when not needed anymore in order to save power.
 311          */
 312         if (HAS_PCH_LPT_LP(i915))
 313                 intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D,
 314                                  0, PCH_LP_PARTITION_LEVEL_DISABLE);
 315
 316         /* WADPOClockGatingDisable:hsw */
 317         intel_uncore_rmw(&i915->uncore, TRANS_CHICKEN1(PIPE_A),
 318                          0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 319 }
 320
 321 static void gen8_set_l3sqc_credits(struct drm_i915_private *i915,
 322                                    int general_prio_credits,
 323                                    int high_prio_credits)
 324 {
 325         u32 misccpctl;
 326         u32 val;
 327
 328         /* WaTempDisableDOPClkGating:bdw */
 329         misccpctl = intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 330                                      GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 331
 332         val = intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 333         val &= ~L3_PRIO_CREDITS_MASK;
 334         val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
 335         val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
 336         intel_gt_mcr_multicast_write(to_gt(i915), GEN8_L3SQCREG1, val);
 337
 338         /*
 339          * Wait at least 100 clocks before re-enabling clock gating.
 340          * See the definition of L3SQCREG1 in BSpec.
 341          */
 342         intel_gt_mcr_read_any(to_gt(i915), GEN8_L3SQCREG1);
 343         udelay(1);
 344         intel_uncore_write(&i915->uncore, GEN7_MISCCPCTL, misccpctl);
 345 }
 346
 347 static void dg2_init_clock_gating(struct drm_i915_private *i915)
 348 {
 349         /* Wa_22010954014:dg2 */
 350         intel_uncore_rmw(&i915->uncore, XEHP_CLOCK_GATE_DIS, 0,
 351                          SGSI_SIDECLK_DIS);
 352 }
 353
 354 static void cnp_init_clock_gating(struct drm_i915_private *i915)
 355 {
 356         if (!HAS_PCH_CNP(i915))
 357                 return;
 358
 359         /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
 360         intel_uncore_rmw(&i915->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
 361 }
 362
 363 static void cfl_init_clock_gating(struct drm_i915_private *i915)
 364 {
 365         cnp_init_clock_gating(i915);
 366         gen9_init_clock_gating(i915);
 367
 368         /* WAC6entrylatency:cfl */
 369         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 370
 371         /*
 372          * WaFbcTurnOffFbcWatermark:cfl
 373          * Display WA #0562: cfl
 374          */
 375         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 376 }
 377
 378 static void kbl_init_clock_gating(struct drm_i915_private *i915)
 379 {
 380         gen9_init_clock_gating(i915);
 381
 382         /* WAC6entrylatency:kbl */
 383         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 384
 385         /* WaDisableSDEUnitClockGating:kbl */
 386         if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
 387                 intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6,
 388                                  0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 389
 390         /* WaDisableGamClockGating:kbl */
 391         if (IS_KABYLAKE(i915) && IS_GRAPHICS_STEP(i915, 0, STEP_C0))
 392                 intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1,
 393                                  0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
 394
 395         /*
 396          * WaFbcTurnOffFbcWatermark:kbl
 397          * Display WA #0562: kbl
 398          */
 399         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 400 }
 401
 402 static void skl_init_clock_gating(struct drm_i915_private *i915)
 403 {
 404         gen9_init_clock_gating(i915);
 405
 406         /* WaDisableDopClockGating:skl */
 407         intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 408                          GEN7_DOP_CLOCK_GATE_ENABLE, 0);
 409
 410         /* WAC6entrylatency:skl */
 411         intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 412
 413         /*
 414          * WaFbcTurnOffFbcWatermark:skl
 415          * Display WA #0562: skl
 416          */
 417         intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 418 }
 419
 420 static void bdw_init_clock_gating(struct drm_i915_private *i915)
 421 {
 422         enum pipe pipe;
 423
 424         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 425         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 426
 427         /* WaSwitchSolVfFArbitrationPriority:bdw */
 428         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 429
 430         /* WaPsrDPAMaskVBlankInSRD:bdw */
 431         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
 432
 433         for_each_pipe(i915, pipe) {
 434                 /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
 435                 intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
 436                                  0, BDW_UNMASK_VBL_TO_REGS_IN_SRD);
 437         }
 438
 439         /* WaVSRefCountFullforceMissDisable:bdw */
 440         /* WaDSRefCountFullforceMissDisable:bdw */
 441         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 442                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 443
 444         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 445                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 446
 447         /* WaDisableSDEUnitClockGating:bdw */
 448         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 449
 450         /* WaProgramL3SqcReg1Default:bdw */
 451         gen8_set_l3sqc_credits(i915, 30, 2);
 452
 453         /* WaKVMNotificationOnConfigChange:bdw */
 454         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR2_1,
 455                          0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
 456
 457         lpt_init_clock_gating(i915);
 458
 459         /* WaDisableDopClockGating:bdw
 460          *
 461          * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
 462          * clock gating.
 463          */
 464         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
 465 }
 466
 467 static void hsw_init_clock_gating(struct drm_i915_private *i915)
 468 {
 469         enum pipe pipe;
 470
 471         /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
 472         intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 473
 474         /* WaPsrDPAMaskVBlankInSRD:hsw */
 475         intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
 476
 477         for_each_pipe(i915, pipe) {
 478                 /* WaPsrDPRSUnmaskVBlankInSRD:hsw */
 479                 intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
 480                                  0, HSW_UNMASK_VBL_TO_REGS_IN_SRD);
 481         }
 482
 483         /* This is required by WaCatErrorRejectionIssue:hsw */
 484         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 485                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 486
 487         /* WaSwitchSolVfFArbitrationPriority:hsw */
 488         intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 489
 490         lpt_init_clock_gating(i915);
 491 }
 492
 493 static void ivb_init_clock_gating(struct drm_i915_private *i915)
 494 {
 495         intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 496
 497         /* WaFbcAsynchFlipDisableFbcQueue:ivb */
 498         intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 499
 500         /* WaDisableBackToBackFlipFix:ivb */
 501         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 502                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 503                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 504
 505         if (IS_IVB_GT1(i915))
 506                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 507                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 508         else {
 509                 /* must write both registers */
 510                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 511                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 512                 intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2_GT2,
 513                                    _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 514         }
 515
 516         /*
 517          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 518          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
 519          */
 520         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 521                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 522
 523         /* This is required by WaCatErrorRejectionIssue:ivb */
 524         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 525                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 526
 527         g4x_disable_trickle_feed(i915);
 528
 529         intel_uncore_rmw(&i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
 530                          GEN6_MBC_SNPCR_MED);
 531
 532         if (!HAS_PCH_NOP(i915))
 533                 cpt_init_clock_gating(i915);
 534
 535         gen6_check_mch_setup(i915);
 536 }
 537
 538 static void vlv_init_clock_gating(struct drm_i915_private *i915)
 539 {
 540         /* WaDisableBackToBackFlipFix:vlv */
 541         intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
 542                            CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
 543                            CHICKEN3_DGMG_DONE_FIX_DISABLE);
 544
 545         /* WaDisableDopClockGating:vlv */
 546         intel_uncore_write(&i915->uncore, GEN7_ROW_CHICKEN2,
 547                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 548
 549         /* This is required by WaCatErrorRejectionIssue:vlv */
 550         intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 551                          0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 552
 553         /*
 554          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
 555          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
 556          */
 557         intel_uncore_write(&i915->uncore, GEN6_UCGCTL2,
 558                            GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 559
 560         /* WaDisableL3Bank2xClockGate:vlv
 561          * Disabling L3 clock gating- MMIO 940c[25] = 1
 562          * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
 563         intel_uncore_rmw(&i915->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
 564
 565         /*
 566          * WaDisableVLVClockGating_VBIIssue:vlv
 567          * Disable clock gating on th GCFG unit to prevent a delay
 568          * in the reporting of vblank events.
 569          */
 570         intel_uncore_write(&i915->uncore, VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
 571 }
 572
 573 static void chv_init_clock_gating(struct drm_i915_private *i915)
 574 {
 575         /* WaVSRefCountFullforceMissDisable:chv */
 576         /* WaDSRefCountFullforceMissDisable:chv */
 577         intel_uncore_rmw(&i915->uncore, GEN7_FF_THREAD_MODE,
 578                          GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 579
 580         /* WaDisableSemaphoreAndSyncFlipWait:chv */
 581         intel_uncore_write(&i915->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
 582                            _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 583
 584         /* WaDisableCSUnitClockGating:chv */
 585         intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 586
 587         /* WaDisableSDEUnitClockGating:chv */
 588         intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 589
 590         /*
 591          * WaProgramL3SqcReg1Default:chv
 592          * See gfxspecs/Related Documents/Performance Guide/
 593          * LSQC Setting Recommendations.
 594          */
 595         gen8_set_l3sqc_credits(i915, 38, 2);
 596 }
 597
 598 static void g4x_init_clock_gating(struct drm_i915_private *i915)
 599 {
 600         u32 dspclk_gate;
 601
 602         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, 0);
 603         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
 604                            GS_UNIT_CLOCK_GATE_DISABLE |
 605                            CL_UNIT_CLOCK_GATE_DISABLE);
 606         intel_uncore_write(&i915->uncore, RAMCLK_GATE_D, 0);
 607         dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
 608                 OVRUNIT_CLOCK_GATE_DISABLE |
 609                 OVCUNIT_CLOCK_GATE_DISABLE;
 610         if (IS_GM45(i915))
 611                 dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
 612         intel_uncore_write(&i915->uncore, DSPCLK_GATE_D(i915), dspclk_gate);
 613
 614         g4x_disable_trickle_feed(i915);
 615 }
 616
 617 static void i965gm_init_clock_gating(struct drm_i915_private *i915)
 618 {
 619         struct intel_uncore *uncore = &i915->uncore;
 620
 621         intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 622         intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
 623         intel_uncore_write(uncore, DSPCLK_GATE_D(i915), 0);
 624         intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
 625         intel_uncore_write16(uncore, DEUC, 0);
 626         intel_uncore_write(uncore,
 627                            MI_ARB_STATE,
 628                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 629 }
 630
 631 static void i965g_init_clock_gating(struct drm_i915_private *i915)
 632 {
 633         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
 634                            I965_RCC_CLOCK_GATE_DISABLE |
 635                            I965_RCPB_CLOCK_GATE_DISABLE |
 636                            I965_ISC_CLOCK_GATE_DISABLE |
 637                            I965_FBC_CLOCK_GATE_DISABLE);
 638         intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, 0);
 639         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 640                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 641 }
 642
 643 static void gen3_init_clock_gating(struct drm_i915_private *i915)
 644 {
 645         u32 dstate = intel_uncore_read(&i915->uncore, D_STATE);
 646
 647         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
 648                 DSTATE_DOT_CLOCK_GATING;
 649         intel_uncore_write(&i915->uncore, D_STATE, dstate);
 650
 651         if (IS_PINEVIEW(i915))
 652                 intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 653                                    _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
 654
 655         /* IIR "flip pending" means done if this bit is set */
 656         intel_uncore_write(&i915->uncore, ECOSKPD(RENDER_RING_BASE),
 657                            _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
 658
 659         /* interrupts should cause a wake up from C3 */
 660         intel_uncore_write(&i915->uncore, INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_INT_EN));
 661
 662         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
 663         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 664                            _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
 665
 666         intel_uncore_write(&i915->uncore, MI_ARB_STATE,
 667                            _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 668 }
 669
 670 static void i85x_init_clock_gating(struct drm_i915_private *i915)
 671 {
 672         intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
 673
 674         /* interrupts should cause a wake up from C3 */
 675         intel_uncore_write(&i915->uncore, MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
 676                            _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
 677
 678         intel_uncore_write(&i915->uncore, MEM_MODE,
 679                            _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
 680
 681         /*
 682          * Have FBC ignore 3D activity since we use software
 683          * render tracking, and otherwise a pure 3D workload
 684          * (even if it just renders a single frame and then does
 685          * abosultely nothing) would not allow FBC to recompress
 686          * until a 2D blit occurs.
 687          */
 688         intel_uncore_write(&i915->uncore, SCPD0,
 689                            _MASKED_BIT_ENABLE(SCPD_FBC_IGNORE_3D));
 690 }
 691
 692 static void i830_init_clock_gating(struct drm_i915_private *i915)
 693 {
 694         intel_uncore_write(&i915->uncore, MEM_MODE,
 695                            _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
 696                            _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 697 }
 698
 699 void intel_clock_gating_init(struct drm_i915_private *i915)
 700 {
 701         i915->clock_gating_funcs->init_clock_gating(i915);
 702 }
 703
 704 static void nop_init_clock_gating(struct drm_i915_private *i915)
 705 {
 706         drm_dbg_kms(&i915->drm,
 707                     "No clock gating settings or workarounds applied.\n");
 708 }
 709
 710 #define CG_FUNCS(platform)                                              \
 711 static const struct drm_i915_clock_gating_funcs platform##_clock_gating_funcs = { \
 712         .init_clock_gating = platform##_init_clock_gating,              \
 713 }
 714
 715 CG_FUNCS(dg2);
 716 CG_FUNCS(cfl);
 717 CG_FUNCS(skl);
 718 CG_FUNCS(kbl);
 719 CG_FUNCS(bxt);
 720 CG_FUNCS(glk);
 721 CG_FUNCS(bdw);
 722 CG_FUNCS(chv);
 723 CG_FUNCS(hsw);
 724 CG_FUNCS(ivb);
 725 CG_FUNCS(vlv);
 726 CG_FUNCS(gen6);
 727 CG_FUNCS(ilk);
 728 CG_FUNCS(g4x);
 729 CG_FUNCS(i965gm);
 730 CG_FUNCS(i965g);
 731 CG_FUNCS(gen3);
 732 CG_FUNCS(i85x);
 733 CG_FUNCS(i830);
 734 CG_FUNCS(nop);
 735 #undef CG_FUNCS
 736
 737 /**
 738  * intel_clock_gating_hooks_init - setup the clock gating hooks
 739  * @i915: device private
 740  *
 741  * Setup the hooks that configure which clocks of a given platform can be
 742  * gated and also apply various GT and display specific workarounds for these
 743  * platforms. Note that some GT specific workarounds are applied separately
 744  * when GPU contexts or batchbuffers start their execution.
 745  */
 746 void intel_clock_gating_hooks_init(struct drm_i915_private *i915)
 747 {
 748         if (IS_DG2(i915))
 749                 i915->clock_gating_funcs = &dg2_clock_gating_funcs;
 750         else if (IS_COFFEELAKE(i915) || IS_COMETLAKE(i915))
 751                 i915->clock_gating_funcs = &cfl_clock_gating_funcs;
 752         else if (IS_SKYLAKE(i915))
 753                 i915->clock_gating_funcs = &skl_clock_gating_funcs;
 754         else if (IS_KABYLAKE(i915))
 755                 i915->clock_gating_funcs = &kbl_clock_gating_funcs;
 756         else if (IS_BROXTON(i915))
 757                 i915->clock_gating_funcs = &bxt_clock_gating_funcs;
 758         else if (IS_GEMINILAKE(i915))
 759                 i915->clock_gating_funcs = &glk_clock_gating_funcs;
 760         else if (IS_BROADWELL(i915))
 761                 i915->clock_gating_funcs = &bdw_clock_gating_funcs;
 762         else if (IS_CHERRYVIEW(i915))
 763                 i915->clock_gating_funcs = &chv_clock_gating_funcs;
 764         else if (IS_HASWELL(i915))
 765                 i915->clock_gating_funcs = &hsw_clock_gating_funcs;
 766         else if (IS_IVYBRIDGE(i915))
 767                 i915->clock_gating_funcs = &ivb_clock_gating_funcs;
 768         else if (IS_VALLEYVIEW(i915))
 769                 i915->clock_gating_funcs = &vlv_clock_gating_funcs;
 770         else if (GRAPHICS_VER(i915) == 6)
 771                 i915->clock_gating_funcs = &gen6_clock_gating_funcs;
 772         else if (GRAPHICS_VER(i915) == 5)
 773                 i915->clock_gating_funcs = &ilk_clock_gating_funcs;
 774         else if (IS_G4X(i915))
 775                 i915->clock_gating_funcs = &g4x_clock_gating_funcs;
 776         else if (IS_I965GM(i915))
 777                 i915->clock_gating_funcs = &i965gm_clock_gating_funcs;
 778         else if (IS_I965G(i915))
 779                 i915->clock_gating_funcs = &i965g_clock_gating_funcs;
 780         else if (GRAPHICS_VER(i915) == 3)
 781                 i915->clock_gating_funcs = &gen3_clock_gating_funcs;
 782         else if (IS_I85X(i915) || IS_I865G(i915))
 783                 i915->clock_gating_funcs = &i85x_clock_gating_funcs;
 784         else if (GRAPHICS_VER(i915) == 2)
 785                 i915->clock_gating_funcs = &i830_clock_gating_funcs;
 786         else
 787                 i915->clock_gating_funcs = &nop_clock_gating_funcs;
 788 }