Merge tag 'drm-intel-gt-next-2022-11-03' of git://anongit.freedesktop.org/drm/drm...
[linux-block.git] / drivers / gpu / drm / i915 / intel_pm.c
index 19d4a88184d7a1b74c1a728afcfc3dead60f4e2a..ee34e278563603ddb08141aa87a59ed3550e1f05 100644 (file)
@@ -30,6 +30,8 @@
 #include "display/skl_watermark.h"
 
 #include "gt/intel_engine_regs.h"
+#include "gt/intel_gt.h"
+#include "gt/intel_gt_mcr.h"
 #include "gt/intel_gt_regs.h"
 
 #include "i915_drv.h"
@@ -58,25 +60,20 @@ static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
                 * Must match Sampler, Pixel Back End, and Media. See
                 * WaCompressedResourceSamplerPbeMediaNewHashMode.
                 */
-               intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
-                          intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) |
-                          SKL_DE_COMPRESSED_HASH_MODE);
+               intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
        }
 
        /* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
-       intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
-                  intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
+       intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
 
        /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
-       intel_uncore_write(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1,
-                  intel_uncore_read(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
+       intel_uncore_rmw(&dev_priv->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
 
        /*
         * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
         * Display WA #0859: skl,bxt,kbl,glk,cfl
         */
-       intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
-                  DISP_FBC_MEMORY_WAKE);
+       intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
 }
 
 static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -84,15 +81,13 @@ static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
        gen9_init_clock_gating(dev_priv);
 
        /* WaDisableSDEUnitClockGating:bxt */
-       intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
-                  GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
+       intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 
        /*
         * FIXME:
         * GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ applies on 3x6 GT SKUs only.
         */
-       intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
-                  GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
+       intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 
        /*
         * Wa: Backlight PWM may stop in the asserted state, causing backlight
@@ -113,16 +108,13 @@ static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
         * WaFbcTurnOffFbcWatermark:bxt
         * Display WA #0562: bxt
         */
-       intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
-                  DISP_FBC_WM_DIS);
+       intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 
        /*
         * WaFbcHighMemBwCorruptionAvoidance:bxt
         * Display WA #0883: bxt
         */
-       intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-                          intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
-                          DPFC_DISABLE_DUMMY0);
+       intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 }
 
 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -4053,9 +4045,9 @@ void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
  */
 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
 {
-       intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK) & ~WM_LP_ENABLE);
-       intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK) & ~WM_LP_ENABLE);
-       intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK) & ~WM_LP_ENABLE);
+       intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
+       intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
+       intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
 
        /*
         * Don't touch WM_LP_SPRITE_ENABLE here.
@@ -4109,9 +4101,7 @@ static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
        enum pipe pipe;
 
        for_each_pipe(dev_priv, pipe) {
-               intel_uncore_write(&dev_priv->uncore, DSPCNTR(pipe),
-                          intel_uncore_read(&dev_priv->uncore, DSPCNTR(pipe)) |
-                          DISP_TRICKLE_FEED_DISABLE);
+               intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(pipe), 0, DISP_TRICKLE_FEED_DISABLE);
 
                intel_uncore_rmw(&dev_priv->uncore, DSPSURF(pipe), 0, 0);
                intel_uncore_posting_read(&dev_priv->uncore, DSPSURF(pipe));
@@ -4160,19 +4150,13 @@ static void ilk_init_clock_gating(struct drm_i915_private *dev_priv)
         */
        if (IS_IRONLAKE_M(dev_priv)) {
                /* WaFbcAsynchFlipDisableFbcQueue:ilk */
-               intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
-                          intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
-                          ILK_FBCQ_DIS);
-               intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
-                          intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
-                          ILK_DPARB_GATE);
+               intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
+               intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
        }
 
        intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 
-       intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
-                  intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
-                  ILK_ELPIN_409_SELECT);
+       intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 
        g4x_disable_trickle_feed(dev_priv);
 
@@ -4192,8 +4176,7 @@ static void cpt_init_clock_gating(struct drm_i915_private *dev_priv)
        intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE |
                   PCH_DPLUNIT_CLOCK_GATE_DISABLE |
                   PCH_CPUNIT_CLOCK_GATE_DISABLE);
-       intel_uncore_write(&dev_priv->uncore, SOUTH_CHICKEN2, intel_uncore_read(&dev_priv->uncore, SOUTH_CHICKEN2) |
-                  DPLS_EDP_PPS_FIX_DIS);
+       intel_uncore_rmw(&dev_priv->uncore, SOUTH_CHICKEN2, 0, DPLS_EDP_PPS_FIX_DIS);
        /* The below fixes the weird display corruption, a few pixels shifted
         * downward, on (only) LVDS of some HP laptops with IVY.
         */
@@ -4231,9 +4214,7 @@ static void gen6_init_clock_gating(struct drm_i915_private *dev_priv)
 
        intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
 
-       intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2,
-                  intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2) |
-                  ILK_ELPIN_409_SELECT);
+       intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
 
        intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
                   intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
@@ -4293,14 +4274,12 @@ static void lpt_init_clock_gating(struct drm_i915_private *dev_priv)
         * disabled when not needed anymore in order to save power.
         */
        if (HAS_PCH_LPT_LP(dev_priv))
-               intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
-                          intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
-                          PCH_LP_PARTITION_LEVEL_DISABLE);
+               intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D,
+                                0, PCH_LP_PARTITION_LEVEL_DISABLE);
 
        /* WADPOClockGatingDisable:hsw */
-       intel_uncore_write(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
-                  intel_uncore_read(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A)) |
-                  TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
+       intel_uncore_rmw(&dev_priv->uncore, TRANS_CHICKEN1(PIPE_A),
+                        0, TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
 }
 
 static void lpt_suspend_hw(struct drm_i915_private *dev_priv)
@@ -4321,22 +4300,22 @@ static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
        u32 val;
 
        /* WaTempDisableDOPClkGating:bdw */
-       misccpctl = intel_uncore_rmw(&dev_priv->uncore, GEN7_MISCCPCTL,
-                                    GEN7_DOP_CLOCK_GATE_ENABLE, 0);
+       misccpctl = intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
+                                              GEN8_DOP_CLOCK_GATE_ENABLE, 0);
 
-       val = intel_uncore_read(&dev_priv->uncore, GEN8_L3SQCREG1);
+       val = intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
        val &= ~L3_PRIO_CREDITS_MASK;
        val |= L3_GENERAL_PRIO_CREDITS(general_prio_credits);
        val |= L3_HIGH_PRIO_CREDITS(high_prio_credits);
-       intel_uncore_write(&dev_priv->uncore, GEN8_L3SQCREG1, val);
+       intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_L3SQCREG1, val);
 
        /*
         * Wait at least 100 clocks before re-enabling clock gating.
         * See the definition of L3SQCREG1 in BSpec.
         */
-       intel_uncore_posting_read(&dev_priv->uncore, GEN8_L3SQCREG1);
+       intel_gt_mcr_read_any(to_gt(dev_priv), GEN8_L3SQCREG1);
        udelay(1);
-       intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, misccpctl);
+       intel_gt_mcr_multicast_write(to_gt(dev_priv), GEN8_MISCCPCTL, misccpctl);
 }
 
 static void icl_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -4359,8 +4338,7 @@ static void gen12lp_init_clock_gating(struct drm_i915_private *dev_priv)
 
        /* Wa_1409825376:tgl (pre-prod)*/
        if (IS_TGL_DISPLAY_STEP(dev_priv, STEP_A0, STEP_C0))
-               intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
-                          TGL_VRH_GATING_DIS);
+               intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, TGL_VRH_GATING_DIS);
 
        /* Wa_14013723622:tgl,rkl,dg1,adl-s */
        if (DISPLAY_VER(dev_priv) == 12)
@@ -4385,8 +4363,7 @@ static void dg1_init_clock_gating(struct drm_i915_private *dev_priv)
 
        /* Wa_1409836686:dg1[a0] */
        if (IS_DG1_GRAPHICS_STEP(dev_priv, STEP_A0, STEP_B0))
-               intel_uncore_write(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, intel_uncore_read(&dev_priv->uncore, GEN9_CLKGATE_DIS_3) |
-                          DPT_GATING_DIS);
+               intel_uncore_rmw(&dev_priv->uncore, GEN9_CLKGATE_DIS_3, 0, DPT_GATING_DIS);
 }
 
 static void xehpsdv_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -4428,8 +4405,7 @@ static void cnp_init_clock_gating(struct drm_i915_private *dev_priv)
                return;
 
        /* Display WA #1181 WaSouthDisplayDisablePWMCGEGating: cnp */
-       intel_uncore_write(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, intel_uncore_read(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D) |
-                  CNP_PWM_CGE_GATING_DISABLE);
+       intel_uncore_rmw(&dev_priv->uncore, SOUTH_DSPCLK_GATE_D, 0, CNP_PWM_CGE_GATING_DISABLE);
 }
 
 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -4438,23 +4414,20 @@ static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
        gen9_init_clock_gating(dev_priv);
 
        /* WAC6entrylatency:cfl */
-       intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
-                  FBC_LLC_FULLY_OPEN);
+       intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 
        /*
         * WaFbcTurnOffFbcWatermark:cfl
         * Display WA #0562: cfl
         */
-       intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
-                  DISP_FBC_WM_DIS);
+       intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 
        /*
         * WaFbcNukeOnHostModify:cfl
         * Display WA #0873: cfl
         */
-       intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-                          intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
-                          DPFC_NUKE_ON_ANY_MODIFICATION);
+       intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
+                        0, DPFC_NUKE_ON_ANY_MODIFICATION);
 }
 
 static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -4462,33 +4435,30 @@ static void kbl_init_clock_gating(struct drm_i915_private *dev_priv)
        gen9_init_clock_gating(dev_priv);
 
        /* WAC6entrylatency:kbl */
-       intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
-                  FBC_LLC_FULLY_OPEN);
+       intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 
        /* WaDisableSDEUnitClockGating:kbl */
        if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
-               intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
-                          GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
+               intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6,
+                                0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 
        /* WaDisableGamClockGating:kbl */
        if (IS_KBL_GRAPHICS_STEP(dev_priv, 0, STEP_C0))
-               intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
-                          GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
+               intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1,
+                                0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
 
        /*
         * WaFbcTurnOffFbcWatermark:kbl
         * Display WA #0562: kbl
         */
-       intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
-                  DISP_FBC_WM_DIS);
+       intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 
        /*
         * WaFbcNukeOnHostModify:kbl
         * Display WA #0873: kbl
         */
-       intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-                          intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
-                          DPFC_NUKE_ON_ANY_MODIFICATION);
+       intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
+                        0, DPFC_NUKE_ON_ANY_MODIFICATION);
 }
 
 static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -4496,35 +4466,30 @@ static void skl_init_clock_gating(struct drm_i915_private *dev_priv)
        gen9_init_clock_gating(dev_priv);
 
        /* WaDisableDopClockGating:skl */
-       intel_uncore_write(&dev_priv->uncore, GEN7_MISCCPCTL, intel_uncore_read(&dev_priv->uncore, GEN7_MISCCPCTL) &
-                  ~GEN7_DOP_CLOCK_GATE_ENABLE);
+       intel_gt_mcr_multicast_rmw(to_gt(dev_priv), GEN8_MISCCPCTL,
+                                  GEN8_DOP_CLOCK_GATE_ENABLE, 0);
 
        /* WAC6entrylatency:skl */
-       intel_uncore_write(&dev_priv->uncore, FBC_LLC_READ_CTRL, intel_uncore_read(&dev_priv->uncore, FBC_LLC_READ_CTRL) |
-                  FBC_LLC_FULLY_OPEN);
+       intel_uncore_rmw(&dev_priv->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 
        /*
         * WaFbcTurnOffFbcWatermark:skl
         * Display WA #0562: skl
         */
-       intel_uncore_write(&dev_priv->uncore, DISP_ARB_CTL, intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) |
-                  DISP_FBC_WM_DIS);
+       intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
 
        /*
         * WaFbcNukeOnHostModify:skl
         * Display WA #0873: skl
         */
-       intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-                          intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
-                          DPFC_NUKE_ON_ANY_MODIFICATION);
+       intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
+                        0, DPFC_NUKE_ON_ANY_MODIFICATION);
 
        /*
         * WaFbcHighMemBwCorruptionAvoidance:skl
         * Display WA #0883: skl
         */
-       intel_uncore_write(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A),
-                          intel_uncore_read(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A)) |
-                          DPFC_DISABLE_DUMMY0);
+       intel_uncore_rmw(&dev_priv->uncore, ILK_DPFC_CHICKEN(INTEL_FBC_A), 0, DPFC_DISABLE_DUMMY0);
 }
 
 static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -4532,43 +4497,37 @@ static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
        enum pipe pipe;
 
        /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
-       intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
-                  intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
-                  HSW_FBCQ_DIS);
+       intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 
        /* WaSwitchSolVfFArbitrationPriority:bdw */
-       intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
+       intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 
        /* WaPsrDPAMaskVBlankInSRD:bdw */
-       intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR1_1,
-                  intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
+       intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR1_1, 0, DPA_MASK_VBLANK_SRD);
 
        for_each_pipe(dev_priv, pipe) {
                /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
-               intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
-                          intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe)) |
-                          BDW_DPRS_MASK_VBLANK_SRD);
+               intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(pipe),
+                                0, BDW_DPRS_MASK_VBLANK_SRD);
        }
 
        /* WaVSRefCountFullforceMissDisable:bdw */
        /* WaDSRefCountFullforceMissDisable:bdw */
-       intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
-                  intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
-                  ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
+       intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
+                        GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 
        intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
                   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 
        /* WaDisableSDEUnitClockGating:bdw */
-       intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
-                  GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
+       intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 
        /* WaProgramL3SqcReg1Default:bdw */
        gen8_set_l3sqc_credits(dev_priv, 30, 2);
 
        /* WaKVMNotificationOnConfigChange:bdw */
-       intel_uncore_write(&dev_priv->uncore, CHICKEN_PAR2_1, intel_uncore_read(&dev_priv->uncore, CHICKEN_PAR2_1)
-                  | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
+       intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PAR2_1,
+                        0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
 
        lpt_init_clock_gating(dev_priv);
 
@@ -4577,24 +4536,20 @@ static void bdw_init_clock_gating(struct drm_i915_private *dev_priv)
         * Also see the CHICKEN2 write in bdw_init_workarounds() to disable DOP
         * clock gating.
         */
-       intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1,
-                  intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) | GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
+       intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
 }
 
 static void hsw_init_clock_gating(struct drm_i915_private *dev_priv)
 {
        /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
-       intel_uncore_write(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A),
-                  intel_uncore_read(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A)) |
-                  HSW_FBCQ_DIS);
+       intel_uncore_rmw(&dev_priv->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
 
        /* This is required by WaCatErrorRejectionIssue:hsw */
-       intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
-                  intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
-                  GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
+       intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
+                        0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
        /* WaSwitchSolVfFArbitrationPriority:hsw */
-       intel_uncore_write(&dev_priv->uncore, GAM_ECOCHK, intel_uncore_read(&dev_priv->uncore, GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
+       intel_uncore_rmw(&dev_priv->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 
        lpt_init_clock_gating(dev_priv);
 }
@@ -4604,9 +4559,7 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
        intel_uncore_write(&dev_priv->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
 
        /* WaFbcAsynchFlipDisableFbcQueue:ivb */
-       intel_uncore_write(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1,
-                  intel_uncore_read(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1) |
-                  ILK_FBCQ_DIS);
+       intel_uncore_rmw(&dev_priv->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
 
        /* WaDisableBackToBackFlipFix:ivb */
        intel_uncore_write(&dev_priv->uncore, IVB_CHICKEN3,
@@ -4632,9 +4585,8 @@ static void ivb_init_clock_gating(struct drm_i915_private *dev_priv)
                   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 
        /* This is required by WaCatErrorRejectionIssue:ivb */
-       intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
-                       intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
-                       GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
+       intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
+                        0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
        g4x_disable_trickle_feed(dev_priv);
 
@@ -4659,9 +4611,8 @@ static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
                   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 
        /* This is required by WaCatErrorRejectionIssue:vlv */
-       intel_uncore_write(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
-                  intel_uncore_read(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
-                  GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
+       intel_uncore_rmw(&dev_priv->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
+                        0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
        /*
         * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
@@ -4673,8 +4624,7 @@ static void vlv_init_clock_gating(struct drm_i915_private *dev_priv)
        /* WaDisableL3Bank2xClockGate:vlv
         * Disabling L3 clock gating- MMIO 940c[25] = 1
         * Set bit 25, to disable L3_BANK_2x_CLK_GATING */
-       intel_uncore_write(&dev_priv->uncore, GEN7_UCGCTL4,
-                  intel_uncore_read(&dev_priv->uncore, GEN7_UCGCTL4) | GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
+       intel_uncore_rmw(&dev_priv->uncore, GEN7_UCGCTL4, 0, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
 
        /*
         * WaDisableVLVClockGating_VBIIssue:vlv
@@ -4688,21 +4638,18 @@ static void chv_init_clock_gating(struct drm_i915_private *dev_priv)
 {
        /* WaVSRefCountFullforceMissDisable:chv */
        /* WaDSRefCountFullforceMissDisable:chv */
-       intel_uncore_write(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
-                  intel_uncore_read(&dev_priv->uncore, GEN7_FF_THREAD_MODE) &
-                  ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
+       intel_uncore_rmw(&dev_priv->uncore, GEN7_FF_THREAD_MODE,
+                        GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME, 0);
 
        /* WaDisableSemaphoreAndSyncFlipWait:chv */
        intel_uncore_write(&dev_priv->uncore, RING_PSMI_CTL(RENDER_RING_BASE),
                   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 
        /* WaDisableCSUnitClockGating:chv */
-       intel_uncore_write(&dev_priv->uncore, GEN6_UCGCTL1, intel_uncore_read(&dev_priv->uncore, GEN6_UCGCTL1) |
-                  GEN6_CSUNIT_CLOCK_GATE_DISABLE);
+       intel_uncore_rmw(&dev_priv->uncore, GEN6_UCGCTL1, 0, GEN6_CSUNIT_CLOCK_GATE_DISABLE);
 
        /* WaDisableSDEUnitClockGating:chv */
-       intel_uncore_write(&dev_priv->uncore, GEN8_UCGCTL6, intel_uncore_read(&dev_priv->uncore, GEN8_UCGCTL6) |
-                  GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
+       intel_uncore_rmw(&dev_priv->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 
        /*
         * WaProgramL3SqcReg1Default:chv