drm/i915: Do one shot unclaimed mmio detection less frequently
[linux-2.6-block.git] / drivers / gpu / drm / i915 / intel_display.c
index 79fc520b018b06ca5b23d45fa1bc6f0db4812ee9..3edf22ad7673e4ddde8caaacd5fb698f06d50ef8 100644 (file)
@@ -44,6 +44,8 @@
 #include <drm/drm_plane_helper.h>
 #include <drm/drm_rect.h>
 #include <linux/dma_remapping.h>
+#include <linux/reservation.h>
+#include <linux/dma-buf.h>
 
 /* Primary plane formats for gen <= 3 */
 static const uint32_t i8xx_primary_formats[] = {
@@ -83,8 +85,6 @@ static const uint32_t intel_cursor_formats[] = {
        DRM_FORMAT_ARGB8888,
 };
 
-static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
-
 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
                                struct intel_crtc_state *pipe_config);
 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
@@ -116,6 +116,7 @@ static void skylake_pfit_enable(struct intel_crtc *crtc);
 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force);
 static void ironlake_pfit_enable(struct intel_crtc *crtc);
 static void intel_modeset_setup_hw_state(struct drm_device *dev);
+static void intel_pre_disable_primary(struct drm_crtc *crtc);
 
 typedef struct {
        int     min, max;
@@ -185,7 +186,7 @@ int intel_hrawclk(struct drm_device *dev)
        uint32_t clkcfg;
 
        /* There is no CLKCFG reg in Valleyview. VLV hrawclk is 200 MHz */
-       if (IS_VALLEYVIEW(dev))
+       if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
                return 200;
 
        clkcfg = I915_READ(CLKCFG);
@@ -213,7 +214,7 @@ int intel_hrawclk(struct drm_device *dev)
 
 static void intel_update_czclk(struct drm_i915_private *dev_priv)
 {
-       if (!IS_VALLEYVIEW(dev_priv))
+       if (!(IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)))
                return;
 
        dev_priv->czclk_freq = vlv_get_cck_clock_hpll(dev_priv, "czclk",
@@ -714,11 +715,12 @@ static bool intel_PLL_is_valid(struct drm_device *dev,
        if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
                INTELPllInvalid("m1 out of range\n");
 
-       if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) && !IS_BROXTON(dev))
+       if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) &&
+           !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev))
                if (clock->m1 <= clock->m2)
                        INTELPllInvalid("m1 <= m2\n");
 
-       if (!IS_VALLEYVIEW(dev) && !IS_BROXTON(dev)) {
+       if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && !IS_BROXTON(dev)) {
                if (clock->p < limit->p.min || limit->p.max < clock->p)
                        INTELPllInvalid("p out of range\n");
                if (clock->m < limit->m.min || limit->m.max < clock->m)
@@ -1303,7 +1305,7 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
                    I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
                        panel_pipe = PIPE_B;
                /* XXX: else fix for eDP */
-       } else if (IS_VALLEYVIEW(dev)) {
+       } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
                /* presumably write lock depends on pipe, not port select */
                pp_reg = VLV_PIPE_PP_CONTROL(pipe);
                panel_pipe = pipe;
@@ -1421,7 +1423,7 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
                             "plane %d assertion failure, should be off on pipe %c but is still active\n",
                             sprite, pipe_name(pipe));
                }
-       } else if (IS_VALLEYVIEW(dev)) {
+       } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
                for_each_sprite(dev_priv, pipe, sprite) {
                        u32 val = I915_READ(SPCNTR(pipe, sprite));
                        I915_STATE_WARN(val & SP_ENABLE,
@@ -1604,9 +1606,6 @@ static void vlv_enable_pll(struct intel_crtc *crtc,
 
        assert_pipe_disabled(dev_priv, crtc->pipe);
 
-       /* No really, not for ILK+ */
-       BUG_ON(!IS_VALLEYVIEW(dev_priv->dev));
-
        /* PLL is protected by panel, make sure we can write it */
        if (IS_MOBILE(dev_priv->dev))
                assert_panel_unlocked(dev_priv, crtc->pipe);
@@ -1644,8 +1643,6 @@ static void chv_enable_pll(struct intel_crtc *crtc,
 
        assert_pipe_disabled(dev_priv, crtc->pipe);
 
-       BUG_ON(!IS_CHERRYVIEW(dev_priv->dev));
-
        mutex_lock(&dev_priv->sb_lock);
 
        /* Enable back the 10bit clock to display controller */
@@ -2130,7 +2127,7 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
         * need the check.
         */
        if (HAS_GMCH_DISPLAY(dev_priv->dev))
-               if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI))
+               if (crtc->config->has_dsi_encoder)
                        assert_dsi_pll_enabled(dev_priv);
                else
                        assert_pll_enabled(dev_priv, pipe);
@@ -2317,7 +2314,7 @@ static unsigned int intel_linear_alignment(struct drm_i915_private *dev_priv)
        if (INTEL_INFO(dev_priv)->gen >= 9)
                return 256 * 1024;
        else if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv) ||
-                IS_VALLEYVIEW(dev_priv))
+                IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                return 128 * 1024;
        else if (INTEL_INFO(dev_priv)->gen >= 4)
                return 4 * 1024;
@@ -2597,6 +2594,10 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
        struct drm_i915_gem_object *obj;
        struct drm_plane *primary = intel_crtc->base.primary;
        struct drm_plane_state *plane_state = primary->state;
+       struct drm_crtc_state *crtc_state = intel_crtc->base.state;
+       struct intel_plane *intel_plane = to_intel_plane(primary);
+       struct intel_plane_state *intel_state =
+               to_intel_plane_state(plane_state);
        struct drm_framebuffer *fb;
 
        if (!plane_config->fb)
@@ -2633,6 +2634,18 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
                }
        }
 
+       /*
+        * We've failed to reconstruct the BIOS FB.  Current display state
+        * indicates that the primary plane is visible, but has a NULL FB,
+        * which will lead to problems later if we don't fix it up.  The
+        * simplest solution is to just disable the primary plane now and
+        * pretend the BIOS never had it enabled.
+        */
+       to_intel_plane_state(plane_state)->visible = false;
+       crtc_state->plane_mask &= ~(1 << drm_plane_index(primary));
+       intel_pre_disable_primary(&intel_crtc->base);
+       intel_plane->disable_plane(primary, &intel_crtc->base);
+
        return;
 
 valid_fb:
@@ -2646,6 +2659,15 @@ valid_fb:
        plane_state->crtc_w = fb->width;
        plane_state->crtc_h = fb->height;
 
+       intel_state->src.x1 = plane_state->src_x;
+       intel_state->src.y1 = plane_state->src_y;
+       intel_state->src.x2 = plane_state->src_x + plane_state->src_w;
+       intel_state->src.y2 = plane_state->src_y + plane_state->src_h;
+       intel_state->dst.x1 = plane_state->crtc_x;
+       intel_state->dst.y1 = plane_state->crtc_y;
+       intel_state->dst.x2 = plane_state->crtc_x + plane_state->crtc_w;
+       intel_state->dst.y2 = plane_state->crtc_y + plane_state->crtc_h;
+
        obj = intel_fb_obj(fb);
        if (obj->tiling_mode != I915_TILING_NONE)
                dev_priv->preserve_bios_swizzle = true;
@@ -2657,36 +2679,23 @@ valid_fb:
        obj->frontbuffer_bits |= to_intel_plane(primary)->frontbuffer_bit;
 }
 
-static void i9xx_update_primary_plane(struct drm_crtc *crtc,
-                                     struct drm_framebuffer *fb,
-                                     int x, int y)
+static void i9xx_update_primary_plane(struct drm_plane *primary,
+                                     const struct intel_crtc_state *crtc_state,
+                                     const struct intel_plane_state *plane_state)
 {
-       struct drm_device *dev = crtc->dev;
+       struct drm_device *dev = primary->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
-       struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-       struct drm_plane *primary = crtc->primary;
-       bool visible = to_intel_plane_state(primary->state)->visible;
-       struct drm_i915_gem_object *obj;
+       struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
+       struct drm_framebuffer *fb = plane_state->base.fb;
+       struct drm_i915_gem_object *obj = intel_fb_obj(fb);
        int plane = intel_crtc->plane;
        unsigned long linear_offset;
+       int x = plane_state->src.x1 >> 16;
+       int y = plane_state->src.y1 >> 16;
        u32 dspcntr;
        i915_reg_t reg = DSPCNTR(plane);
        int pixel_size;
 
-       if (!visible || !fb) {
-               I915_WRITE(reg, 0);
-               if (INTEL_INFO(dev)->gen >= 4)
-                       I915_WRITE(DSPSURF(plane), 0);
-               else
-                       I915_WRITE(DSPADDR(plane), 0);
-               POSTING_READ(reg);
-               return;
-       }
-
-       obj = intel_fb_obj(fb);
-       if (WARN_ON(obj == NULL))
-               return;
-
        pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
 
        dspcntr = DISPPLANE_GAMMA_ENABLE;
@@ -2701,13 +2710,13 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc,
                 * which should always be the user's requested size.
                 */
                I915_WRITE(DSPSIZE(plane),
-                          ((intel_crtc->config->pipe_src_h - 1) << 16) |
-                          (intel_crtc->config->pipe_src_w - 1));
+                          ((crtc_state->pipe_src_h - 1) << 16) |
+                          (crtc_state->pipe_src_w - 1));
                I915_WRITE(DSPPOS(plane), 0);
        } else if (IS_CHERRYVIEW(dev) && plane == PLANE_B) {
                I915_WRITE(PRIMSIZE(plane),
-                          ((intel_crtc->config->pipe_src_h - 1) << 16) |
-                          (intel_crtc->config->pipe_src_w - 1));
+                          ((crtc_state->pipe_src_h - 1) << 16) |
+                          (crtc_state->pipe_src_w - 1));
                I915_WRITE(PRIMPOS(plane), 0);
                I915_WRITE(PRIMCNSTALPHA(plane), 0);
        }
@@ -2758,17 +2767,17 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc,
                intel_crtc->dspaddr_offset = linear_offset;
        }
 
-       if (crtc->primary->state->rotation == BIT(DRM_ROTATE_180)) {
+       if (plane_state->base.rotation == BIT(DRM_ROTATE_180)) {
                dspcntr |= DISPPLANE_ROTATE_180;
 
-               x += (intel_crtc->config->pipe_src_w - 1);
-               y += (intel_crtc->config->pipe_src_h - 1);
+               x += (crtc_state->pipe_src_w - 1);
+               y += (crtc_state->pipe_src_h - 1);
 
                /* Finding the last pixel of the last line of the display
                data and adding to linear_offset*/
                linear_offset +=
-                       (intel_crtc->config->pipe_src_h - 1) * fb->pitches[0] +
-                       (intel_crtc->config->pipe_src_w - 1) * pixel_size;
+                       (crtc_state->pipe_src_h - 1) * fb->pitches[0] +
+                       (crtc_state->pipe_src_w - 1) * pixel_size;
        }
 
        intel_crtc->adjusted_x = x;
@@ -2787,37 +2796,40 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc,
        POSTING_READ(reg);
 }
 
-static void ironlake_update_primary_plane(struct drm_crtc *crtc,
-                                         struct drm_framebuffer *fb,
-                                         int x, int y)
+static void i9xx_disable_primary_plane(struct drm_plane *primary,
+                                      struct drm_crtc *crtc)
 {
        struct drm_device *dev = crtc->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-       struct drm_plane *primary = crtc->primary;
-       bool visible = to_intel_plane_state(primary->state)->visible;
-       struct drm_i915_gem_object *obj;
        int plane = intel_crtc->plane;
-       unsigned long linear_offset;
-       u32 dspcntr;
-       i915_reg_t reg = DSPCNTR(plane);
-       int pixel_size;
 
-       if (!visible || !fb) {
-               I915_WRITE(reg, 0);
+       I915_WRITE(DSPCNTR(plane), 0);
+       if (INTEL_INFO(dev_priv)->gen >= 4)
                I915_WRITE(DSPSURF(plane), 0);
-               POSTING_READ(reg);
-               return;
-       }
-
-       obj = intel_fb_obj(fb);
-       if (WARN_ON(obj == NULL))
-               return;
+       else
+               I915_WRITE(DSPADDR(plane), 0);
+       POSTING_READ(DSPCNTR(plane));
+}
 
-       pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
+static void ironlake_update_primary_plane(struct drm_plane *primary,
+                                         const struct intel_crtc_state *crtc_state,
+                                         const struct intel_plane_state *plane_state)
+{
+       struct drm_device *dev = primary->dev;
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
+       struct drm_framebuffer *fb = plane_state->base.fb;
+       struct drm_i915_gem_object *obj = intel_fb_obj(fb);
+       int plane = intel_crtc->plane;
+       unsigned long linear_offset;
+       u32 dspcntr;
+       i915_reg_t reg = DSPCNTR(plane);
+       int pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
+       int x = plane_state->src.x1 >> 16;
+       int y = plane_state->src.y1 >> 16;
 
        dspcntr = DISPPLANE_GAMMA_ENABLE;
-
        dspcntr |= DISPLAY_PLANE_ENABLE;
 
        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
@@ -2859,18 +2871,18 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc,
                                               pixel_size,
                                               fb->pitches[0]);
        linear_offset -= intel_crtc->dspaddr_offset;
-       if (crtc->primary->state->rotation == BIT(DRM_ROTATE_180)) {
+       if (plane_state->base.rotation == BIT(DRM_ROTATE_180)) {
                dspcntr |= DISPPLANE_ROTATE_180;
 
                if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
-                       x += (intel_crtc->config->pipe_src_w - 1);
-                       y += (intel_crtc->config->pipe_src_h - 1);
+                       x += (crtc_state->pipe_src_w - 1);
+                       y += (crtc_state->pipe_src_h - 1);
 
                        /* Finding the last pixel of the last line of the display
                        data and adding to linear_offset*/
                        linear_offset +=
-                               (intel_crtc->config->pipe_src_h - 1) * fb->pitches[0] +
-                               (intel_crtc->config->pipe_src_w - 1) * pixel_size;
+                               (crtc_state->pipe_src_h - 1) * fb->pitches[0] +
+                               (crtc_state->pipe_src_w - 1) * pixel_size;
                }
        }
 
@@ -3061,36 +3073,30 @@ u32 skl_plane_ctl_rotation(unsigned int rotation)
        return 0;
 }
 
-static void skylake_update_primary_plane(struct drm_crtc *crtc,
-                                        struct drm_framebuffer *fb,
-                                        int x, int y)
+static void skylake_update_primary_plane(struct drm_plane *plane,
+                                        const struct intel_crtc_state *crtc_state,
+                                        const struct intel_plane_state *plane_state)
 {
-       struct drm_device *dev = crtc->dev;
+       struct drm_device *dev = plane->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
-       struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-       struct drm_plane *plane = crtc->primary;
-       bool visible = to_intel_plane_state(plane->state)->visible;
-       struct drm_i915_gem_object *obj;
+       struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
+       struct drm_framebuffer *fb = plane_state->base.fb;
+       struct drm_i915_gem_object *obj = intel_fb_obj(fb);
        int pipe = intel_crtc->pipe;
        u32 plane_ctl, stride_div, stride;
        u32 tile_height, plane_offset, plane_size;
-       unsigned int rotation;
+       unsigned int rotation = plane_state->base.rotation;
        int x_offset, y_offset;
        u32 surf_addr;
-       struct intel_crtc_state *crtc_state = intel_crtc->config;
-       struct intel_plane_state *plane_state;
-       int src_x = 0, src_y = 0, src_w = 0, src_h = 0;
-       int dst_x = 0, dst_y = 0, dst_w = 0, dst_h = 0;
-       int scaler_id = -1;
-
-       plane_state = to_intel_plane_state(plane->state);
-
-       if (!visible || !fb) {
-               I915_WRITE(PLANE_CTL(pipe, 0), 0);
-               I915_WRITE(PLANE_SURF(pipe, 0), 0);
-               POSTING_READ(PLANE_CTL(pipe, 0));
-               return;
-       }
+       int scaler_id = plane_state->scaler_id;
+       int src_x = plane_state->src.x1 >> 16;
+       int src_y = plane_state->src.y1 >> 16;
+       int src_w = drm_rect_width(&plane_state->src) >> 16;
+       int src_h = drm_rect_height(&plane_state->src) >> 16;
+       int dst_x = plane_state->dst.x1;
+       int dst_y = plane_state->dst.y1;
+       int dst_w = drm_rect_width(&plane_state->dst);
+       int dst_h = drm_rect_height(&plane_state->dst);
 
        plane_ctl = PLANE_CTL_ENABLE |
                    PLANE_CTL_PIPE_GAMMA_ENABLE |
@@ -3099,41 +3105,26 @@ static void skylake_update_primary_plane(struct drm_crtc *crtc,
        plane_ctl |= skl_plane_ctl_format(fb->pixel_format);
        plane_ctl |= skl_plane_ctl_tiling(fb->modifier[0]);
        plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
-
-       rotation = plane->state->rotation;
        plane_ctl |= skl_plane_ctl_rotation(rotation);
 
-       obj = intel_fb_obj(fb);
        stride_div = intel_fb_stride_alignment(dev, fb->modifier[0],
                                               fb->pixel_format);
        surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj, 0);
 
        WARN_ON(drm_rect_width(&plane_state->src) == 0);
 
-       scaler_id = plane_state->scaler_id;
-       src_x = plane_state->src.x1 >> 16;
-       src_y = plane_state->src.y1 >> 16;
-       src_w = drm_rect_width(&plane_state->src) >> 16;
-       src_h = drm_rect_height(&plane_state->src) >> 16;
-       dst_x = plane_state->dst.x1;
-       dst_y = plane_state->dst.y1;
-       dst_w = drm_rect_width(&plane_state->dst);
-       dst_h = drm_rect_height(&plane_state->dst);
-
-       WARN_ON(x != src_x || y != src_y);
-
        if (intel_rotation_90_or_270(rotation)) {
                /* stride = Surface height in tiles */
                tile_height = intel_tile_height(dev, fb->pixel_format,
                                                fb->modifier[0], 0);
                stride = DIV_ROUND_UP(fb->height, tile_height);
-               x_offset = stride * tile_height - y - src_h;
-               y_offset = x;
+               x_offset = stride * tile_height - src_y - src_h;
+               y_offset = src_x;
                plane_size = (src_w - 1) << 16 | (src_h - 1);
        } else {
                stride = fb->pitches[0] / stride_div;
-               x_offset = x;
-               y_offset = y;
+               x_offset = src_x;
+               y_offset = src_y;
                plane_size = (src_h - 1) << 16 | (src_w - 1);
        }
        plane_offset = y_offset << 16 | x_offset;
@@ -3166,20 +3157,30 @@ static void skylake_update_primary_plane(struct drm_crtc *crtc,
        POSTING_READ(PLANE_SURF(pipe, 0));
 }
 
-/* Assume fb object is pinned & idle & fenced and just update base pointers */
-static int
-intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
-                          int x, int y, enum mode_set_atomic state)
+static void skylake_disable_primary_plane(struct drm_plane *primary,
+                                         struct drm_crtc *crtc)
 {
        struct drm_device *dev = crtc->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
+       int pipe = to_intel_crtc(crtc)->pipe;
 
-       if (dev_priv->fbc.disable_fbc)
-               dev_priv->fbc.disable_fbc(dev_priv);
+       if (dev_priv->fbc.deactivate)
+               dev_priv->fbc.deactivate(dev_priv);
 
-       dev_priv->display.update_primary_plane(crtc, fb, x, y);
+       I915_WRITE(PLANE_CTL(pipe, 0), 0);
+       I915_WRITE(PLANE_SURF(pipe, 0), 0);
+       POSTING_READ(PLANE_SURF(pipe, 0));
+}
 
-       return 0;
+/* Assume fb object is pinned & idle & fenced and just update base pointers */
+static int
+intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
+                          int x, int y, enum mode_set_atomic state)
+{
+       /* Support for kgdboc is disabled, this needs a major rework. */
+       DRM_ERROR("legacy panic handler not supported any more.\n");
+
+       return -ENODEV;
 }
 
 static void intel_complete_page_flips(struct drm_device *dev)
@@ -3206,8 +3207,10 @@ static void intel_update_primary_planes(struct drm_device *dev)
                drm_modeset_lock_crtc(crtc, &plane->base);
                plane_state = to_intel_plane_state(plane->base.state);
 
-               if (crtc->state->active && plane_state->base.fb)
-                       plane->commit_plane(&plane->base, plane_state);
+               if (plane_state->visible)
+                       plane->update_plane(&plane->base,
+                                           to_intel_crtc_state(crtc->state),
+                                           plane_state);
 
                drm_modeset_unlock_crtc(crtc);
        }
@@ -3938,6 +3941,21 @@ static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
        return 0;
 }
 
+static void lpt_disable_iclkip(struct drm_i915_private *dev_priv)
+{
+       u32 temp;
+
+       I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
+
+       mutex_lock(&dev_priv->sb_lock);
+
+       temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
+       temp |= SBI_SSCCTL_DISABLE;
+       intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
+
+       mutex_unlock(&dev_priv->sb_lock);
+}
+
 /* Program iCLKIP clock to the desired frequency */
 static void lpt_program_iclkip(struct drm_crtc *crtc)
 {
@@ -3947,18 +3965,7 @@ static void lpt_program_iclkip(struct drm_crtc *crtc)
        u32 divsel, phaseinc, auxdiv, phasedir = 0;
        u32 temp;
 
-       mutex_lock(&dev_priv->sb_lock);
-
-       /* It is necessary to ungate the pixclk gate prior to programming
-        * the divisors, and gate it back when it is done.
-        */
-       I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
-
-       /* Disable SSCCTL */
-       intel_sbi_write(dev_priv, SBI_SSCCTL6,
-                       intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK) |
-                               SBI_SSCCTL_DISABLE,
-                       SBI_ICLK);
+       lpt_disable_iclkip(dev_priv);
 
        /* 20MHz is a corner case which is out of range for the 7-bit divisor */
        if (clock == 20000) {
@@ -3976,7 +3983,7 @@ static void lpt_program_iclkip(struct drm_crtc *crtc)
                u32 iclk_pi_range = 64;
                u32 desired_divisor, msb_divisor_value, pi_value;
 
-               desired_divisor = (iclk_virtual_root_freq / clock);
+               desired_divisor = DIV_ROUND_CLOSEST(iclk_virtual_root_freq, clock);
                msb_divisor_value = desired_divisor / iclk_pi_range;
                pi_value = desired_divisor % iclk_pi_range;
 
@@ -3998,6 +4005,8 @@ static void lpt_program_iclkip(struct drm_crtc *crtc)
                        phasedir,
                        phaseinc);
 
+       mutex_lock(&dev_priv->sb_lock);
+
        /* Program SSCDIVINTPHASE6 */
        temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
        temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
@@ -4019,12 +4028,12 @@ static void lpt_program_iclkip(struct drm_crtc *crtc)
        temp &= ~SBI_SSCCTL_DISABLE;
        intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
 
+       mutex_unlock(&dev_priv->sb_lock);
+
        /* Wait for initialization time */
        udelay(24);
 
        I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
-
-       mutex_unlock(&dev_priv->sb_lock);
 }
 
 static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
@@ -4636,7 +4645,7 @@ static void intel_crtc_load_lut(struct drm_crtc *crtc)
                return;
 
        if (HAS_GMCH_DISPLAY(dev_priv->dev)) {
-               if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI))
+               if (intel_crtc->config->has_dsi_encoder)
                        assert_dsi_pll_enabled(dev_priv);
                else
                        assert_pll_enabled(dev_priv, pipe);
@@ -4706,14 +4715,6 @@ intel_post_enable_primary(struct drm_crtc *crtc)
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
        int pipe = intel_crtc->pipe;
 
-       /*
-        * BDW signals flip done immediately if the plane
-        * is disabled, even if the plane enable is already
-        * armed to occur at the next vblank :(
-        */
-       if (IS_BROADWELL(dev))
-               intel_wait_for_vblank(dev, pipe);
-
        /*
         * FIXME IPS should be fine as long as one plane is
         * enabled, but in practice it seems to have problems
@@ -4791,22 +4792,22 @@ intel_pre_disable_primary(struct drm_crtc *crtc)
 static void intel_post_plane_update(struct intel_crtc *crtc)
 {
        struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
+       struct intel_crtc_state *pipe_config =
+               to_intel_crtc_state(crtc->base.state);
        struct drm_device *dev = crtc->base.dev;
-       struct drm_i915_private *dev_priv = dev->dev_private;
 
        if (atomic->wait_vblank)
                intel_wait_for_vblank(dev, crtc->pipe);
 
        intel_frontbuffer_flip(dev, atomic->fb_bits);
 
-       if (atomic->disable_cxsr)
-               crtc->wm.cxsr_allowed = true;
+       crtc->wm.cxsr_allowed = true;
 
-       if (crtc->atomic.update_wm_post)
+       if (pipe_config->wm_changed && pipe_config->base.active)
                intel_update_watermarks(&crtc->base);
 
        if (atomic->update_fbc)
-               intel_fbc_update(dev_priv);
+               intel_fbc_update(crtc);
 
        if (atomic->post_enable_primary)
                intel_post_enable_primary(&crtc->base);
@@ -4819,9 +4820,11 @@ static void intel_pre_plane_update(struct intel_crtc *crtc)
        struct drm_device *dev = crtc->base.dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
+       struct intel_crtc_state *pipe_config =
+               to_intel_crtc_state(crtc->base.state);
 
        if (atomic->disable_fbc)
-               intel_fbc_disable_crtc(crtc);
+               intel_fbc_deactivate(crtc);
 
        if (crtc->atomic.disable_ips)
                hsw_disable_ips(crtc);
@@ -4829,10 +4832,48 @@ static void intel_pre_plane_update(struct intel_crtc *crtc)
        if (atomic->pre_disable_primary)
                intel_pre_disable_primary(&crtc->base);
 
-       if (atomic->disable_cxsr) {
+       if (pipe_config->disable_cxsr) {
                crtc->wm.cxsr_allowed = false;
                intel_set_memory_cxsr(dev_priv, false);
        }
+
+       /*
+        * IVB workaround: must disable low power watermarks for at least
+        * one frame before enabling scaling.  LP watermarks can be re-enabled
+        * when scaling is disabled.
+        *
+        * WaCxSRDisabledForSpriteScaling:ivb
+        */
+       if (pipe_config->disable_lp_wm) {
+               ilk_disable_lp_wm(dev);
+               intel_wait_for_vblank(dev, crtc->pipe);
+       }
+
+       /*
+        * If we're doing a modeset, we're done.  No need to do any pre-vblank
+        * watermark programming here.
+        */
+       if (needs_modeset(&pipe_config->base))
+               return;
+
+       /*
+        * For platforms that support atomic watermarks, program the
+        * 'intermediate' watermarks immediately.  On pre-gen9 platforms, these
+        * will be the intermediate values that are safe for both pre- and
+        * post- vblank; when vblank happens, the 'active' values will be set
+        * to the final 'target' values and we'll do this again to get the
+        * optimal watermarks.  For gen9+ platforms, the values we program here
+        * will be the final target values which will get automatically latched
+        * at vblank time; no further programming will be necessary.
+        *
+        * If a platform hasn't been transitioned to atomic watermarks yet,
+        * we'll continue to update watermarks the old way, if flags tell
+        * us to.
+        */
+       if (dev_priv->display.initial_watermarks != NULL)
+               dev_priv->display.initial_watermarks(pipe_config);
+       else if (pipe_config->wm_changed)
+               intel_update_watermarks(&crtc->base);
 }
 
 static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask)
@@ -4929,6 +4970,8 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
        if (intel_crtc->config->has_pch_encoder)
                intel_wait_for_vblank(dev, pipe);
        intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
+
+       intel_fbc_enable(intel_crtc);
 }
 
 /* IPS only exists on ULT machines and is tied to pipe A. */
@@ -4946,7 +4989,6 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
        int pipe = intel_crtc->pipe, hsw_workaround_pipe;
        struct intel_crtc_state *pipe_config =
                to_intel_crtc_state(crtc->state);
-       bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
 
        if (WARN_ON(intel_crtc->active))
                return;
@@ -4979,10 +5021,12 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 
        intel_crtc->active = true;
 
-       intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
+       if (intel_crtc->config->has_pch_encoder)
+               intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
+       else
+               intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
+
        for_each_encoder_on_crtc(dev, crtc, encoder) {
-               if (encoder->pre_pll_enable)
-                       encoder->pre_pll_enable(encoder);
                if (encoder->pre_enable)
                        encoder->pre_enable(encoder);
        }
@@ -4990,7 +5034,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
        if (intel_crtc->config->has_pch_encoder)
                dev_priv->display.fdi_link_train(crtc);
 
-       if (!is_dsi)
+       if (!intel_crtc->config->has_dsi_encoder)
                intel_ddi_enable_pipe_clock(intel_crtc);
 
        if (INTEL_INFO(dev)->gen >= 9)
@@ -5005,7 +5049,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
        intel_crtc_load_lut(crtc);
 
        intel_ddi_set_pipe_settings(crtc);
-       if (!is_dsi)
+       if (!intel_crtc->config->has_dsi_encoder)
                intel_ddi_enable_transcoder_func(crtc);
 
        intel_update_watermarks(crtc);
@@ -5014,7 +5058,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
        if (intel_crtc->config->has_pch_encoder)
                lpt_pch_enable(crtc);
 
-       if (intel_crtc->config->dp_encoder_is_mst && !is_dsi)
+       if (intel_crtc->config->dp_encoder_is_mst)
                intel_ddi_set_vc_payload_alloc(crtc, true);
 
        assert_vblank_disabled(crtc);
@@ -5025,9 +5069,13 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
                intel_opregion_notify_encoder(encoder, true);
        }
 
-       if (intel_crtc->config->has_pch_encoder)
+       if (intel_crtc->config->has_pch_encoder) {
+               intel_wait_for_vblank(dev, pipe);
+               intel_wait_for_vblank(dev, pipe);
+               intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
                intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
                                                      true);
+       }
 
        /* If we change the relative order between pipe/planes enabling, we need
         * to change the workaround. */
@@ -5036,6 +5084,8 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
                intel_wait_for_vblank(dev, hsw_workaround_pipe);
                intel_wait_for_vblank(dev, hsw_workaround_pipe);
        }
+
+       intel_fbc_enable(intel_crtc);
 }
 
 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
@@ -5116,6 +5166,8 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
        }
 
        intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
+
+       intel_fbc_disable_crtc(intel_crtc);
 }
 
 static void haswell_crtc_disable(struct drm_crtc *crtc)
@@ -5125,7 +5177,6 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
        struct intel_encoder *encoder;
        enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
-       bool is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
 
        if (intel_crtc->config->has_pch_encoder)
                intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
@@ -5144,7 +5195,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
        if (intel_crtc->config->dp_encoder_is_mst)
                intel_ddi_set_vc_payload_alloc(crtc, false);
 
-       if (!is_dsi)
+       if (!intel_crtc->config->has_dsi_encoder)
                intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
 
        if (INTEL_INFO(dev)->gen >= 9)
@@ -5152,21 +5203,23 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
        else
                ironlake_pfit_disable(intel_crtc, false);
 
-       if (!is_dsi)
+       if (!intel_crtc->config->has_dsi_encoder)
                intel_ddi_disable_pipe_clock(intel_crtc);
 
-       if (intel_crtc->config->has_pch_encoder) {
-               lpt_disable_pch_transcoder(dev_priv);
-               intel_ddi_fdi_disable(crtc);
-       }
-
        for_each_encoder_on_crtc(dev, crtc, encoder)
                if (encoder->post_disable)
                        encoder->post_disable(encoder);
 
-       if (intel_crtc->config->has_pch_encoder)
+       if (intel_crtc->config->has_pch_encoder) {
+               lpt_disable_pch_transcoder(dev_priv);
+               lpt_disable_iclkip(dev_priv);
+               intel_ddi_fdi_disable(crtc);
+
                intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
                                                      true);
+       }
+
+       intel_fbc_disable_crtc(intel_crtc);
 }
 
 static void i9xx_pfit_enable(struct intel_crtc *crtc)
@@ -5342,6 +5395,7 @@ static void modeset_put_power_domains(struct drm_i915_private *dev_priv,
 
 static void modeset_update_crtc_power_domains(struct drm_atomic_state *state)
 {
+       struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
        struct drm_device *dev = state->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        unsigned long put_domains[I915_MAX_PIPES] = {};
@@ -5355,13 +5409,9 @@ static void modeset_update_crtc_power_domains(struct drm_atomic_state *state)
                                modeset_get_crtc_power_domains(crtc);
        }
 
-       if (dev_priv->display.modeset_commit_cdclk) {
-               unsigned int cdclk = to_intel_atomic_state(state)->cdclk;
-
-               if (cdclk != dev_priv->cdclk_freq &&
-                   !WARN_ON(!state->allow_modeset))
-                       dev_priv->display.modeset_commit_cdclk(state);
-       }
+       if (dev_priv->display.modeset_commit_cdclk &&
+           intel_state->dev_cdclk != dev_priv->cdclk_freq)
+               dev_priv->display.modeset_commit_cdclk(state);
 
        for (i = 0; i < I915_MAX_PIPES; i++)
                if (put_domains[i])
@@ -5444,7 +5494,7 @@ static void intel_update_cdclk(struct drm_device *dev)
         * BSpec erroneously claims we should aim for 4MHz, but
         * in fact 1MHz is the correct frequency.
         */
-       if (IS_VALLEYVIEW(dev)) {
+       if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
                /*
                 * Program the gmbus_freq based on the cdclk frequency.
                 * BSpec erroneously claims we should aim for 4MHz, but
@@ -6035,22 +6085,31 @@ static int broxton_calc_cdclk(struct drm_i915_private *dev_priv,
 static int intel_mode_max_pixclk(struct drm_device *dev,
                                 struct drm_atomic_state *state)
 {
-       struct intel_crtc *intel_crtc;
-       struct intel_crtc_state *crtc_state;
-       int max_pixclk = 0;
+       struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+       struct drm_i915_private *dev_priv = dev->dev_private;
+       struct drm_crtc *crtc;
+       struct drm_crtc_state *crtc_state;
+       unsigned max_pixclk = 0, i;
+       enum pipe pipe;
 
-       for_each_intel_crtc(dev, intel_crtc) {
-               crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
-               if (IS_ERR(crtc_state))
-                       return PTR_ERR(crtc_state);
+       memcpy(intel_state->min_pixclk, dev_priv->min_pixclk,
+              sizeof(intel_state->min_pixclk));
 
-               if (!crtc_state->base.enable)
-                       continue;
+       for_each_crtc_in_state(state, crtc, crtc_state, i) {
+               int pixclk = 0;
+
+               if (crtc_state->enable)
+                       pixclk = crtc_state->adjusted_mode.crtc_clock;
 
-               max_pixclk = max(max_pixclk,
-                                crtc_state->base.adjusted_mode.crtc_clock);
+               intel_state->min_pixclk[i] = pixclk;
        }
 
+       if (!intel_state->active_crtcs)
+               return 0;
+
+       for_each_pipe(dev_priv, pipe)
+               max_pixclk = max(intel_state->min_pixclk[pipe], max_pixclk);
+
        return max_pixclk;
 }
 
@@ -6059,13 +6118,18 @@ static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state)
        struct drm_device *dev = state->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        int max_pixclk = intel_mode_max_pixclk(dev, state);
+       struct intel_atomic_state *intel_state =
+               to_intel_atomic_state(state);
 
        if (max_pixclk < 0)
                return max_pixclk;
 
-       to_intel_atomic_state(state)->cdclk =
+       intel_state->cdclk = intel_state->dev_cdclk =
                valleyview_calc_cdclk(dev_priv, max_pixclk);
 
+       if (!intel_state->active_crtcs)
+               intel_state->dev_cdclk = valleyview_calc_cdclk(dev_priv, 0);
+
        return 0;
 }
 
@@ -6074,13 +6138,18 @@ static int broxton_modeset_calc_cdclk(struct drm_atomic_state *state)
        struct drm_device *dev = state->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        int max_pixclk = intel_mode_max_pixclk(dev, state);
+       struct intel_atomic_state *intel_state =
+               to_intel_atomic_state(state);
 
        if (max_pixclk < 0)
                return max_pixclk;
 
-       to_intel_atomic_state(state)->cdclk =
+       intel_state->cdclk = intel_state->dev_cdclk =
                broxton_calc_cdclk(dev_priv, max_pixclk);
 
+       if (!intel_state->active_crtcs)
+               intel_state->dev_cdclk = broxton_calc_cdclk(dev_priv, 0);
+
        return 0;
 }
 
@@ -6123,8 +6192,10 @@ static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state)
 {
        struct drm_device *dev = old_state->dev;
-       unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
        struct drm_i915_private *dev_priv = dev->dev_private;
+       struct intel_atomic_state *old_intel_state =
+               to_intel_atomic_state(old_state);
+       unsigned req_cdclk = old_intel_state->dev_cdclk;
 
        /*
         * FIXME: We can end up here with all power domains off, yet
@@ -6154,13 +6225,10 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
        struct intel_encoder *encoder;
        int pipe = intel_crtc->pipe;
-       bool is_dsi;
 
        if (WARN_ON(intel_crtc->active))
                return;
 
-       is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
-
        if (intel_crtc->config->has_dp_encoder)
                intel_dp_set_m_n(intel_crtc, M1_N1);
 
@@ -6183,7 +6251,7 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
                if (encoder->pre_pll_enable)
                        encoder->pre_pll_enable(encoder);
 
-       if (!is_dsi) {
+       if (!intel_crtc->config->has_dsi_encoder) {
                if (IS_CHERRYVIEW(dev)) {
                        chv_prepare_pll(intel_crtc, intel_crtc->config);
                        chv_enable_pll(intel_crtc, intel_crtc->config);
@@ -6262,6 +6330,8 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 
        for_each_encoder_on_crtc(dev, crtc, encoder)
                encoder->enable(encoder);
+
+       intel_fbc_enable(intel_crtc);
 }
 
 static void i9xx_pfit_disable(struct intel_crtc *crtc)
@@ -6309,7 +6379,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
                if (encoder->post_disable)
                        encoder->post_disable(encoder);
 
-       if (!intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI)) {
+       if (!intel_crtc->config->has_dsi_encoder) {
                if (IS_CHERRYVIEW(dev))
                        chv_disable_pll(dev_priv, pipe);
                else if (IS_VALLEYVIEW(dev))
@@ -6324,6 +6394,8 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 
        if (!IS_GEN2(dev))
                intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
+
+       intel_fbc_disable_crtc(intel_crtc);
 }
 
 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
@@ -6340,9 +6412,11 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
                WARN_ON(intel_crtc->unpin_work);
 
                intel_pre_disable_primary(crtc);
+
+               intel_crtc_disable_planes(crtc, 1 << drm_plane_index(crtc->primary));
+               to_intel_plane_state(crtc->primary->state)->visible = false;
        }
 
-       intel_crtc_disable_planes(crtc, crtc->state->plane_mask);
        dev_priv->display.crtc_disable(crtc);
        intel_crtc->active = false;
        intel_update_watermarks(crtc);
@@ -6352,6 +6426,9 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
        for_each_power_domain(domain, domains)
                intel_display_power_put(dev_priv, domain);
        intel_crtc->enabled_power_domains = 0;
+
+       dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
+       dev_priv->min_pixclk[intel_crtc->pipe] = 0;
 }
 
 /*
@@ -7168,7 +7245,7 @@ static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state,
 
        WARN_ON(!crtc_state->base.state);
 
-       if (IS_VALLEYVIEW(dev) || IS_BROXTON(dev)) {
+       if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev) || IS_BROXTON(dev)) {
                refclk = 100000;
        } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
            intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
@@ -7867,7 +7944,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
                pipeconf |= PIPECONF_DOUBLE_WIDE;
 
        /* only g4x and later have fancy bpc/dither controls */
-       if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
+       if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
                /* Bspec claims that we can't use dithering for 30bpp pipes. */
                if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30)
                        pipeconf |= PIPECONF_DITHER_EN |
@@ -7907,7 +7984,8 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
        } else
                pipeconf |= PIPECONF_PROGRESSIVE;
 
-       if (IS_VALLEYVIEW(dev) && intel_crtc->config->limited_color_range)
+       if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+            intel_crtc->config->limited_color_range)
                pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
 
        I915_WRITE(PIPECONF(intel_crtc->pipe), pipeconf);
@@ -7922,8 +8000,6 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
        int refclk, num_connectors = 0;
        intel_clock_t clock;
        bool ok;
-       bool is_dsi = false;
-       struct intel_encoder *encoder;
        const intel_limit_t *limit;
        struct drm_atomic_state *state = crtc_state->base.state;
        struct drm_connector *connector;
@@ -7933,26 +8009,14 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
        memset(&crtc_state->dpll_hw_state, 0,
               sizeof(crtc_state->dpll_hw_state));
 
-       for_each_connector_in_state(state, connector, connector_state, i) {
-               if (connector_state->crtc != &crtc->base)
-                       continue;
-
-               encoder = to_intel_encoder(connector_state->best_encoder);
-
-               switch (encoder->type) {
-               case INTEL_OUTPUT_DSI:
-                       is_dsi = true;
-                       break;
-               default:
-                       break;
-               }
+       if (crtc_state->has_dsi_encoder)
+               return 0;
 
-               num_connectors++;
+       for_each_connector_in_state(state, connector, connector_state, i) {
+               if (connector_state->crtc == &crtc->base)
+                       num_connectors++;
        }
 
-       if (is_dsi)
-               return 0;
-
        if (!crtc_state->clock_set) {
                refclk = i9xx_get_refclk(crtc_state, num_connectors);
 
@@ -8168,7 +8232,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
        if (!(tmp & PIPECONF_ENABLE))
                return false;
 
-       if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
+       if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
                switch (tmp & PIPECONF_BPC_MASK) {
                case PIPECONF_6BPC:
                        pipe_config->pipe_bpp = 18;
@@ -8184,7 +8248,8 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
                }
        }
 
-       if (IS_VALLEYVIEW(dev) && (tmp & PIPECONF_COLOR_RANGE_SELECT))
+       if ((IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) &&
+           (tmp & PIPECONF_COLOR_RANGE_SELECT))
                pipe_config->limited_color_range = true;
 
        if (INTEL_INFO(dev)->gen < 4)
@@ -8212,7 +8277,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
                pipe_config->pixel_multiplier = 1;
        }
        pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
-       if (!IS_VALLEYVIEW(dev)) {
+       if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
                /*
                 * DPLL_DVO_2X_MODE must be enabled for both DPLLs
                 * on 830. Filter it out here so that we don't
@@ -8564,6 +8629,67 @@ static void lpt_disable_clkout_dp(struct drm_device *dev)
        mutex_unlock(&dev_priv->sb_lock);
 }
 
+#define BEND_IDX(steps) ((50 + (steps)) / 5)
+
+static const uint16_t sscdivintphase[] = {
+       [BEND_IDX( 50)] = 0x3B23,
+       [BEND_IDX( 45)] = 0x3B23,
+       [BEND_IDX( 40)] = 0x3C23,
+       [BEND_IDX( 35)] = 0x3C23,
+       [BEND_IDX( 30)] = 0x3D23,
+       [BEND_IDX( 25)] = 0x3D23,
+       [BEND_IDX( 20)] = 0x3E23,
+       [BEND_IDX( 15)] = 0x3E23,
+       [BEND_IDX( 10)] = 0x3F23,
+       [BEND_IDX(  5)] = 0x3F23,
+       [BEND_IDX(  0)] = 0x0025,
+       [BEND_IDX( -5)] = 0x0025,
+       [BEND_IDX(-10)] = 0x0125,
+       [BEND_IDX(-15)] = 0x0125,
+       [BEND_IDX(-20)] = 0x0225,
+       [BEND_IDX(-25)] = 0x0225,
+       [BEND_IDX(-30)] = 0x0325,
+       [BEND_IDX(-35)] = 0x0325,
+       [BEND_IDX(-40)] = 0x0425,
+       [BEND_IDX(-45)] = 0x0425,
+       [BEND_IDX(-50)] = 0x0525,
+};
+
+/*
+ * Bend CLKOUT_DP
+ * steps -50 to 50 inclusive, in steps of 5
+ * < 0 slow down the clock, > 0 speed up the clock, 0 == no bend (135MHz)
+ * change in clock period = -(steps / 10) * 5.787 ps
+ */
+static void lpt_bend_clkout_dp(struct drm_i915_private *dev_priv, int steps)
+{
+       uint32_t tmp;
+       int idx = BEND_IDX(steps);
+
+       if (WARN_ON(steps % 5 != 0))
+               return;
+
+       if (WARN_ON(idx >= ARRAY_SIZE(sscdivintphase)))
+               return;
+
+       mutex_lock(&dev_priv->sb_lock);
+
+       if (steps % 10 != 0)
+               tmp = 0xAAAAAAAB;
+       else
+               tmp = 0x00000000;
+       intel_sbi_write(dev_priv, SBI_SSCDITHPHASE, tmp, SBI_ICLK);
+
+       tmp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE, SBI_ICLK);
+       tmp &= 0xffff0000;
+       tmp |= sscdivintphase[idx];
+       intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE, tmp, SBI_ICLK);
+
+       mutex_unlock(&dev_priv->sb_lock);
+}
+
+#undef BEND_IDX
+
 static void lpt_init_pch_refclk(struct drm_device *dev)
 {
        struct intel_encoder *encoder;
@@ -8579,10 +8705,12 @@ static void lpt_init_pch_refclk(struct drm_device *dev)
                }
        }
 
-       if (has_vga)
+       if (has_vga) {
+               lpt_bend_clkout_dp(to_i915(dev), 0);
                lpt_enable_clkout_dp(dev, true, true);
-       else
+       } else {
                lpt_disable_clkout_dp(dev);
+       }
 }
 
 /*
@@ -8945,7 +9073,7 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
        memset(&crtc_state->dpll_hw_state, 0,
               sizeof(crtc_state->dpll_hw_state));
 
-       is_lvds = intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS);
+       is_lvds = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS);
 
        WARN(!(HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)),
             "Unexpected PCH type %d\n", INTEL_PCH_TYPE(dev));
@@ -9589,7 +9717,9 @@ void hsw_disable_pc8(struct drm_i915_private *dev_priv)
 static void broxton_modeset_commit_cdclk(struct drm_atomic_state *old_state)
 {
        struct drm_device *dev = old_state->dev;
-       unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
+       struct intel_atomic_state *old_intel_state =
+               to_intel_atomic_state(old_state);
+       unsigned int req_cdclk = old_intel_state->dev_cdclk;
 
        broxton_set_cdclk(dev, req_cdclk);
 }
@@ -9597,29 +9727,41 @@ static void broxton_modeset_commit_cdclk(struct drm_atomic_state *old_state)
 /* compute the max rate for new configuration */
 static int ilk_max_pixel_rate(struct drm_atomic_state *state)
 {
-       struct intel_crtc *intel_crtc;
+       struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+       struct drm_i915_private *dev_priv = state->dev->dev_private;
+       struct drm_crtc *crtc;
+       struct drm_crtc_state *cstate;
        struct intel_crtc_state *crtc_state;
-       int max_pixel_rate = 0;
+       unsigned max_pixel_rate = 0, i;
+       enum pipe pipe;
 
-       for_each_intel_crtc(state->dev, intel_crtc) {
-               int pixel_rate;
+       memcpy(intel_state->min_pixclk, dev_priv->min_pixclk,
+              sizeof(intel_state->min_pixclk));
 
-               crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
-               if (IS_ERR(crtc_state))
-                       return PTR_ERR(crtc_state);
+       for_each_crtc_in_state(state, crtc, cstate, i) {
+               int pixel_rate;
 
-               if (!crtc_state->base.enable)
+               crtc_state = to_intel_crtc_state(cstate);
+               if (!crtc_state->base.enable) {
+                       intel_state->min_pixclk[i] = 0;
                        continue;
+               }
 
                pixel_rate = ilk_pipe_pixel_rate(crtc_state);
 
                /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
-               if (IS_BROADWELL(state->dev) && crtc_state->ips_enabled)
+               if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled)
                        pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95);
 
-               max_pixel_rate = max(max_pixel_rate, pixel_rate);
+               intel_state->min_pixclk[i] = pixel_rate;
        }
 
+       if (!intel_state->active_crtcs)
+               return 0;
+
+       for_each_pipe(dev_priv, pipe)
+               max_pixel_rate = max(intel_state->min_pixclk[pipe], max_pixel_rate);
+
        return max_pixel_rate;
 }
 
@@ -9703,6 +9845,7 @@ static void broadwell_set_cdclk(struct drm_device *dev, int cdclk)
 static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
 {
        struct drm_i915_private *dev_priv = to_i915(state->dev);
+       struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
        int max_pixclk = ilk_max_pixel_rate(state);
        int cdclk;
 
@@ -9719,17 +9862,15 @@ static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
        else
                cdclk = 337500;
 
-       /*
-        * FIXME move the cdclk caclulation to
-        * compute_config() so we can fail gracegully.
-        */
        if (cdclk > dev_priv->max_cdclk_freq) {
-               DRM_ERROR("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
-                         cdclk, dev_priv->max_cdclk_freq);
-               cdclk = dev_priv->max_cdclk_freq;
+               DRM_DEBUG_KMS("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
+                             cdclk, dev_priv->max_cdclk_freq);
+               return -EINVAL;
        }
 
-       to_intel_atomic_state(state)->cdclk = cdclk;
+       intel_state->cdclk = intel_state->dev_cdclk = cdclk;
+       if (!intel_state->active_crtcs)
+               intel_state->dev_cdclk = 337500;
 
        return 0;
 }
@@ -9737,7 +9878,9 @@ static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
 static void broadwell_modeset_commit_cdclk(struct drm_atomic_state *old_state)
 {
        struct drm_device *dev = old_state->dev;
-       unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
+       struct intel_atomic_state *old_intel_state =
+               to_intel_atomic_state(old_state);
+       unsigned req_cdclk = old_intel_state->dev_cdclk;
 
        broadwell_set_cdclk(dev, req_cdclk);
 }
@@ -9821,6 +9964,7 @@ static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
                break;
        case PORT_CLK_SEL_SPLL:
                pipe_config->shared_dpll = DPLL_ID_SPLL;
+               break;
        }
 }
 
@@ -9949,16 +10093,17 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
        return true;
 }
 
-static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
+static void i845_update_cursor(struct drm_crtc *crtc, u32 base,
+                              const struct intel_plane_state *plane_state)
 {
        struct drm_device *dev = crtc->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
        uint32_t cntl = 0, size = 0;
 
-       if (base) {
-               unsigned int width = intel_crtc->base.cursor->state->crtc_w;
-               unsigned int height = intel_crtc->base.cursor->state->crtc_h;
+       if (plane_state && plane_state->visible) {
+               unsigned int width = plane_state->base.crtc_w;
+               unsigned int height = plane_state->base.crtc_h;
                unsigned int stride = roundup_pow_of_two(width) * 4;
 
                switch (stride) {
@@ -10011,18 +10156,18 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
        }
 }
 
-static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
+static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
+                              const struct intel_plane_state *plane_state)
 {
        struct drm_device *dev = crtc->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
        int pipe = intel_crtc->pipe;
-       uint32_t cntl;
+       uint32_t cntl = 0;
 
-       cntl = 0;
-       if (base) {
+       if (plane_state && plane_state->visible) {
                cntl = MCURSOR_GAMMA_ENABLE;
-               switch (intel_crtc->base.cursor->state->crtc_w) {
+               switch (plane_state->base.crtc_w) {
                        case 64:
                                cntl |= CURSOR_MODE_64_ARGB_AX;
                                break;
@@ -10033,17 +10178,17 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
                                cntl |= CURSOR_MODE_256_ARGB_AX;
                                break;
                        default:
-                               MISSING_CASE(intel_crtc->base.cursor->state->crtc_w);
+                               MISSING_CASE(plane_state->base.crtc_w);
                                return;
                }
                cntl |= pipe << 28; /* Connect to correct pipe */
 
                if (HAS_DDI(dev))
                        cntl |= CURSOR_PIPE_CSC_ENABLE;
-       }
 
-       if (crtc->cursor->state->rotation == BIT(DRM_ROTATE_180))
-               cntl |= CURSOR_ROTATE_180;
+               if (plane_state->base.rotation == BIT(DRM_ROTATE_180))
+                       cntl |= CURSOR_ROTATE_180;
+       }
 
        if (intel_crtc->cursor_cntl != cntl) {
                I915_WRITE(CURCNTR(pipe), cntl);
@@ -10060,60 +10205,45 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
 
 /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
 static void intel_crtc_update_cursor(struct drm_crtc *crtc,
-                                    bool on)
+                                    const struct intel_plane_state *plane_state)
 {
        struct drm_device *dev = crtc->dev;
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
        int pipe = intel_crtc->pipe;
-       struct drm_plane_state *cursor_state = crtc->cursor->state;
-       int x = cursor_state->crtc_x;
-       int y = cursor_state->crtc_y;
-       u32 base = 0, pos = 0;
-
-       if (on)
-               base = intel_crtc->cursor_addr;
+       u32 base = intel_crtc->cursor_addr;
+       u32 pos = 0;
 
-       if (x >= intel_crtc->config->pipe_src_w)
-               base = 0;
+       if (plane_state) {
+               int x = plane_state->base.crtc_x;
+               int y = plane_state->base.crtc_y;
 
-       if (y >= intel_crtc->config->pipe_src_h)
-               base = 0;
-
-       if (x < 0) {
-               if (x + cursor_state->crtc_w <= 0)
-                       base = 0;
-
-               pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
-               x = -x;
-       }
-       pos |= x << CURSOR_X_SHIFT;
+               if (x < 0) {
+                       pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
+                       x = -x;
+               }
+               pos |= x << CURSOR_X_SHIFT;
 
-       if (y < 0) {
-               if (y + cursor_state->crtc_h <= 0)
-                       base = 0;
+               if (y < 0) {
+                       pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
+                       y = -y;
+               }
+               pos |= y << CURSOR_Y_SHIFT;
 
-               pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
-               y = -y;
+               /* ILK+ do this automagically */
+               if (HAS_GMCH_DISPLAY(dev) &&
+                   plane_state->base.rotation == BIT(DRM_ROTATE_180)) {
+                       base += (plane_state->base.crtc_h *
+                                plane_state->base.crtc_w - 1) * 4;
+               }
        }
-       pos |= y << CURSOR_Y_SHIFT;
-
-       if (base == 0 && intel_crtc->cursor_base == 0)
-               return;
 
        I915_WRITE(CURPOS(pipe), pos);
 
-       /* ILK+ do this automagically */
-       if (HAS_GMCH_DISPLAY(dev) &&
-           crtc->cursor->state->rotation == BIT(DRM_ROTATE_180)) {
-               base += (cursor_state->crtc_h *
-                        cursor_state->crtc_w - 1) * 4;
-       }
-
        if (IS_845G(dev) || IS_I865G(dev))
-               i845_update_cursor(crtc, base);
+               i845_update_cursor(crtc, base, plane_state);
        else
-               i9xx_update_cursor(crtc, base);
+               i9xx_update_cursor(crtc, base, plane_state);
 }
 
 static bool cursor_size_ok(struct drm_device *dev,
@@ -11205,6 +11335,10 @@ static bool use_mmio_flip(struct intel_engine_cs *ring,
                return true;
        else if (i915.enable_execlists)
                return true;
+       else if (obj->base.dma_buf &&
+                !reservation_object_test_signaled_rcu(obj->base.dma_buf->resv,
+                                                      false))
+               return true;
        else
                return ring != i915_gem_request_get_ring(obj->last_write_req);
 }
@@ -11319,6 +11453,9 @@ static void intel_mmio_flip_work_func(struct work_struct *work)
 {
        struct intel_mmio_flip *mmio_flip =
                container_of(work, struct intel_mmio_flip, work);
+       struct intel_framebuffer *intel_fb =
+               to_intel_framebuffer(mmio_flip->crtc->base.primary->fb);
+       struct drm_i915_gem_object *obj = intel_fb->obj;
 
        if (mmio_flip->req) {
                WARN_ON(__i915_wait_request(mmio_flip->req,
@@ -11328,6 +11465,12 @@ static void intel_mmio_flip_work_func(struct work_struct *work)
                i915_gem_request_unreference__unlocked(mmio_flip->req);
        }
 
+       /* For framebuffer backed by dmabuf, wait for fence */
+       if (obj->base.dma_buf)
+               WARN_ON(reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv,
+                                                           false, false,
+                                                           MAX_SCHEDULE_TIMEOUT) < 0);
+
        intel_do_mmio_flip(mmio_flip);
        kfree(mmio_flip);
 }
@@ -11529,7 +11672,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
        if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
                work->flip_count = I915_READ(PIPE_FLIPCOUNT_G4X(pipe)) + 1;
 
-       if (IS_VALLEYVIEW(dev)) {
+       if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
                ring = &dev_priv->ring[BCS];
                if (obj->tiling_mode != intel_fb_obj(work->old_fb)->tiling_mode)
                        /* vlv: DISPLAY_FLIP fails to change tiling */
@@ -11598,7 +11741,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
                          to_intel_plane(primary)->frontbuffer_bit);
        mutex_unlock(&dev->struct_mutex);
 
-       intel_fbc_disable_crtc(intel_crtc);
+       intel_fbc_deactivate(intel_crtc);
        intel_frontbuffer_flip_prepare(dev,
                                       to_intel_plane(primary)->frontbuffer_bit);
 
@@ -11685,9 +11828,14 @@ static bool intel_wm_need_update(struct drm_plane *plane,
        struct intel_plane_state *cur = to_intel_plane_state(plane->state);
 
        /* Update watermarks on tiling or size changes. */
-       if (!plane->state->fb || !state->fb ||
-           plane->state->fb->modifier[0] != state->fb->modifier[0] ||
-           plane->state->rotation != state->rotation ||
+       if (new->visible != cur->visible)
+               return true;
+
+       if (!cur->base.fb || !new->base.fb)
+               return false;
+
+       if (cur->base.fb->modifier[0] != new->base.fb->modifier[0] ||
+           cur->base.rotation != new->base.rotation ||
            drm_rect_width(&new->src) != drm_rect_width(&cur->src) ||
            drm_rect_height(&new->src) != drm_rect_height(&cur->src) ||
            drm_rect_width(&new->dst) != drm_rect_width(&cur->dst) ||
@@ -11710,6 +11858,7 @@ static bool needs_scaling(struct intel_plane_state *state)
 int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
                                    struct drm_plane_state *plane_state)
 {
+       struct intel_crtc_state *pipe_config = to_intel_crtc_state(crtc_state);
        struct drm_crtc *crtc = crtc_state->crtc;
        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
        struct drm_plane *plane = plane_state->plane;
@@ -11740,8 +11889,13 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
        if (!was_crtc_enabled && WARN_ON(was_visible))
                was_visible = false;
 
-       if (!is_crtc_enabled && WARN_ON(visible))
-               visible = false;
+       /*
+        * Visibility is calculated as if the crtc was on, but
+        * after scaler setup everything depends on it being off
+        * when the crtc isn't active.
+        */
+       if (!is_crtc_enabled)
+               to_intel_plane_state(plane_state)->visible = visible = false;
 
        if (!was_visible && !visible)
                return 0;
@@ -11756,27 +11910,24 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
                         plane->base.id, was_visible, visible,
                         turn_off, turn_on, mode_changed);
 
-       if (turn_on) {
-               intel_crtc->atomic.update_wm_pre = true;
-               /* must disable cxsr around plane enable/disable */
-               if (plane->type != DRM_PLANE_TYPE_CURSOR) {
-                       intel_crtc->atomic.disable_cxsr = true;
-                       /* to potentially re-enable cxsr */
-                       intel_crtc->atomic.wait_vblank = true;
-                       intel_crtc->atomic.update_wm_post = true;
-               }
-       } else if (turn_off) {
-               intel_crtc->atomic.update_wm_post = true;
+       if (turn_on || turn_off) {
+               pipe_config->wm_changed = true;
+
                /* must disable cxsr around plane enable/disable */
                if (plane->type != DRM_PLANE_TYPE_CURSOR) {
                        if (is_crtc_enabled)
                                intel_crtc->atomic.wait_vblank = true;
-                       intel_crtc->atomic.disable_cxsr = true;
+                       pipe_config->disable_cxsr = true;
                }
        } else if (intel_wm_need_update(plane, plane_state)) {
-               intel_crtc->atomic.update_wm_pre = true;
+               pipe_config->wm_changed = true;
        }
 
+       /* Pre-gen9 platforms need two-step watermark updates */
+       if (pipe_config->wm_changed && INTEL_INFO(dev)->gen < 9 &&
+           dev_priv->display.optimize_watermarks)
+               to_intel_crtc_state(crtc_state)->wm.need_postvbl_update = true;
+
        if (visible || was_visible)
                intel_crtc->atomic.fb_bits |=
                        to_intel_plane(plane)->frontbuffer_bit;
@@ -11919,7 +12070,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
        }
 
        if (mode_changed && !crtc_state->active)
-               intel_crtc->atomic.update_wm_post = true;
+               pipe_config->wm_changed = true;
 
        if (mode_changed && crtc_state->enable &&
            dev_priv->display.crtc_compute_clock &&
@@ -11933,8 +12084,29 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
        ret = 0;
        if (dev_priv->display.compute_pipe_wm) {
                ret = dev_priv->display.compute_pipe_wm(intel_crtc, state);
-               if (ret)
+               if (ret) {
+                       DRM_DEBUG_KMS("Target pipe watermarks are invalid\n");
                        return ret;
+               }
+       }
+
+       if (dev_priv->display.compute_intermediate_wm &&
+           !to_intel_atomic_state(state)->skip_intermediate_wm) {
+               if (WARN_ON(!dev_priv->display.compute_pipe_wm))
+                       return 0;
+
+               /*
+                * Calculate 'intermediate' watermarks that satisfy both the
+                * old state and the new state.  We can program these
+                * immediately.
+                */
+               ret = dev_priv->display.compute_intermediate_wm(crtc->dev,
+                                                               intel_crtc,
+                                                               pipe_config);
+               if (ret) {
+                       DRM_DEBUG_KMS("No valid intermediate pipe watermarks are possible\n");
+                       return ret;
+               }
        }
 
        if (INTEL_INFO(dev)->gen >= 9) {
@@ -12010,7 +12182,7 @@ compute_baseline_pipe_bpp(struct intel_crtc *crtc,
        struct drm_connector_state *connector_state;
        int bpp, i;
 
-       if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)))
+       if ((IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)))
                bpp = 10*3;
        else if (INTEL_INFO(dev)->gen >= 5)
                bpp = 12*3;
@@ -12185,18 +12357,22 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
 static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 {
        struct drm_device *dev = state->dev;
-       struct intel_encoder *encoder;
        struct drm_connector *connector;
-       struct drm_connector_state *connector_state;
        unsigned int used_ports = 0;
-       int i;
 
        /*
         * Walk the connector list instead of the encoder
         * list to detect the problem on ddi platforms
         * where there's just one encoder per digital port.
         */
-       for_each_connector_in_state(state, connector, connector_state, i) {
+       drm_for_each_connector(connector, dev) {
+               struct drm_connector_state *connector_state;
+               struct intel_encoder *encoder;
+
+               connector_state = drm_atomic_get_existing_connector_state(state, connector);
+               if (!connector_state)
+                       connector_state = connector->state;
+
                if (!connector_state->best_encoder)
                        continue;
 
@@ -12417,7 +12593,7 @@ static bool intel_fuzzy_clock_check(int clock1, int clock2)
        list_for_each_entry((intel_crtc), \
                            &(dev)->mode_config.crtc_list, \
                            base.head) \
-               if (mask & (1 <<(intel_crtc)->pipe))
+               for_each_if (mask & (1 <<(intel_crtc)->pipe))
 
 static bool
 intel_compare_m_n(unsigned int m, unsigned int n,
@@ -12432,19 +12608,22 @@ intel_compare_m_n(unsigned int m, unsigned int n,
 
        BUILD_BUG_ON(DATA_LINK_M_N_MASK > INT_MAX);
 
-       if (m > m2) {
-               while (m > m2) {
+       if (n > n2) {
+               while (n > n2) {
                        m2 <<= 1;
                        n2 <<= 1;
                }
-       } else if (m < m2) {
-               while (m < m2) {
+       } else if (n < n2) {
+               while (n < n2) {
                        m <<= 1;
                        n <<= 1;
                }
        }
 
-       return m == m2 && n == n2;
+       if (n != n2)
+               return false;
+
+       return intel_fuzzy_clock_check(m, m2);
 }
 
 static bool
@@ -12596,12 +12775,13 @@ intel_pipe_config_compare(struct drm_device *dev,
        if (INTEL_INFO(dev)->gen < 8) {
                PIPE_CONF_CHECK_M_N(dp_m_n);
 
-               PIPE_CONF_CHECK_I(has_drrs);
                if (current_config->has_drrs)
                        PIPE_CONF_CHECK_M_N(dp_m2_n2);
        } else
                PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
 
+       PIPE_CONF_CHECK_I(has_dsi_encoder);
+
        PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
        PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
        PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
@@ -12619,7 +12799,7 @@ intel_pipe_config_compare(struct drm_device *dev,
        PIPE_CONF_CHECK_I(pixel_multiplier);
        PIPE_CONF_CHECK_I(has_hdmi_sink);
        if ((INTEL_INFO(dev)->gen < 8 && !IS_HASWELL(dev)) ||
-           IS_VALLEYVIEW(dev))
+           IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
                PIPE_CONF_CHECK_I(limited_color_range);
        PIPE_CONF_CHECK_I(has_infoframe);
 
@@ -13118,15 +13298,27 @@ static int intel_modeset_all_pipes(struct drm_atomic_state *state)
 
 static int intel_modeset_checks(struct drm_atomic_state *state)
 {
-       struct drm_device *dev = state->dev;
-       struct drm_i915_private *dev_priv = dev->dev_private;
-       int ret;
+       struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+       struct drm_i915_private *dev_priv = state->dev->dev_private;
+       struct drm_crtc *crtc;
+       struct drm_crtc_state *crtc_state;
+       int ret = 0, i;
 
        if (!check_digital_port_conflicts(state)) {
                DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
                return -EINVAL;
        }
 
+       intel_state->modeset = true;
+       intel_state->active_crtcs = dev_priv->active_crtcs;
+
+       for_each_crtc_in_state(state, crtc, crtc_state, i) {
+               if (crtc_state->active)
+                       intel_state->active_crtcs |= 1 << i;
+               else
+                       intel_state->active_crtcs &= ~(1 << i);
+       }
+
        /*
         * See if the config requires any additional preparation, e.g.
         * to adjust global state with pipes off.  We need to do this
@@ -13135,22 +13327,19 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
         * adjusted_mode bits in the crtc directly.
         */
        if (dev_priv->display.modeset_calc_cdclk) {
-               unsigned int cdclk;
-
                ret = dev_priv->display.modeset_calc_cdclk(state);
 
-               cdclk = to_intel_atomic_state(state)->cdclk;
-               if (!ret && cdclk != dev_priv->cdclk_freq)
+               if (!ret && intel_state->dev_cdclk != dev_priv->cdclk_freq)
                        ret = intel_modeset_all_pipes(state);
 
                if (ret < 0)
                        return ret;
        } else
-               to_intel_atomic_state(state)->cdclk = dev_priv->cdclk_freq;
+               to_intel_atomic_state(state)->cdclk = dev_priv->atomic_cdclk_freq;
 
        intel_modeset_clear_plls(state);
 
-       if (IS_HASWELL(dev))
+       if (IS_HASWELL(dev_priv))
                return haswell_mode_set_planes_workaround(state);
 
        return 0;
@@ -13368,12 +13557,13 @@ static int intel_atomic_commit(struct drm_device *dev,
                               struct drm_atomic_state *state,
                               bool async)
 {
+       struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
        struct drm_i915_private *dev_priv = dev->dev_private;
        struct drm_crtc_state *crtc_state;
        struct drm_crtc *crtc;
-       int ret = 0;
-       int i;
-       bool any_ms = false;
+       struct intel_crtc_state *intel_cstate;
+       int ret = 0, i;
+       bool hw_check = intel_state->modeset;
 
        ret = intel_atomic_prepare_commit(dev, state, async);
        if (ret) {
@@ -13384,13 +13574,19 @@ static int intel_atomic_commit(struct drm_device *dev,
        drm_atomic_helper_swap_state(dev, state);
        dev_priv->wm.config = to_intel_atomic_state(state)->wm_config;
 
+       if (intel_state->modeset) {
+               memcpy(dev_priv->min_pixclk, intel_state->min_pixclk,
+                      sizeof(intel_state->min_pixclk));
+               dev_priv->active_crtcs = intel_state->active_crtcs;
+               dev_priv->atomic_cdclk_freq = intel_state->cdclk;
+       }
+
        for_each_crtc_in_state(state, crtc, crtc_state, i) {
                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
                if (!needs_modeset(crtc->state))
                        continue;
 
-               any_ms = true;
                intel_pre_plane_update(intel_crtc);
 
                if (crtc_state->active) {
@@ -13405,6 +13601,9 @@ static int intel_atomic_commit(struct drm_device *dev,
                         */
                        intel_check_cpu_fifo_underruns(dev_priv);
                        intel_check_pch_fifo_underruns(dev_priv);
+
+                       if (!crtc->state->active)
+                               intel_update_watermarks(crtc);
                }
        }
 
@@ -13412,7 +13611,7 @@ static int intel_atomic_commit(struct drm_device *dev,
         * update the the output configuration. */
        intel_modeset_update_crtc_state(state);
 
-       if (any_ms) {
+       if (intel_state->modeset) {
                intel_shared_dpll_commit(state);
 
                drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
@@ -13439,7 +13638,7 @@ static int intel_atomic_commit(struct drm_device *dev,
                        put_domains = modeset_get_crtc_power_domains(crtc);
 
                        /* make sure intel_modeset_check_state runs */
-                       any_ms = true;
+                       hw_check = true;
                }
 
                if (!modeset)
@@ -13462,15 +13661,42 @@ static int intel_atomic_commit(struct drm_device *dev,
 
        drm_atomic_helper_wait_for_vblanks(dev, state);
 
+       /*
+        * Now that the vblank has passed, we can go ahead and program the
+        * optimal watermarks on platforms that need two-step watermark
+        * programming.
+        *
+        * TODO: Move this (and other cleanup) to an async worker eventually.
+        */
+       for_each_crtc_in_state(state, crtc, crtc_state, i) {
+               intel_cstate = to_intel_crtc_state(crtc->state);
+
+               if (dev_priv->display.optimize_watermarks)
+                       dev_priv->display.optimize_watermarks(intel_cstate);
+       }
+
        mutex_lock(&dev->struct_mutex);
        drm_atomic_helper_cleanup_planes(dev, state);
        mutex_unlock(&dev->struct_mutex);
 
-       if (any_ms)
+       if (hw_check)
                intel_modeset_check_state(dev, state);
 
        drm_atomic_state_free(state);
 
+       /* As one of the primary mmio accessors, KMS has a high likelihood
+        * of triggering bugs in unclaimed access. After we finish
+        * modesetting, see if an error has been flagged, and if so
+        * enable debugging for the next modeset - and hope we catch
+        * the culprit.
+        *
+        * XXX note that we assume display power is on at this point.
+        * This might hold true now but we need to add pm helper to check
+        * unclaimed only when the hardware is on, as atomic commits
+        * can happen also when the device is completely off.
+        */
+       intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
+
        return 0;
 }
 
@@ -13674,6 +13900,17 @@ intel_prepare_plane_fb(struct drm_plane *plane,
                        return ret;
        }
 
+       /* For framebuffer backed by dmabuf, wait for fence */
+       if (obj && obj->base.dma_buf) {
+               ret = reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv,
+                                                         false, true,
+                                                         MAX_SCHEDULE_TIMEOUT);
+               if (ret == -ERESTARTSYS)
+                       return ret;
+
+               WARN_ON(ret < 0);
+       }
+
        if (!obj) {
                ret = 0;
        } else if (plane->type == DRM_PLANE_TYPE_CURSOR &&
@@ -13746,7 +13983,7 @@ skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state
        struct drm_i915_private *dev_priv;
        int crtc_clock, cdclk;
 
-       if (!intel_crtc || !crtc_state)
+       if (!intel_crtc || !crtc_state->base.enable)
                return DRM_PLANE_HELPER_NO_SCALING;
 
        dev = intel_crtc->base.dev;
@@ -13794,32 +14031,6 @@ intel_check_primary_plane(struct drm_plane *plane,
                                             &state->visible);
 }
 
-static void
-intel_commit_primary_plane(struct drm_plane *plane,
-                          struct intel_plane_state *state)
-{
-       struct drm_crtc *crtc = state->base.crtc;
-       struct drm_framebuffer *fb = state->base.fb;
-       struct drm_device *dev = plane->dev;
-       struct drm_i915_private *dev_priv = dev->dev_private;
-
-       crtc = crtc ? crtc : plane->crtc;
-
-       dev_priv->display.update_primary_plane(crtc, fb,
-                                              state->src.x1 >> 16,
-                                              state->src.y1 >> 16);
-}
-
-static void
-intel_disable_primary_plane(struct drm_plane *plane,
-                           struct drm_crtc *crtc)
-{
-       struct drm_device *dev = plane->dev;
-       struct drm_i915_private *dev_priv = dev->dev_private;
-
-       dev_priv->display.update_primary_plane(crtc, NULL, 0, 0);
-}
-
 static void intel_begin_crtc_commit(struct drm_crtc *crtc,
                                    struct drm_crtc_state *old_crtc_state)
 {
@@ -13829,9 +14040,6 @@ static void intel_begin_crtc_commit(struct drm_crtc *crtc,
                to_intel_crtc_state(old_crtc_state);
        bool modeset = needs_modeset(crtc->state);
 
-       if (intel_crtc->atomic.update_wm_pre)
-               intel_update_watermarks(crtc);
-
        /* Perform vblank evasion around commit operation */
        intel_pipe_update_start(intel_crtc);
 
@@ -13907,20 +14115,33 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
        primary->plane = pipe;
        primary->frontbuffer_bit = INTEL_FRONTBUFFER_PRIMARY(pipe);
        primary->check_plane = intel_check_primary_plane;
-       primary->commit_plane = intel_commit_primary_plane;
-       primary->disable_plane = intel_disable_primary_plane;
        if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4)
                primary->plane = !pipe;
 
        if (INTEL_INFO(dev)->gen >= 9) {
                intel_primary_formats = skl_primary_formats;
                num_formats = ARRAY_SIZE(skl_primary_formats);
+
+               primary->update_plane = skylake_update_primary_plane;
+               primary->disable_plane = skylake_disable_primary_plane;
+       } else if (HAS_PCH_SPLIT(dev)) {
+               intel_primary_formats = i965_primary_formats;
+               num_formats = ARRAY_SIZE(i965_primary_formats);
+
+               primary->update_plane = ironlake_update_primary_plane;
+               primary->disable_plane = i9xx_disable_primary_plane;
        } else if (INTEL_INFO(dev)->gen >= 4) {
                intel_primary_formats = i965_primary_formats;
                num_formats = ARRAY_SIZE(i965_primary_formats);
+
+               primary->update_plane = i9xx_update_primary_plane;
+               primary->disable_plane = i9xx_disable_primary_plane;
        } else {
                intel_primary_formats = i8xx_primary_formats;
                num_formats = ARRAY_SIZE(i8xx_primary_formats);
+
+               primary->update_plane = i9xx_update_primary_plane;
+               primary->disable_plane = i9xx_disable_primary_plane;
        }
 
        drm_universal_plane_init(dev, &primary->base, 0,
@@ -13962,6 +14183,7 @@ intel_check_cursor_plane(struct drm_plane *plane,
        struct drm_crtc *crtc = crtc_state->base.crtc;
        struct drm_framebuffer *fb = state->base.fb;
        struct drm_i915_gem_object *obj = intel_fb_obj(fb);
+       enum pipe pipe = to_intel_plane(plane)->pipe;
        unsigned stride;
        int ret;
 
@@ -13995,6 +14217,22 @@ intel_check_cursor_plane(struct drm_plane *plane,
                return -EINVAL;
        }
 
+       /*
+        * There's something wrong with the cursor on CHV pipe C.
+        * If it straddles the left edge of the screen then
+        * moving it away from the edge or disabling it often
+        * results in a pipe underrun, and often that can lead to
+        * dead pipe (constant underrun reported, and it scans
+        * out just a solid color). To recover from that, the
+        * display power well must be turned off and on again.
+        * Refuse the put the cursor into that compromised position.
+        */
+       if (IS_CHERRYVIEW(plane->dev) && pipe == PIPE_C &&
+           state->visible && state->base.crtc_x < 0) {
+               DRM_DEBUG_KMS("CHV cursor C not allowed to straddle the left screen edge\n");
+               return -EINVAL;
+       }
+
        return 0;
 }
 
@@ -14002,25 +14240,23 @@ static void
 intel_disable_cursor_plane(struct drm_plane *plane,
                           struct drm_crtc *crtc)
 {
-       intel_crtc_update_cursor(crtc, false);
+       struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
+       intel_crtc->cursor_addr = 0;
+       intel_crtc_update_cursor(crtc, NULL);
 }
 
 static void
-intel_commit_cursor_plane(struct drm_plane *plane,
-                         struct intel_plane_state *state)
+intel_update_cursor_plane(struct drm_plane *plane,
+                         const struct intel_crtc_state *crtc_state,
+                         const struct intel_plane_state *state)
 {
-       struct drm_crtc *crtc = state->base.crtc;
+       struct drm_crtc *crtc = crtc_state->base.crtc;
+       struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
        struct drm_device *dev = plane->dev;
-       struct intel_crtc *intel_crtc;
        struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb);
        uint32_t addr;
 
-       crtc = crtc ? crtc : plane->crtc;
-       intel_crtc = to_intel_crtc(crtc);
-
-       if (intel_crtc->cursor_bo == obj)
-               goto update;
-
        if (!obj)
                addr = 0;
        else if (!INTEL_INFO(dev)->cursor_needs_physical)
@@ -14029,10 +14265,7 @@ intel_commit_cursor_plane(struct drm_plane *plane,
                addr = obj->phys_handle->busaddr;
 
        intel_crtc->cursor_addr = addr;
-       intel_crtc->cursor_bo = obj;
-
-update:
-       intel_crtc_update_cursor(crtc, state->visible);
+       intel_crtc_update_cursor(crtc, state);
 }
 
 static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
@@ -14058,7 +14291,7 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
        cursor->plane = pipe;
        cursor->frontbuffer_bit = INTEL_FRONTBUFFER_CURSOR(pipe);
        cursor->check_plane = intel_check_cursor_plane;
-       cursor->commit_plane = intel_commit_cursor_plane;
+       cursor->update_plane = intel_update_cursor_plane;
        cursor->disable_plane = intel_disable_cursor_plane;
 
        drm_universal_plane_init(dev, &cursor->base, 0,
@@ -14268,7 +14501,14 @@ static bool intel_crt_present(struct drm_device *dev)
        if (IS_CHERRYVIEW(dev))
                return false;
 
-       if (IS_VALLEYVIEW(dev) && !dev_priv->vbt.int_crt_support)
+       if (HAS_PCH_LPT_H(dev) && I915_READ(SFUSE_STRAP) & SFUSE_STRAP_CRT_DISABLED)
+               return false;
+
+       /* DDI E can't be used if DDI A requires 4 lanes */
+       if (HAS_DDI(dev) && I915_READ(DDI_BUF_CTL(PORT_A)) & DDI_A_4_LANES)
+               return false;
+
+       if (!dev_priv->vbt.int_crt_support)
                return false;
 
        return true;
@@ -14353,7 +14593,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
                if (I915_READ(PCH_DP_D) & DP_DETECTED)
                        intel_dp_init(dev, PCH_DP_D, PORT_D);
-       } else if (IS_VALLEYVIEW(dev)) {
+       } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
                /*
                 * The DP_DETECTED bit is the latched state of the DDC
                 * SDA pin at boot. However since eDP doesn't require DDC
@@ -14502,7 +14742,7 @@ u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
                 *  pixels and 32K bytes."
                 */
                 return min(8192*drm_format_plane_cpp(pixel_format, 0), 32768);
-       } else if (gen >= 5 && !IS_VALLEYVIEW(dev)) {
+       } else if (gen >= 5 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
                return 32*1024;
        } else if (gen >= 4) {
                if (fb_modifier == I915_FORMAT_MOD_X_TILED)
@@ -14606,7 +14846,8 @@ static int intel_framebuffer_init(struct drm_device *dev,
                }
                break;
        case DRM_FORMAT_ABGR8888:
-               if (!IS_VALLEYVIEW(dev) && INTEL_INFO(dev)->gen < 9) {
+               if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
+                   INTEL_INFO(dev)->gen < 9) {
                        DRM_DEBUG("unsupported pixel format: %s\n",
                                  drm_get_format_name(mode_cmd->pixel_format));
                        return -EINVAL;
@@ -14622,7 +14863,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
                }
                break;
        case DRM_FORMAT_ABGR2101010:
-               if (!IS_VALLEYVIEW(dev)) {
+               if (!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)) {
                        DRM_DEBUG("unsupported pixel format: %s\n",
                                  drm_get_format_name(mode_cmd->pixel_format));
                        return -EINVAL;
@@ -14671,7 +14912,7 @@ static int intel_framebuffer_init(struct drm_device *dev,
 static struct drm_framebuffer *
 intel_user_framebuffer_create(struct drm_device *dev,
                              struct drm_file *filp,
-                             struct drm_mode_fb_cmd2 *user_mode_cmd)
+                             const struct drm_mode_fb_cmd2 *user_mode_cmd)
 {
        struct drm_framebuffer *fb;
        struct drm_i915_gem_object *obj;
@@ -14728,8 +14969,6 @@ static void intel_init_display(struct drm_device *dev)
                        haswell_crtc_compute_clock;
                dev_priv->display.crtc_enable = haswell_crtc_enable;
                dev_priv->display.crtc_disable = haswell_crtc_disable;
-               dev_priv->display.update_primary_plane =
-                       skylake_update_primary_plane;
        } else if (HAS_DDI(dev)) {
                dev_priv->display.get_pipe_config = haswell_get_pipe_config;
                dev_priv->display.get_initial_plane_config =
@@ -14738,8 +14977,6 @@ static void intel_init_display(struct drm_device *dev)
                        haswell_crtc_compute_clock;
                dev_priv->display.crtc_enable = haswell_crtc_enable;
                dev_priv->display.crtc_disable = haswell_crtc_disable;
-               dev_priv->display.update_primary_plane =
-                       ironlake_update_primary_plane;
        } else if (HAS_PCH_SPLIT(dev)) {
                dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
                dev_priv->display.get_initial_plane_config =
@@ -14748,17 +14985,13 @@ static void intel_init_display(struct drm_device *dev)
                        ironlake_crtc_compute_clock;
                dev_priv->display.crtc_enable = ironlake_crtc_enable;
                dev_priv->display.crtc_disable = ironlake_crtc_disable;
-               dev_priv->display.update_primary_plane =
-                       ironlake_update_primary_plane;
-       } else if (IS_VALLEYVIEW(dev)) {
+       } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
                dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
                dev_priv->display.get_initial_plane_config =
                        i9xx_get_initial_plane_config;
                dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
                dev_priv->display.crtc_enable = valleyview_crtc_enable;
                dev_priv->display.crtc_disable = i9xx_crtc_disable;
-               dev_priv->display.update_primary_plane =
-                       i9xx_update_primary_plane;
        } else {
                dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
                dev_priv->display.get_initial_plane_config =
@@ -14766,8 +14999,6 @@ static void intel_init_display(struct drm_device *dev)
                dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
                dev_priv->display.crtc_enable = i9xx_crtc_enable;
                dev_priv->display.crtc_disable = i9xx_crtc_disable;
-               dev_priv->display.update_primary_plane =
-                       i9xx_update_primary_plane;
        }
 
        /* Returns the core display clock speed */
@@ -14783,7 +15014,7 @@ static void intel_init_display(struct drm_device *dev)
        else if (IS_HASWELL(dev))
                dev_priv->display.get_display_clock_speed =
                        haswell_get_display_clock_speed;
-       else if (IS_VALLEYVIEW(dev))
+       else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
                dev_priv->display.get_display_clock_speed =
                        valleyview_get_display_clock_speed;
        else if (IS_GEN5(dev))
@@ -14811,9 +15042,6 @@ static void intel_init_display(struct drm_device *dev)
        else if (IS_I945GM(dev) || IS_845G(dev))
                dev_priv->display.get_display_clock_speed =
                        i9xx_misc_get_display_clock_speed;
-       else if (IS_PINEVIEW(dev))
-               dev_priv->display.get_display_clock_speed =
-                       pnv_get_display_clock_speed;
        else if (IS_I915GM(dev))
                dev_priv->display.get_display_clock_speed =
                        i915gm_get_display_clock_speed;
@@ -14844,7 +15072,7 @@ static void intel_init_display(struct drm_device *dev)
                        dev_priv->display.modeset_calc_cdclk =
                                broadwell_modeset_calc_cdclk;
                }
-       } else if (IS_VALLEYVIEW(dev)) {
+       } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
                dev_priv->display.modeset_commit_cdclk =
                        valleyview_modeset_commit_cdclk;
                dev_priv->display.modeset_calc_cdclk =
@@ -15076,12 +15304,97 @@ static void i915_disable_vga(struct drm_device *dev)
 
 void intel_modeset_init_hw(struct drm_device *dev)
 {
+       struct drm_i915_private *dev_priv = dev->dev_private;
+
        intel_update_cdclk(dev);
+
+       dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq;
+
        intel_prepare_ddi(dev);
        intel_init_clock_gating(dev);
        intel_enable_gt_powersave(dev);
 }
 
+/*
+ * Calculate what we think the watermarks should be for the state we've read
+ * out of the hardware and then immediately program those watermarks so that
+ * we ensure the hardware settings match our internal state.
+ *
+ * We can calculate what we think WM's should be by creating a duplicate of the
+ * current state (which was constructed during hardware readout) and running it
+ * through the atomic check code to calculate new watermark values in the
+ * state object.
+ */
+static void sanitize_watermarks(struct drm_device *dev)
+{
+       struct drm_i915_private *dev_priv = to_i915(dev);
+       struct drm_atomic_state *state;
+       struct drm_crtc *crtc;
+       struct drm_crtc_state *cstate;
+       struct drm_modeset_acquire_ctx ctx;
+       int ret;
+       int i;
+
+       /* Only supported on platforms that use atomic watermark design */
+       if (!dev_priv->display.optimize_watermarks)
+               return;
+
+       /*
+        * We need to hold connection_mutex before calling duplicate_state so
+        * that the connector loop is protected.
+        */
+       drm_modeset_acquire_init(&ctx, 0);
+retry:
+       ret = drm_modeset_lock(&dev->mode_config.connection_mutex, &ctx);
+       if (ret == -EDEADLK) {
+               drm_modeset_backoff(&ctx);
+               goto retry;
+       } else if (WARN_ON(ret)) {
+               return;
+       }
+
+       state = drm_atomic_helper_duplicate_state(dev, &ctx);
+       if (WARN_ON(IS_ERR(state)))
+               return;
+
+       /*
+        * Hardware readout is the only time we don't want to calculate
+        * intermediate watermarks (since we don't trust the current
+        * watermarks).
+        */
+       to_intel_atomic_state(state)->skip_intermediate_wm = true;
+
+       ret = intel_atomic_check(dev, state);
+       if (ret) {
+               /*
+                * If we fail here, it means that the hardware appears to be
+                * programmed in a way that shouldn't be possible, given our
+                * understanding of watermark requirements.  This might mean a
+                * mistake in the hardware readout code or a mistake in the
+                * watermark calculations for a given platform.  Raise a WARN
+                * so that this is noticeable.
+                *
+                * If this actually happens, we'll have to just leave the
+                * BIOS-programmed watermarks untouched and hope for the best.
+                */
+               WARN(true, "Could not determine valid watermarks for inherited state\n");
+               return;
+       }
+
+       /* Write calculated watermark values back */
+       to_i915(dev)->wm.config = to_intel_atomic_state(state)->wm_config;
+       for_each_crtc_in_state(state, crtc, cstate, i) {
+               struct intel_crtc_state *cs = to_intel_crtc_state(cstate);
+
+               cs->wm.need_postvbl_update = true;
+               dev_priv->display.optimize_watermarks(cs);
+       }
+
+       drm_atomic_state_free(state);
+       drm_modeset_drop_locks(&ctx);
+       drm_modeset_acquire_fini(&ctx);
+}
+
 void intel_modeset_init(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
@@ -15202,6 +15515,13 @@ void intel_modeset_init(struct drm_device *dev)
                 */
                intel_find_initial_plane_obj(crtc, &plane_config);
        }
+
+       /*
+        * Make sure hardware watermarks really match the state we read out.
+        * Note that we need to do this after reconstructing the BIOS fb's
+        * since the watermark calculation done here will use pstate->fb.
+        */
+       sanitize_watermarks(dev);
 }
 
 static void intel_enable_pipe_a(struct drm_device *dev)
@@ -15473,16 +15793,40 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
        struct intel_connector *connector;
        int i;
 
+       dev_priv->active_crtcs = 0;
+
        for_each_intel_crtc(dev, crtc) {
-               __drm_atomic_helper_crtc_destroy_state(&crtc->base, crtc->base.state);
-               memset(crtc->config, 0, sizeof(*crtc->config));
-               crtc->config->base.crtc = &crtc->base;
+               struct intel_crtc_state *crtc_state = crtc->config;
+               int pixclk = 0;
 
-               crtc->active = dev_priv->display.get_pipe_config(crtc,
-                                                                crtc->config);
+               __drm_atomic_helper_crtc_destroy_state(&crtc->base, &crtc_state->base);
+               memset(crtc_state, 0, sizeof(*crtc_state));
+               crtc_state->base.crtc = &crtc->base;
 
-               crtc->base.state->active = crtc->active;
-               crtc->base.enabled = crtc->active;
+               crtc_state->base.active = crtc_state->base.enable =
+                       dev_priv->display.get_pipe_config(crtc, crtc_state);
+
+               crtc->base.enabled = crtc_state->base.enable;
+               crtc->active = crtc_state->base.active;
+
+               if (crtc_state->base.active) {
+                       dev_priv->active_crtcs |= 1 << crtc->pipe;
+
+                       if (IS_BROADWELL(dev_priv)) {
+                               pixclk = ilk_pipe_pixel_rate(crtc_state);
+
+                               /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
+                               if (crtc_state->ips_enabled)
+                                       pixclk = DIV_ROUND_UP(pixclk * 100, 95);
+                       } else if (IS_VALLEYVIEW(dev_priv) ||
+                                  IS_CHERRYVIEW(dev_priv) ||
+                                  IS_BROXTON(dev_priv))
+                               pixclk = crtc_state->base.adjusted_mode.crtc_clock;
+                       else
+                               WARN_ON(dev_priv->display.modeset_calc_cdclk);
+               }
+
+               dev_priv->min_pixclk[crtc->pipe] = pixclk;
 
                readout_plane_state(crtc);
 
@@ -15618,7 +15962,7 @@ intel_modeset_setup_hw_state(struct drm_device *dev)
                pll->on = false;
        }
 
-       if (IS_VALLEYVIEW(dev))
+       if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
                vlv_wm_get_hw_state(dev);
        else if (IS_GEN9(dev))
                skl_wm_get_hw_state(dev);
@@ -15741,7 +16085,7 @@ void intel_connector_unregister(struct intel_connector *intel_connector)
 void intel_modeset_cleanup(struct drm_device *dev)
 {
        struct drm_i915_private *dev_priv = dev->dev_private;
-       struct drm_connector *connector;
+       struct intel_connector *connector;
 
        intel_disable_gt_powersave(dev);
 
@@ -15768,12 +16112,8 @@ void intel_modeset_cleanup(struct drm_device *dev)
        flush_scheduled_work();
 
        /* destroy the backlight and sysfs files before encoders/connectors */
-       list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-               struct intel_connector *intel_connector;
-
-               intel_connector = to_intel_connector(connector);
-               intel_connector->unregister(intel_connector);
-       }
+       for_each_intel_connector(dev, connector)
+               connector->unregister(connector);
 
        drm_mode_config_cleanup(dev);