drivers/gpu/drm/i915/intel_display.c

   1 /*
   2  * Copyright © 2006-2007 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21  * DEALINGS IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *      Eric Anholt <eric@anholt.net>
  25  */
  26
  27 #include <linux/dmi.h>
  28 #include <linux/module.h>
  29 #include <linux/input.h>
  30 #include <linux/i2c.h>
  31 #include <linux/kernel.h>
  32 #include <linux/slab.h>
  33 #include <linux/vgaarb.h>
  34 #include <drm/drm_edid.h>
  35 #include <drm/drmP.h>
  36 #include "intel_drv.h"
  37 #include <drm/i915_drm.h>
  38 #include "i915_drv.h"
  39 #include "i915_trace.h"
  40 #include <drm/drm_atomic.h>
  41 #include <drm/drm_atomic_helper.h>
  42 #include <drm/drm_dp_helper.h>
  43 #include <drm/drm_crtc_helper.h>
  44 #include <drm/drm_plane_helper.h>
  45 #include <drm/drm_rect.h>
  46 #include <linux/dma_remapping.h>
  47
  48 /* Primary plane formats for gen <= 3 */
  49 static const uint32_t i8xx_primary_formats[] = {
  50         DRM_FORMAT_C8,
  51         DRM_FORMAT_RGB565,
  52         DRM_FORMAT_XRGB1555,
  53         DRM_FORMAT_XRGB8888,
  54 };
  55
  56 /* Primary plane formats for gen >= 4 */
  57 static const uint32_t i965_primary_formats[] = {
  58         DRM_FORMAT_C8,
  59         DRM_FORMAT_RGB565,
  60         DRM_FORMAT_XRGB8888,
  61         DRM_FORMAT_XBGR8888,
  62         DRM_FORMAT_XRGB2101010,
  63         DRM_FORMAT_XBGR2101010,
  64 };
  65
  66 static const uint32_t skl_primary_formats[] = {
  67         DRM_FORMAT_C8,
  68         DRM_FORMAT_RGB565,
  69         DRM_FORMAT_XRGB8888,
  70         DRM_FORMAT_XBGR8888,
  71         DRM_FORMAT_ARGB8888,
  72         DRM_FORMAT_ABGR8888,
  73         DRM_FORMAT_XRGB2101010,
  74         DRM_FORMAT_XBGR2101010,
  75 };
  76
  77 /* Cursor formats */
  78 static const uint32_t intel_cursor_formats[] = {
  79         DRM_FORMAT_ARGB8888,
  80 };
  81
  82 static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
  83
  84 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
  85                                 struct intel_crtc_state *pipe_config);
  86 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
  87                                    struct intel_crtc_state *pipe_config);
  88
  89 static int intel_set_mode(struct drm_atomic_state *state);
  90 static int intel_framebuffer_init(struct drm_device *dev,
  91                                   struct intel_framebuffer *ifb,
  92                                   struct drm_mode_fb_cmd2 *mode_cmd,
  93                                   struct drm_i915_gem_object *obj);
  94 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc);
  95 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc);
  96 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
  97                                          struct intel_link_m_n *m_n,
  98                                          struct intel_link_m_n *m2_n2);
  99 static void ironlake_set_pipeconf(struct drm_crtc *crtc);
 100 static void haswell_set_pipeconf(struct drm_crtc *crtc);
 101 static void intel_set_pipe_csc(struct drm_crtc *crtc);
 102 static void vlv_prepare_pll(struct intel_crtc *crtc,
 103                             const struct intel_crtc_state *pipe_config);
 104 static void chv_prepare_pll(struct intel_crtc *crtc,
 105                             const struct intel_crtc_state *pipe_config);
 106 static void intel_begin_crtc_commit(struct drm_crtc *crtc);
 107 static void intel_finish_crtc_commit(struct drm_crtc *crtc);
 108 static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
 109         struct intel_crtc_state *crtc_state);
 110 static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state,
 111                            int num_connectors);
 112
 113 static struct intel_encoder *intel_find_encoder(struct intel_connector *connector, int pipe)
 114 {
 115         if (!connector->mst_port)
 116                 return connector->encoder;
 117         else
 118                 return &connector->mst_port->mst_encoders[pipe]->base;
 119 }
 120
 121 typedef struct {
 122         int     min, max;
 123 } intel_range_t;
 124
 125 typedef struct {
 126         int     dot_limit;
 127         int     p2_slow, p2_fast;
 128 } intel_p2_t;
 129
 130 typedef struct intel_limit intel_limit_t;
 131 struct intel_limit {
 132         intel_range_t   dot, vco, n, m, m1, m2, p, p1;
 133         intel_p2_t          p2;
 134 };
 135
 136 int
 137 intel_pch_rawclk(struct drm_device *dev)
 138 {
 139         struct drm_i915_private *dev_priv = dev->dev_private;
 140
 141         WARN_ON(!HAS_PCH_SPLIT(dev));
 142
 143         return I915_READ(PCH_RAWCLK_FREQ) & RAWCLK_FREQ_MASK;
 144 }
 145
 146 static inline u32 /* units of 100MHz */
 147 intel_fdi_link_freq(struct drm_device *dev)
 148 {
 149         if (IS_GEN5(dev)) {
 150                 struct drm_i915_private *dev_priv = dev->dev_private;
 151                 return (I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2;
 152         } else
 153                 return 27;
 154 }
 155
 156 static const intel_limit_t intel_limits_i8xx_dac = {
 157         .dot = { .min = 25000, .max = 350000 },
 158         .vco = { .min = 908000, .max = 1512000 },
 159         .n = { .min = 2, .max = 16 },
 160         .m = { .min = 96, .max = 140 },
 161         .m1 = { .min = 18, .max = 26 },
 162         .m2 = { .min = 6, .max = 16 },
 163         .p = { .min = 4, .max = 128 },
 164         .p1 = { .min = 2, .max = 33 },
 165         .p2 = { .dot_limit = 165000,
 166                 .p2_slow = 4, .p2_fast = 2 },
 167 };
 168
 169 static const intel_limit_t intel_limits_i8xx_dvo = {
 170         .dot = { .min = 25000, .max = 350000 },
 171         .vco = { .min = 908000, .max = 1512000 },
 172         .n = { .min = 2, .max = 16 },
 173         .m = { .min = 96, .max = 140 },
 174         .m1 = { .min = 18, .max = 26 },
 175         .m2 = { .min = 6, .max = 16 },
 176         .p = { .min = 4, .max = 128 },
 177         .p1 = { .min = 2, .max = 33 },
 178         .p2 = { .dot_limit = 165000,
 179                 .p2_slow = 4, .p2_fast = 4 },
 180 };
 181
 182 static const intel_limit_t intel_limits_i8xx_lvds = {
 183         .dot = { .min = 25000, .max = 350000 },
 184         .vco = { .min = 908000, .max = 1512000 },
 185         .n = { .min = 2, .max = 16 },
 186         .m = { .min = 96, .max = 140 },
 187         .m1 = { .min = 18, .max = 26 },
 188         .m2 = { .min = 6, .max = 16 },
 189         .p = { .min = 4, .max = 128 },
 190         .p1 = { .min = 1, .max = 6 },
 191         .p2 = { .dot_limit = 165000,
 192                 .p2_slow = 14, .p2_fast = 7 },
 193 };
 194
 195 static const intel_limit_t intel_limits_i9xx_sdvo = {
 196         .dot = { .min = 20000, .max = 400000 },
 197         .vco = { .min = 1400000, .max = 2800000 },
 198         .n = { .min = 1, .max = 6 },
 199         .m = { .min = 70, .max = 120 },
 200         .m1 = { .min = 8, .max = 18 },
 201         .m2 = { .min = 3, .max = 7 },
 202         .p = { .min = 5, .max = 80 },
 203         .p1 = { .min = 1, .max = 8 },
 204         .p2 = { .dot_limit = 200000,
 205                 .p2_slow = 10, .p2_fast = 5 },
 206 };
 207
 208 static const intel_limit_t intel_limits_i9xx_lvds = {
 209         .dot = { .min = 20000, .max = 400000 },
 210         .vco = { .min = 1400000, .max = 2800000 },
 211         .n = { .min = 1, .max = 6 },
 212         .m = { .min = 70, .max = 120 },
 213         .m1 = { .min = 8, .max = 18 },
 214         .m2 = { .min = 3, .max = 7 },
 215         .p = { .min = 7, .max = 98 },
 216         .p1 = { .min = 1, .max = 8 },
 217         .p2 = { .dot_limit = 112000,
 218                 .p2_slow = 14, .p2_fast = 7 },
 219 };
 220
 221
 222 static const intel_limit_t intel_limits_g4x_sdvo = {
 223         .dot = { .min = 25000, .max = 270000 },
 224         .vco = { .min = 1750000, .max = 3500000},
 225         .n = { .min = 1, .max = 4 },
 226         .m = { .min = 104, .max = 138 },
 227         .m1 = { .min = 17, .max = 23 },
 228         .m2 = { .min = 5, .max = 11 },
 229         .p = { .min = 10, .max = 30 },
 230         .p1 = { .min = 1, .max = 3},
 231         .p2 = { .dot_limit = 270000,
 232                 .p2_slow = 10,
 233                 .p2_fast = 10
 234         },
 235 };
 236
 237 static const intel_limit_t intel_limits_g4x_hdmi = {
 238         .dot = { .min = 22000, .max = 400000 },
 239         .vco = { .min = 1750000, .max = 3500000},
 240         .n = { .min = 1, .max = 4 },
 241         .m = { .min = 104, .max = 138 },
 242         .m1 = { .min = 16, .max = 23 },
 243         .m2 = { .min = 5, .max = 11 },
 244         .p = { .min = 5, .max = 80 },
 245         .p1 = { .min = 1, .max = 8},
 246         .p2 = { .dot_limit = 165000,
 247                 .p2_slow = 10, .p2_fast = 5 },
 248 };
 249
 250 static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
 251         .dot = { .min = 20000, .max = 115000 },
 252         .vco = { .min = 1750000, .max = 3500000 },
 253         .n = { .min = 1, .max = 3 },
 254         .m = { .min = 104, .max = 138 },
 255         .m1 = { .min = 17, .max = 23 },
 256         .m2 = { .min = 5, .max = 11 },
 257         .p = { .min = 28, .max = 112 },
 258         .p1 = { .min = 2, .max = 8 },
 259         .p2 = { .dot_limit = 0,
 260                 .p2_slow = 14, .p2_fast = 14
 261         },
 262 };
 263
 264 static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
 265         .dot = { .min = 80000, .max = 224000 },
 266         .vco = { .min = 1750000, .max = 3500000 },
 267         .n = { .min = 1, .max = 3 },
 268         .m = { .min = 104, .max = 138 },
 269         .m1 = { .min = 17, .max = 23 },
 270         .m2 = { .min = 5, .max = 11 },
 271         .p = { .min = 14, .max = 42 },
 272         .p1 = { .min = 2, .max = 6 },
 273         .p2 = { .dot_limit = 0,
 274                 .p2_slow = 7, .p2_fast = 7
 275         },
 276 };
 277
 278 static const intel_limit_t intel_limits_pineview_sdvo = {
 279         .dot = { .min = 20000, .max = 400000},
 280         .vco = { .min = 1700000, .max = 3500000 },
 281         /* Pineview's Ncounter is a ring counter */
 282         .n = { .min = 3, .max = 6 },
 283         .m = { .min = 2, .max = 256 },
 284         /* Pineview only has one combined m divider, which we treat as m2. */
 285         .m1 = { .min = 0, .max = 0 },
 286         .m2 = { .min = 0, .max = 254 },
 287         .p = { .min = 5, .max = 80 },
 288         .p1 = { .min = 1, .max = 8 },
 289         .p2 = { .dot_limit = 200000,
 290                 .p2_slow = 10, .p2_fast = 5 },
 291 };
 292
 293 static const intel_limit_t intel_limits_pineview_lvds = {
 294         .dot = { .min = 20000, .max = 400000 },
 295         .vco = { .min = 1700000, .max = 3500000 },
 296         .n = { .min = 3, .max = 6 },
 297         .m = { .min = 2, .max = 256 },
 298         .m1 = { .min = 0, .max = 0 },
 299         .m2 = { .min = 0, .max = 254 },
 300         .p = { .min = 7, .max = 112 },
 301         .p1 = { .min = 1, .max = 8 },
 302         .p2 = { .dot_limit = 112000,
 303                 .p2_slow = 14, .p2_fast = 14 },
 304 };
 305
 306 /* Ironlake / Sandybridge
 307  *
 308  * We calculate clock using (register_value + 2) for N/M1/M2, so here
 309  * the range value for them is (actual_value - 2).
 310  */
 311 static const intel_limit_t intel_limits_ironlake_dac = {
 312         .dot = { .min = 25000, .max = 350000 },
 313         .vco = { .min = 1760000, .max = 3510000 },
 314         .n = { .min = 1, .max = 5 },
 315         .m = { .min = 79, .max = 127 },
 316         .m1 = { .min = 12, .max = 22 },
 317         .m2 = { .min = 5, .max = 9 },
 318         .p = { .min = 5, .max = 80 },
 319         .p1 = { .min = 1, .max = 8 },
 320         .p2 = { .dot_limit = 225000,
 321                 .p2_slow = 10, .p2_fast = 5 },
 322 };
 323
 324 static const intel_limit_t intel_limits_ironlake_single_lvds = {
 325         .dot = { .min = 25000, .max = 350000 },
 326         .vco = { .min = 1760000, .max = 3510000 },
 327         .n = { .min = 1, .max = 3 },
 328         .m = { .min = 79, .max = 118 },
 329         .m1 = { .min = 12, .max = 22 },
 330         .m2 = { .min = 5, .max = 9 },
 331         .p = { .min = 28, .max = 112 },
 332         .p1 = { .min = 2, .max = 8 },
 333         .p2 = { .dot_limit = 225000,
 334                 .p2_slow = 14, .p2_fast = 14 },
 335 };
 336
 337 static const intel_limit_t intel_limits_ironlake_dual_lvds = {
 338         .dot = { .min = 25000, .max = 350000 },
 339         .vco = { .min = 1760000, .max = 3510000 },
 340         .n = { .min = 1, .max = 3 },
 341         .m = { .min = 79, .max = 127 },
 342         .m1 = { .min = 12, .max = 22 },
 343         .m2 = { .min = 5, .max = 9 },
 344         .p = { .min = 14, .max = 56 },
 345         .p1 = { .min = 2, .max = 8 },
 346         .p2 = { .dot_limit = 225000,
 347                 .p2_slow = 7, .p2_fast = 7 },
 348 };
 349
 350 /* LVDS 100mhz refclk limits. */
 351 static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
 352         .dot = { .min = 25000, .max = 350000 },
 353         .vco = { .min = 1760000, .max = 3510000 },
 354         .n = { .min = 1, .max = 2 },
 355         .m = { .min = 79, .max = 126 },
 356         .m1 = { .min = 12, .max = 22 },
 357         .m2 = { .min = 5, .max = 9 },
 358         .p = { .min = 28, .max = 112 },
 359         .p1 = { .min = 2, .max = 8 },
 360         .p2 = { .dot_limit = 225000,
 361                 .p2_slow = 14, .p2_fast = 14 },
 362 };
 363
 364 static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
 365         .dot = { .min = 25000, .max = 350000 },
 366         .vco = { .min = 1760000, .max = 3510000 },
 367         .n = { .min = 1, .max = 3 },
 368         .m = { .min = 79, .max = 126 },
 369         .m1 = { .min = 12, .max = 22 },
 370         .m2 = { .min = 5, .max = 9 },
 371         .p = { .min = 14, .max = 42 },
 372         .p1 = { .min = 2, .max = 6 },
 373         .p2 = { .dot_limit = 225000,
 374                 .p2_slow = 7, .p2_fast = 7 },
 375 };
 376
 377 static const intel_limit_t intel_limits_vlv = {
 378          /*
 379           * These are the data rate limits (measured in fast clocks)
 380           * since those are the strictest limits we have. The fast
 381           * clock and actual rate limits are more relaxed, so checking
 382           * them would make no difference.
 383           */
 384         .dot = { .min = 25000 * 5, .max = 270000 * 5 },
 385         .vco = { .min = 4000000, .max = 6000000 },
 386         .n = { .min = 1, .max = 7 },
 387         .m1 = { .min = 2, .max = 3 },
 388         .m2 = { .min = 11, .max = 156 },
 389         .p1 = { .min = 2, .max = 3 },
 390         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 391 };
 392
 393 static const intel_limit_t intel_limits_chv = {
 394         /*
 395          * These are the data rate limits (measured in fast clocks)
 396          * since those are the strictest limits we have.  The fast
 397          * clock and actual rate limits are more relaxed, so checking
 398          * them would make no difference.
 399          */
 400         .dot = { .min = 25000 * 5, .max = 540000 * 5},
 401         .vco = { .min = 4800000, .max = 6480000 },
 402         .n = { .min = 1, .max = 1 },
 403         .m1 = { .min = 2, .max = 2 },
 404         .m2 = { .min = 24 << 22, .max = 175 << 22 },
 405         .p1 = { .min = 2, .max = 4 },
 406         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 407 };
 408
 409 static const intel_limit_t intel_limits_bxt = {
 410         /* FIXME: find real dot limits */
 411         .dot = { .min = 0, .max = INT_MAX },
 412         .vco = { .min = 4800000, .max = 6480000 },
 413         .n = { .min = 1, .max = 1 },
 414         .m1 = { .min = 2, .max = 2 },
 415         /* FIXME: find real m2 limits */
 416         .m2 = { .min = 2 << 22, .max = 255 << 22 },
 417         .p1 = { .min = 2, .max = 4 },
 418         .p2 = { .p2_slow = 1, .p2_fast = 20 },
 419 };
 420
 421 static void vlv_clock(int refclk, intel_clock_t *clock)
 422 {
 423         clock->m = clock->m1 * clock->m2;
 424         clock->p = clock->p1 * clock->p2;
 425         if (WARN_ON(clock->n == 0 || clock->p == 0))
 426                 return;
 427         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 428         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 429 }
 430
 431 static bool
 432 needs_modeset(struct drm_crtc_state *state)
 433 {
 434         return state->mode_changed || state->active_changed;
 435 }
 436
 437 /**
 438  * Returns whether any output on the specified pipe is of the specified type
 439  */
 440 bool intel_pipe_has_type(struct intel_crtc *crtc, enum intel_output_type type)
 441 {
 442         struct drm_device *dev = crtc->base.dev;
 443         struct intel_encoder *encoder;
 444
 445         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
 446                 if (encoder->type == type)
 447                         return true;
 448
 449         return false;
 450 }
 451
 452 /**
 453  * Returns whether any output on the specified pipe will have the specified
 454  * type after a staged modeset is complete, i.e., the same as
 455  * intel_pipe_has_type() but looking at encoder->new_crtc instead of
 456  * encoder->crtc.
 457  */
 458 static bool intel_pipe_will_have_type(const struct intel_crtc_state *crtc_state,
 459                                       int type)
 460 {
 461         struct drm_atomic_state *state = crtc_state->base.state;
 462         struct drm_connector *connector;
 463         struct drm_connector_state *connector_state;
 464         struct intel_encoder *encoder;
 465         int i, num_connectors = 0;
 466
 467         for_each_connector_in_state(state, connector, connector_state, i) {
 468                 if (connector_state->crtc != crtc_state->base.crtc)
 469                         continue;
 470
 471                 num_connectors++;
 472
 473                 encoder = to_intel_encoder(connector_state->best_encoder);
 474                 if (encoder->type == type)
 475                         return true;
 476         }
 477
 478         WARN_ON(num_connectors == 0);
 479
 480         return false;
 481 }
 482
 483 static const intel_limit_t *
 484 intel_ironlake_limit(struct intel_crtc_state *crtc_state, int refclk)
 485 {
 486         struct drm_device *dev = crtc_state->base.crtc->dev;
 487         const intel_limit_t *limit;
 488
 489         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 490                 if (intel_is_dual_link_lvds(dev)) {
 491                         if (refclk == 100000)
 492                                 limit = &intel_limits_ironlake_dual_lvds_100m;
 493                         else
 494                                 limit = &intel_limits_ironlake_dual_lvds;
 495                 } else {
 496                         if (refclk == 100000)
 497                                 limit = &intel_limits_ironlake_single_lvds_100m;
 498                         else
 499                                 limit = &intel_limits_ironlake_single_lvds;
 500                 }
 501         } else
 502                 limit = &intel_limits_ironlake_dac;
 503
 504         return limit;
 505 }
 506
 507 static const intel_limit_t *
 508 intel_g4x_limit(struct intel_crtc_state *crtc_state)
 509 {
 510         struct drm_device *dev = crtc_state->base.crtc->dev;
 511         const intel_limit_t *limit;
 512
 513         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 514                 if (intel_is_dual_link_lvds(dev))
 515                         limit = &intel_limits_g4x_dual_channel_lvds;
 516                 else
 517                         limit = &intel_limits_g4x_single_channel_lvds;
 518         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI) ||
 519                    intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
 520                 limit = &intel_limits_g4x_hdmi;
 521         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO)) {
 522                 limit = &intel_limits_g4x_sdvo;
 523         } else /* The option is for other outputs */
 524                 limit = &intel_limits_i9xx_sdvo;
 525
 526         return limit;
 527 }
 528
 529 static const intel_limit_t *
 530 intel_limit(struct intel_crtc_state *crtc_state, int refclk)
 531 {
 532         struct drm_device *dev = crtc_state->base.crtc->dev;
 533         const intel_limit_t *limit;
 534
 535         if (IS_BROXTON(dev))
 536                 limit = &intel_limits_bxt;
 537         else if (HAS_PCH_SPLIT(dev))
 538                 limit = intel_ironlake_limit(crtc_state, refclk);
 539         else if (IS_G4X(dev)) {
 540                 limit = intel_g4x_limit(crtc_state);
 541         } else if (IS_PINEVIEW(dev)) {
 542                 if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
 543                         limit = &intel_limits_pineview_lvds;
 544                 else
 545                         limit = &intel_limits_pineview_sdvo;
 546         } else if (IS_CHERRYVIEW(dev)) {
 547                 limit = &intel_limits_chv;
 548         } else if (IS_VALLEYVIEW(dev)) {
 549                 limit = &intel_limits_vlv;
 550         } else if (!IS_GEN2(dev)) {
 551                 if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
 552                         limit = &intel_limits_i9xx_lvds;
 553                 else
 554                         limit = &intel_limits_i9xx_sdvo;
 555         } else {
 556                 if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
 557                         limit = &intel_limits_i8xx_lvds;
 558                 else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO))
 559                         limit = &intel_limits_i8xx_dvo;
 560                 else
 561                         limit = &intel_limits_i8xx_dac;
 562         }
 563         return limit;
 564 }
 565
 566 /* m1 is reserved as 0 in Pineview, n is a ring counter */
 567 static void pineview_clock(int refclk, intel_clock_t *clock)
 568 {
 569         clock->m = clock->m2 + 2;
 570         clock->p = clock->p1 * clock->p2;
 571         if (WARN_ON(clock->n == 0 || clock->p == 0))
 572                 return;
 573         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n);
 574         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 575 }
 576
 577 static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
 578 {
 579         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 580 }
 581
 582 static void i9xx_clock(int refclk, intel_clock_t *clock)
 583 {
 584         clock->m = i9xx_dpll_compute_m(clock);
 585         clock->p = clock->p1 * clock->p2;
 586         if (WARN_ON(clock->n + 2 == 0 || clock->p == 0))
 587                 return;
 588         clock->vco = DIV_ROUND_CLOSEST(refclk * clock->m, clock->n + 2);
 589         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 590 }
 591
 592 static void chv_clock(int refclk, intel_clock_t *clock)
 593 {
 594         clock->m = clock->m1 * clock->m2;
 595         clock->p = clock->p1 * clock->p2;
 596         if (WARN_ON(clock->n == 0 || clock->p == 0))
 597                 return;
 598         clock->vco = DIV_ROUND_CLOSEST_ULL((uint64_t)refclk * clock->m,
 599                         clock->n << 22);
 600         clock->dot = DIV_ROUND_CLOSEST(clock->vco, clock->p);
 601 }
 602
 603 #define INTELPllInvalid(s)   do { /* DRM_DEBUG(s); */ return false; } while (0)
 604 /**
 605  * Returns whether the given set of divisors are valid for a given refclk with
 606  * the given connectors.
 607  */
 608
 609 static bool intel_PLL_is_valid(struct drm_device *dev,
 610                                const intel_limit_t *limit,
 611                                const intel_clock_t *clock)
 612 {
 613         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
 614                 INTELPllInvalid("n out of range\n");
 615         if (clock->p1  < limit->p1.min  || limit->p1.max  < clock->p1)
 616                 INTELPllInvalid("p1 out of range\n");
 617         if (clock->m2  < limit->m2.min  || limit->m2.max  < clock->m2)
 618                 INTELPllInvalid("m2 out of range\n");
 619         if (clock->m1  < limit->m1.min  || limit->m1.max  < clock->m1)
 620                 INTELPllInvalid("m1 out of range\n");
 621
 622         if (!IS_PINEVIEW(dev) && !IS_VALLEYVIEW(dev) && !IS_BROXTON(dev))
 623                 if (clock->m1 <= clock->m2)
 624                         INTELPllInvalid("m1 <= m2\n");
 625
 626         if (!IS_VALLEYVIEW(dev) && !IS_BROXTON(dev)) {
 627                 if (clock->p < limit->p.min || limit->p.max < clock->p)
 628                         INTELPllInvalid("p out of range\n");
 629                 if (clock->m < limit->m.min || limit->m.max < clock->m)
 630                         INTELPllInvalid("m out of range\n");
 631         }
 632
 633         if (clock->vco < limit->vco.min || limit->vco.max < clock->vco)
 634                 INTELPllInvalid("vco out of range\n");
 635         /* XXX: We may need to be checking "Dot clock" depending on the multiplier,
 636          * connector, etc., rather than just a single range.
 637          */
 638         if (clock->dot < limit->dot.min || limit->dot.max < clock->dot)
 639                 INTELPllInvalid("dot out of range\n");
 640
 641         return true;
 642 }
 643
 644 static int
 645 i9xx_select_p2_div(const intel_limit_t *limit,
 646                    const struct intel_crtc_state *crtc_state,
 647                    int target)
 648 {
 649         struct drm_device *dev = crtc_state->base.crtc->dev;
 650
 651         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
 652                 /*
 653                  * For LVDS just rely on its current settings for dual-channel.
 654                  * We haven't figured out how to reliably set up different
 655                  * single/dual channel state, if we even can.
 656                  */
 657                 if (intel_is_dual_link_lvds(dev))
 658                         return limit->p2.p2_fast;
 659                 else
 660                         return limit->p2.p2_slow;
 661         } else {
 662                 if (target < limit->p2.dot_limit)
 663                         return limit->p2.p2_slow;
 664                 else
 665                         return limit->p2.p2_fast;
 666         }
 667 }
 668
 669 static bool
 670 i9xx_find_best_dpll(const intel_limit_t *limit,
 671                     struct intel_crtc_state *crtc_state,
 672                     int target, int refclk, intel_clock_t *match_clock,
 673                     intel_clock_t *best_clock)
 674 {
 675         struct drm_device *dev = crtc_state->base.crtc->dev;
 676         intel_clock_t clock;
 677         int err = target;
 678
 679         memset(best_clock, 0, sizeof(*best_clock));
 680
 681         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 682
 683         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 684              clock.m1++) {
 685                 for (clock.m2 = limit->m2.min;
 686                      clock.m2 <= limit->m2.max; clock.m2++) {
 687                         if (clock.m2 >= clock.m1)
 688                                 break;
 689                         for (clock.n = limit->n.min;
 690                              clock.n <= limit->n.max; clock.n++) {
 691                                 for (clock.p1 = limit->p1.min;
 692                                         clock.p1 <= limit->p1.max; clock.p1++) {
 693                                         int this_err;
 694
 695                                         i9xx_clock(refclk, &clock);
 696                                         if (!intel_PLL_is_valid(dev, limit,
 697                                                                 &clock))
 698                                                 continue;
 699                                         if (match_clock &&
 700                                             clock.p != match_clock->p)
 701                                                 continue;
 702
 703                                         this_err = abs(clock.dot - target);
 704                                         if (this_err < err) {
 705                                                 *best_clock = clock;
 706                                                 err = this_err;
 707                                         }
 708                                 }
 709                         }
 710                 }
 711         }
 712
 713         return (err != target);
 714 }
 715
 716 static bool
 717 pnv_find_best_dpll(const intel_limit_t *limit,
 718                    struct intel_crtc_state *crtc_state,
 719                    int target, int refclk, intel_clock_t *match_clock,
 720                    intel_clock_t *best_clock)
 721 {
 722         struct drm_device *dev = crtc_state->base.crtc->dev;
 723         intel_clock_t clock;
 724         int err = target;
 725
 726         memset(best_clock, 0, sizeof(*best_clock));
 727
 728         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 729
 730         for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max;
 731              clock.m1++) {
 732                 for (clock.m2 = limit->m2.min;
 733                      clock.m2 <= limit->m2.max; clock.m2++) {
 734                         for (clock.n = limit->n.min;
 735                              clock.n <= limit->n.max; clock.n++) {
 736                                 for (clock.p1 = limit->p1.min;
 737                                         clock.p1 <= limit->p1.max; clock.p1++) {
 738                                         int this_err;
 739
 740                                         pineview_clock(refclk, &clock);
 741                                         if (!intel_PLL_is_valid(dev, limit,
 742                                                                 &clock))
 743                                                 continue;
 744                                         if (match_clock &&
 745                                             clock.p != match_clock->p)
 746                                                 continue;
 747
 748                                         this_err = abs(clock.dot - target);
 749                                         if (this_err < err) {
 750                                                 *best_clock = clock;
 751                                                 err = this_err;
 752                                         }
 753                                 }
 754                         }
 755                 }
 756         }
 757
 758         return (err != target);
 759 }
 760
 761 static bool
 762 g4x_find_best_dpll(const intel_limit_t *limit,
 763                    struct intel_crtc_state *crtc_state,
 764                    int target, int refclk, intel_clock_t *match_clock,
 765                    intel_clock_t *best_clock)
 766 {
 767         struct drm_device *dev = crtc_state->base.crtc->dev;
 768         intel_clock_t clock;
 769         int max_n;
 770         bool found = false;
 771         /* approximately equals target * 0.00585 */
 772         int err_most = (target >> 8) + (target >> 9);
 773
 774         memset(best_clock, 0, sizeof(*best_clock));
 775
 776         clock.p2 = i9xx_select_p2_div(limit, crtc_state, target);
 777
 778         max_n = limit->n.max;
 779         /* based on hardware requirement, prefer smaller n to precision */
 780         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 781                 /* based on hardware requirement, prefere larger m1,m2 */
 782                 for (clock.m1 = limit->m1.max;
 783                      clock.m1 >= limit->m1.min; clock.m1--) {
 784                         for (clock.m2 = limit->m2.max;
 785                              clock.m2 >= limit->m2.min; clock.m2--) {
 786                                 for (clock.p1 = limit->p1.max;
 787                                      clock.p1 >= limit->p1.min; clock.p1--) {
 788                                         int this_err;
 789
 790                                         i9xx_clock(refclk, &clock);
 791                                         if (!intel_PLL_is_valid(dev, limit,
 792                                                                 &clock))
 793                                                 continue;
 794
 795                                         this_err = abs(clock.dot - target);
 796                                         if (this_err < err_most) {
 797                                                 *best_clock = clock;
 798                                                 err_most = this_err;
 799                                                 max_n = clock.n;
 800                                                 found = true;
 801                                         }
 802                                 }
 803                         }
 804                 }
 805         }
 806         return found;
 807 }
 808
 809 /*
 810  * Check if the calculated PLL configuration is more optimal compared to the
 811  * best configuration and error found so far. Return the calculated error.
 812  */
 813 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
 814                                const intel_clock_t *calculated_clock,
 815                                const intel_clock_t *best_clock,
 816                                unsigned int best_error_ppm,
 817                                unsigned int *error_ppm)
 818 {
 819         /*
 820          * For CHV ignore the error and consider only the P value.
 821          * Prefer a bigger P value based on HW requirements.
 822          */
 823         if (IS_CHERRYVIEW(dev)) {
 824                 *error_ppm = 0;
 825
 826                 return calculated_clock->p > best_clock->p;
 827         }
 828
 829         if (WARN_ON_ONCE(!target_freq))
 830                 return false;
 831
 832         *error_ppm = div_u64(1000000ULL *
 833                                 abs(target_freq - calculated_clock->dot),
 834                              target_freq);
 835         /*
 836          * Prefer a better P value over a better (smaller) error if the error
 837          * is small. Ensure this preference for future configurations too by
 838          * setting the error to 0.
 839          */
 840         if (*error_ppm < 100 && calculated_clock->p > best_clock->p) {
 841                 *error_ppm = 0;
 842
 843                 return true;
 844         }
 845
 846         return *error_ppm + 10 < best_error_ppm;
 847 }
 848
 849 static bool
 850 vlv_find_best_dpll(const intel_limit_t *limit,
 851                    struct intel_crtc_state *crtc_state,
 852                    int target, int refclk, intel_clock_t *match_clock,
 853                    intel_clock_t *best_clock)
 854 {
 855         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 856         struct drm_device *dev = crtc->base.dev;
 857         intel_clock_t clock;
 858         unsigned int bestppm = 1000000;
 859         /* min update 19.2 MHz */
 860         int max_n = min(limit->n.max, refclk / 19200);
 861         bool found = false;
 862
 863         target *= 5; /* fast clock */
 864
 865         memset(best_clock, 0, sizeof(*best_clock));
 866
 867         /* based on hardware requirement, prefer smaller n to precision */
 868         for (clock.n = limit->n.min; clock.n <= max_n; clock.n++) {
 869                 for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 870                         for (clock.p2 = limit->p2.p2_fast; clock.p2 >= limit->p2.p2_slow;
 871                              clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 872                                 clock.p = clock.p1 * clock.p2;
 873                                 /* based on hardware requirement, prefer bigger m1,m2 values */
 874                                 for (clock.m1 = limit->m1.min; clock.m1 <= limit->m1.max; clock.m1++) {
 875                                         unsigned int ppm;
 876
 877                                         clock.m2 = DIV_ROUND_CLOSEST(target * clock.p * clock.n,
 878                                                                      refclk * clock.m1);
 879
 880                                         vlv_clock(refclk, &clock);
 881
 882                                         if (!intel_PLL_is_valid(dev, limit,
 883                                                                 &clock))
 884                                                 continue;
 885
 886                                         if (!vlv_PLL_is_optimal(dev, target,
 887                                                                 &clock,
 888                                                                 best_clock,
 889                                                                 bestppm, &ppm))
 890                                                 continue;
 891
 892                                         *best_clock = clock;
 893                                         bestppm = ppm;
 894                                         found = true;
 895                                 }
 896                         }
 897                 }
 898         }
 899
 900         return found;
 901 }
 902
 903 static bool
 904 chv_find_best_dpll(const intel_limit_t *limit,
 905                    struct intel_crtc_state *crtc_state,
 906                    int target, int refclk, intel_clock_t *match_clock,
 907                    intel_clock_t *best_clock)
 908 {
 909         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 910         struct drm_device *dev = crtc->base.dev;
 911         unsigned int best_error_ppm;
 912         intel_clock_t clock;
 913         uint64_t m2;
 914         int found = false;
 915
 916         memset(best_clock, 0, sizeof(*best_clock));
 917         best_error_ppm = 1000000;
 918
 919         /*
 920          * Based on hardware doc, the n always set to 1, and m1 always
 921          * set to 2.  If requires to support 200Mhz refclk, we need to
 922          * revisit this because n may not 1 anymore.
 923          */
 924         clock.n = 1, clock.m1 = 2;
 925         target *= 5;    /* fast clock */
 926
 927         for (clock.p1 = limit->p1.max; clock.p1 >= limit->p1.min; clock.p1--) {
 928                 for (clock.p2 = limit->p2.p2_fast;
 929                                 clock.p2 >= limit->p2.p2_slow;
 930                                 clock.p2 -= clock.p2 > 10 ? 2 : 1) {
 931                         unsigned int error_ppm;
 932
 933                         clock.p = clock.p1 * clock.p2;
 934
 935                         m2 = DIV_ROUND_CLOSEST_ULL(((uint64_t)target * clock.p *
 936                                         clock.n) << 22, refclk * clock.m1);
 937
 938                         if (m2 > INT_MAX/clock.m1)
 939                                 continue;
 940
 941                         clock.m2 = m2;
 942
 943                         chv_clock(refclk, &clock);
 944
 945                         if (!intel_PLL_is_valid(dev, limit, &clock))
 946                                 continue;
 947
 948                         if (!vlv_PLL_is_optimal(dev, target, &clock, best_clock,
 949                                                 best_error_ppm, &error_ppm))
 950                                 continue;
 951
 952                         *best_clock = clock;
 953                         best_error_ppm = error_ppm;
 954                         found = true;
 955                 }
 956         }
 957
 958         return found;
 959 }
 960
 961 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
 962                         intel_clock_t *best_clock)
 963 {
 964         int refclk = i9xx_get_refclk(crtc_state, 0);
 965
 966         return chv_find_best_dpll(intel_limit(crtc_state, refclk), crtc_state,
 967                                   target_clock, refclk, NULL, best_clock);
 968 }
 969
 970 bool intel_crtc_active(struct drm_crtc *crtc)
 971 {
 972         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 973
 974         /* Be paranoid as we can arrive here with only partial
 975          * state retrieved from the hardware during setup.
 976          *
 977          * We can ditch the adjusted_mode.crtc_clock check as soon
 978          * as Haswell has gained clock readout/fastboot support.
 979          *
 980          * We can ditch the crtc->primary->fb check as soon as we can
 981          * properly reconstruct framebuffers.
 982          *
 983          * FIXME: The intel_crtc->active here should be switched to
 984          * crtc->state->active once we have proper CRTC states wired up
 985          * for atomic.
 986          */
 987         return intel_crtc->active && crtc->primary->state->fb &&
 988                 intel_crtc->config->base.adjusted_mode.crtc_clock;
 989 }
 990
 991 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
 992                                              enum pipe pipe)
 993 {
 994         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 995         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 996
 997         return intel_crtc->config->cpu_transcoder;
 998 }
 999
1000 static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
1001 {
1002         struct drm_i915_private *dev_priv = dev->dev_private;
1003         u32 reg = PIPEDSL(pipe);
1004         u32 line1, line2;
1005         u32 line_mask;
1006
1007         if (IS_GEN2(dev))
1008                 line_mask = DSL_LINEMASK_GEN2;
1009         else
1010                 line_mask = DSL_LINEMASK_GEN3;
1011
1012         line1 = I915_READ(reg) & line_mask;
1013         mdelay(5);
1014         line2 = I915_READ(reg) & line_mask;
1015
1016         return line1 == line2;
1017 }
1018
1019 /*
1020  * intel_wait_for_pipe_off - wait for pipe to turn off
1021  * @crtc: crtc whose pipe to wait for
1022  *
1023  * After disabling a pipe, we can't wait for vblank in the usual way,
1024  * spinning on the vblank interrupt status bit, since we won't actually
1025  * see an interrupt when the pipe is disabled.
1026  *
1027  * On Gen4 and above:
1028  *   wait for the pipe register state bit to turn off
1029  *
1030  * Otherwise:
1031  *   wait for the display line value to settle (it usually
1032  *   ends up stopping at the start of the next frame).
1033  *
1034  */
1035 static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
1036 {
1037         struct drm_device *dev = crtc->base.dev;
1038         struct drm_i915_private *dev_priv = dev->dev_private;
1039         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
1040         enum pipe pipe = crtc->pipe;
1041
1042         if (INTEL_INFO(dev)->gen >= 4) {
1043                 int reg = PIPECONF(cpu_transcoder);
1044
1045                 /* Wait for the Pipe State to go off */
1046                 if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0,
1047                              100))
1048                         WARN(1, "pipe_off wait timed out\n");
1049         } else {
1050                 /* Wait for the display line to settle */
1051                 if (wait_for(pipe_dsl_stopped(dev, pipe), 100))
1052                         WARN(1, "pipe_off wait timed out\n");
1053         }
1054 }
1055
1056 /*
1057  * ibx_digital_port_connected - is the specified port connected?
1058  * @dev_priv: i915 private structure
1059  * @port: the port to test
1060  *
1061  * Returns true if @port is connected, false otherwise.
1062  */
1063 bool ibx_digital_port_connected(struct drm_i915_private *dev_priv,
1064                                 struct intel_digital_port *port)
1065 {
1066         u32 bit;
1067
1068         if (HAS_PCH_IBX(dev_priv->dev)) {
1069                 switch (port->port) {
1070                 case PORT_B:
1071                         bit = SDE_PORTB_HOTPLUG;
1072                         break;
1073                 case PORT_C:
1074                         bit = SDE_PORTC_HOTPLUG;
1075                         break;
1076                 case PORT_D:
1077                         bit = SDE_PORTD_HOTPLUG;
1078                         break;
1079                 default:
1080                         return true;
1081                 }
1082         } else {
1083                 switch (port->port) {
1084                 case PORT_B:
1085                         bit = SDE_PORTB_HOTPLUG_CPT;
1086                         break;
1087                 case PORT_C:
1088                         bit = SDE_PORTC_HOTPLUG_CPT;
1089                         break;
1090                 case PORT_D:
1091                         bit = SDE_PORTD_HOTPLUG_CPT;
1092                         break;
1093                 default:
1094                         return true;
1095                 }
1096         }
1097
1098         return I915_READ(SDEISR) & bit;
1099 }
1100
1101 static const char *state_string(bool enabled)
1102 {
1103         return enabled ? "on" : "off";
1104 }
1105
1106 /* Only for pre-ILK configs */
1107 void assert_pll(struct drm_i915_private *dev_priv,
1108                 enum pipe pipe, bool state)
1109 {
1110         int reg;
1111         u32 val;
1112         bool cur_state;
1113
1114         reg = DPLL(pipe);
1115         val = I915_READ(reg);
1116         cur_state = !!(val & DPLL_VCO_ENABLE);
1117         I915_STATE_WARN(cur_state != state,
1118              "PLL state assertion failure (expected %s, current %s)\n",
1119              state_string(state), state_string(cur_state));
1120 }
1121
1122 /* XXX: the dsi pll is shared between MIPI DSI ports */
1123 static void assert_dsi_pll(struct drm_i915_private *dev_priv, bool state)
1124 {
1125         u32 val;
1126         bool cur_state;
1127
1128         mutex_lock(&dev_priv->sb_lock);
1129         val = vlv_cck_read(dev_priv, CCK_REG_DSI_PLL_CONTROL);
1130         mutex_unlock(&dev_priv->sb_lock);
1131
1132         cur_state = val & DSI_PLL_VCO_EN;
1133         I915_STATE_WARN(cur_state != state,
1134              "DSI PLL state assertion failure (expected %s, current %s)\n",
1135              state_string(state), state_string(cur_state));
1136 }
1137 #define assert_dsi_pll_enabled(d) assert_dsi_pll(d, true)
1138 #define assert_dsi_pll_disabled(d) assert_dsi_pll(d, false)
1139
1140 struct intel_shared_dpll *
1141 intel_crtc_to_shared_dpll(struct intel_crtc *crtc)
1142 {
1143         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
1144
1145         if (crtc->config->shared_dpll < 0)
1146                 return NULL;
1147
1148         return &dev_priv->shared_dplls[crtc->config->shared_dpll];
1149 }
1150
1151 /* For ILK+ */
1152 void assert_shared_dpll(struct drm_i915_private *dev_priv,
1153                         struct intel_shared_dpll *pll,
1154                         bool state)
1155 {
1156         bool cur_state;
1157         struct intel_dpll_hw_state hw_state;
1158
1159         if (WARN (!pll,
1160                   "asserting DPLL %s with no DPLL\n", state_string(state)))
1161                 return;
1162
1163         cur_state = pll->get_hw_state(dev_priv, pll, &hw_state);
1164         I915_STATE_WARN(cur_state != state,
1165              "%s assertion failure (expected %s, current %s)\n",
1166              pll->name, state_string(state), state_string(cur_state));
1167 }
1168
1169 static void assert_fdi_tx(struct drm_i915_private *dev_priv,
1170                           enum pipe pipe, bool state)
1171 {
1172         int reg;
1173         u32 val;
1174         bool cur_state;
1175         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1176                                                                       pipe);
1177
1178         if (HAS_DDI(dev_priv->dev)) {
1179                 /* DDI does not have a specific FDI_TX register */
1180                 reg = TRANS_DDI_FUNC_CTL(cpu_transcoder);
1181                 val = I915_READ(reg);
1182                 cur_state = !!(val & TRANS_DDI_FUNC_ENABLE);
1183         } else {
1184                 reg = FDI_TX_CTL(pipe);
1185                 val = I915_READ(reg);
1186                 cur_state = !!(val & FDI_TX_ENABLE);
1187         }
1188         I915_STATE_WARN(cur_state != state,
1189              "FDI TX state assertion failure (expected %s, current %s)\n",
1190              state_string(state), state_string(cur_state));
1191 }
1192 #define assert_fdi_tx_enabled(d, p) assert_fdi_tx(d, p, true)
1193 #define assert_fdi_tx_disabled(d, p) assert_fdi_tx(d, p, false)
1194
1195 static void assert_fdi_rx(struct drm_i915_private *dev_priv,
1196                           enum pipe pipe, bool state)
1197 {
1198         int reg;
1199         u32 val;
1200         bool cur_state;
1201
1202         reg = FDI_RX_CTL(pipe);
1203         val = I915_READ(reg);
1204         cur_state = !!(val & FDI_RX_ENABLE);
1205         I915_STATE_WARN(cur_state != state,
1206              "FDI RX state assertion failure (expected %s, current %s)\n",
1207              state_string(state), state_string(cur_state));
1208 }
1209 #define assert_fdi_rx_enabled(d, p) assert_fdi_rx(d, p, true)
1210 #define assert_fdi_rx_disabled(d, p) assert_fdi_rx(d, p, false)
1211
1212 static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
1213                                       enum pipe pipe)
1214 {
1215         int reg;
1216         u32 val;
1217
1218         /* ILK FDI PLL is always enabled */
1219         if (INTEL_INFO(dev_priv->dev)->gen == 5)
1220                 return;
1221
1222         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
1223         if (HAS_DDI(dev_priv->dev))
1224                 return;
1225
1226         reg = FDI_TX_CTL(pipe);
1227         val = I915_READ(reg);
1228         I915_STATE_WARN(!(val & FDI_TX_PLL_ENABLE), "FDI TX PLL assertion failure, should be active but is disabled\n");
1229 }
1230
1231 void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
1232                        enum pipe pipe, bool state)
1233 {
1234         int reg;
1235         u32 val;
1236         bool cur_state;
1237
1238         reg = FDI_RX_CTL(pipe);
1239         val = I915_READ(reg);
1240         cur_state = !!(val & FDI_RX_PLL_ENABLE);
1241         I915_STATE_WARN(cur_state != state,
1242              "FDI RX PLL assertion failure (expected %s, current %s)\n",
1243              state_string(state), state_string(cur_state));
1244 }
1245
1246 void assert_panel_unlocked(struct drm_i915_private *dev_priv,
1247                            enum pipe pipe)
1248 {
1249         struct drm_device *dev = dev_priv->dev;
1250         int pp_reg;
1251         u32 val;
1252         enum pipe panel_pipe = PIPE_A;
1253         bool locked = true;
1254
1255         if (WARN_ON(HAS_DDI(dev)))
1256                 return;
1257
1258         if (HAS_PCH_SPLIT(dev)) {
1259                 u32 port_sel;
1260
1261                 pp_reg = PCH_PP_CONTROL;
1262                 port_sel = I915_READ(PCH_PP_ON_DELAYS) & PANEL_PORT_SELECT_MASK;
1263
1264                 if (port_sel == PANEL_PORT_SELECT_LVDS &&
1265                     I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
1266                         panel_pipe = PIPE_B;
1267                 /* XXX: else fix for eDP */
1268         } else if (IS_VALLEYVIEW(dev)) {
1269                 /* presumably write lock depends on pipe, not port select */
1270                 pp_reg = VLV_PIPE_PP_CONTROL(pipe);
1271                 panel_pipe = pipe;
1272         } else {
1273                 pp_reg = PP_CONTROL;
1274                 if (I915_READ(LVDS) & LVDS_PIPEB_SELECT)
1275                         panel_pipe = PIPE_B;
1276         }
1277
1278         val = I915_READ(pp_reg);
1279         if (!(val & PANEL_POWER_ON) ||
1280             ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
1281                 locked = false;
1282
1283         I915_STATE_WARN(panel_pipe == pipe && locked,
1284              "panel assertion failure, pipe %c regs locked\n",
1285              pipe_name(pipe));
1286 }
1287
1288 static void assert_cursor(struct drm_i915_private *dev_priv,
1289                           enum pipe pipe, bool state)
1290 {
1291         struct drm_device *dev = dev_priv->dev;
1292         bool cur_state;
1293
1294         if (IS_845G(dev) || IS_I865G(dev))
1295                 cur_state = I915_READ(_CURACNTR) & CURSOR_ENABLE;
1296         else
1297                 cur_state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
1298
1299         I915_STATE_WARN(cur_state != state,
1300              "cursor on pipe %c assertion failure (expected %s, current %s)\n",
1301              pipe_name(pipe), state_string(state), state_string(cur_state));
1302 }
1303 #define assert_cursor_enabled(d, p) assert_cursor(d, p, true)
1304 #define assert_cursor_disabled(d, p) assert_cursor(d, p, false)
1305
1306 void assert_pipe(struct drm_i915_private *dev_priv,
1307                  enum pipe pipe, bool state)
1308 {
1309         int reg;
1310         u32 val;
1311         bool cur_state;
1312         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
1313                                                                       pipe);
1314
1315         /* if we need the pipe quirk it must be always on */
1316         if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
1317             (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
1318                 state = true;
1319
1320         if (!intel_display_power_is_enabled(dev_priv,
1321                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder))) {
1322                 cur_state = false;
1323         } else {
1324                 reg = PIPECONF(cpu_transcoder);
1325                 val = I915_READ(reg);
1326                 cur_state = !!(val & PIPECONF_ENABLE);
1327         }
1328
1329         I915_STATE_WARN(cur_state != state,
1330              "pipe %c assertion failure (expected %s, current %s)\n",
1331              pipe_name(pipe), state_string(state), state_string(cur_state));
1332 }
1333
1334 static void assert_plane(struct drm_i915_private *dev_priv,
1335                          enum plane plane, bool state)
1336 {
1337         int reg;
1338         u32 val;
1339         bool cur_state;
1340
1341         reg = DSPCNTR(plane);
1342         val = I915_READ(reg);
1343         cur_state = !!(val & DISPLAY_PLANE_ENABLE);
1344         I915_STATE_WARN(cur_state != state,
1345              "plane %c assertion failure (expected %s, current %s)\n",
1346              plane_name(plane), state_string(state), state_string(cur_state));
1347 }
1348
1349 #define assert_plane_enabled(d, p) assert_plane(d, p, true)
1350 #define assert_plane_disabled(d, p) assert_plane(d, p, false)
1351
1352 static void assert_planes_disabled(struct drm_i915_private *dev_priv,
1353                                    enum pipe pipe)
1354 {
1355         struct drm_device *dev = dev_priv->dev;
1356         int reg, i;
1357         u32 val;
1358         int cur_pipe;
1359
1360         /* Primary planes are fixed to pipes on gen4+ */
1361         if (INTEL_INFO(dev)->gen >= 4) {
1362                 reg = DSPCNTR(pipe);
1363                 val = I915_READ(reg);
1364                 I915_STATE_WARN(val & DISPLAY_PLANE_ENABLE,
1365                      "plane %c assertion failure, should be disabled but not\n",
1366                      plane_name(pipe));
1367                 return;
1368         }
1369
1370         /* Need to check both planes against the pipe */
1371         for_each_pipe(dev_priv, i) {
1372                 reg = DSPCNTR(i);
1373                 val = I915_READ(reg);
1374                 cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
1375                         DISPPLANE_SEL_PIPE_SHIFT;
1376                 I915_STATE_WARN((val & DISPLAY_PLANE_ENABLE) && pipe == cur_pipe,
1377                      "plane %c assertion failure, should be off on pipe %c but is still active\n",
1378                      plane_name(i), pipe_name(pipe));
1379         }
1380 }
1381
1382 static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
1383                                     enum pipe pipe)
1384 {
1385         struct drm_device *dev = dev_priv->dev;
1386         int reg, sprite;
1387         u32 val;
1388
1389         if (INTEL_INFO(dev)->gen >= 9) {
1390                 for_each_sprite(dev_priv, pipe, sprite) {
1391                         val = I915_READ(PLANE_CTL(pipe, sprite));
1392                         I915_STATE_WARN(val & PLANE_CTL_ENABLE,
1393                              "plane %d assertion failure, should be off on pipe %c but is still active\n",
1394                              sprite, pipe_name(pipe));
1395                 }
1396         } else if (IS_VALLEYVIEW(dev)) {
1397                 for_each_sprite(dev_priv, pipe, sprite) {
1398                         reg = SPCNTR(pipe, sprite);
1399                         val = I915_READ(reg);
1400                         I915_STATE_WARN(val & SP_ENABLE,
1401                              "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1402                              sprite_name(pipe, sprite), pipe_name(pipe));
1403                 }
1404         } else if (INTEL_INFO(dev)->gen >= 7) {
1405                 reg = SPRCTL(pipe);
1406                 val = I915_READ(reg);
1407                 I915_STATE_WARN(val & SPRITE_ENABLE,
1408                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1409                      plane_name(pipe), pipe_name(pipe));
1410         } else if (INTEL_INFO(dev)->gen >= 5) {
1411                 reg = DVSCNTR(pipe);
1412                 val = I915_READ(reg);
1413                 I915_STATE_WARN(val & DVS_ENABLE,
1414                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
1415                      plane_name(pipe), pipe_name(pipe));
1416         }
1417 }
1418
1419 static void assert_vblank_disabled(struct drm_crtc *crtc)
1420 {
1421         if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
1422                 drm_crtc_vblank_put(crtc);
1423 }
1424
1425 static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
1426 {
1427         u32 val;
1428         bool enabled;
1429
1430         I915_STATE_WARN_ON(!(HAS_PCH_IBX(dev_priv->dev) || HAS_PCH_CPT(dev_priv->dev)));
1431
1432         val = I915_READ(PCH_DREF_CONTROL);
1433         enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK |
1434                             DREF_SUPERSPREAD_SOURCE_MASK));
1435         I915_STATE_WARN(!enabled, "PCH refclk assertion failure, should be active but is disabled\n");
1436 }
1437
1438 static void assert_pch_transcoder_disabled(struct drm_i915_private *dev_priv,
1439                                            enum pipe pipe)
1440 {
1441         int reg;
1442         u32 val;
1443         bool enabled;
1444
1445         reg = PCH_TRANSCONF(pipe);
1446         val = I915_READ(reg);
1447         enabled = !!(val & TRANS_ENABLE);
1448         I915_STATE_WARN(enabled,
1449              "transcoder assertion failed, should be off on pipe %c but is still active\n",
1450              pipe_name(pipe));
1451 }
1452
1453 static bool dp_pipe_enabled(struct drm_i915_private *dev_priv,
1454                             enum pipe pipe, u32 port_sel, u32 val)
1455 {
1456         if ((val & DP_PORT_EN) == 0)
1457                 return false;
1458
1459         if (HAS_PCH_CPT(dev_priv->dev)) {
1460                 u32     trans_dp_ctl_reg = TRANS_DP_CTL(pipe);
1461                 u32     trans_dp_ctl = I915_READ(trans_dp_ctl_reg);
1462                 if ((trans_dp_ctl & TRANS_DP_PORT_SEL_MASK) != port_sel)
1463                         return false;
1464         } else if (IS_CHERRYVIEW(dev_priv->dev)) {
1465                 if ((val & DP_PIPE_MASK_CHV) != DP_PIPE_SELECT_CHV(pipe))
1466                         return false;
1467         } else {
1468                 if ((val & DP_PIPE_MASK) != (pipe << 30))
1469                         return false;
1470         }
1471         return true;
1472 }
1473
1474 static bool hdmi_pipe_enabled(struct drm_i915_private *dev_priv,
1475                               enum pipe pipe, u32 val)
1476 {
1477         if ((val & SDVO_ENABLE) == 0)
1478                 return false;
1479
1480         if (HAS_PCH_CPT(dev_priv->dev)) {
1481                 if ((val & SDVO_PIPE_SEL_MASK_CPT) != SDVO_PIPE_SEL_CPT(pipe))
1482                         return false;
1483         } else if (IS_CHERRYVIEW(dev_priv->dev)) {
1484                 if ((val & SDVO_PIPE_SEL_MASK_CHV) != SDVO_PIPE_SEL_CHV(pipe))
1485                         return false;
1486         } else {
1487                 if ((val & SDVO_PIPE_SEL_MASK) != SDVO_PIPE_SEL(pipe))
1488                         return false;
1489         }
1490         return true;
1491 }
1492
1493 static bool lvds_pipe_enabled(struct drm_i915_private *dev_priv,
1494                               enum pipe pipe, u32 val)
1495 {
1496         if ((val & LVDS_PORT_EN) == 0)
1497                 return false;
1498
1499         if (HAS_PCH_CPT(dev_priv->dev)) {
1500                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1501                         return false;
1502         } else {
1503                 if ((val & LVDS_PIPE_MASK) != LVDS_PIPE(pipe))
1504                         return false;
1505         }
1506         return true;
1507 }
1508
1509 static bool adpa_pipe_enabled(struct drm_i915_private *dev_priv,
1510                               enum pipe pipe, u32 val)
1511 {
1512         if ((val & ADPA_DAC_ENABLE) == 0)
1513                 return false;
1514         if (HAS_PCH_CPT(dev_priv->dev)) {
1515                 if ((val & PORT_TRANS_SEL_MASK) != PORT_TRANS_SEL_CPT(pipe))
1516                         return false;
1517         } else {
1518                 if ((val & ADPA_PIPE_SELECT_MASK) != ADPA_PIPE_SELECT(pipe))
1519                         return false;
1520         }
1521         return true;
1522 }
1523
1524 static void assert_pch_dp_disabled(struct drm_i915_private *dev_priv,
1525                                    enum pipe pipe, int reg, u32 port_sel)
1526 {
1527         u32 val = I915_READ(reg);
1528         I915_STATE_WARN(dp_pipe_enabled(dev_priv, pipe, port_sel, val),
1529              "PCH DP (0x%08x) enabled on transcoder %c, should be disabled\n",
1530              reg, pipe_name(pipe));
1531
1532         I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & DP_PORT_EN) == 0
1533              && (val & DP_PIPEB_SELECT),
1534              "IBX PCH dp port still using transcoder B\n");
1535 }
1536
1537 static void assert_pch_hdmi_disabled(struct drm_i915_private *dev_priv,
1538                                      enum pipe pipe, int reg)
1539 {
1540         u32 val = I915_READ(reg);
1541         I915_STATE_WARN(hdmi_pipe_enabled(dev_priv, pipe, val),
1542              "PCH HDMI (0x%08x) enabled on transcoder %c, should be disabled\n",
1543              reg, pipe_name(pipe));
1544
1545         I915_STATE_WARN(HAS_PCH_IBX(dev_priv->dev) && (val & SDVO_ENABLE) == 0
1546              && (val & SDVO_PIPE_B_SELECT),
1547              "IBX PCH hdmi port still using transcoder B\n");
1548 }
1549
1550 static void assert_pch_ports_disabled(struct drm_i915_private *dev_priv,
1551                                       enum pipe pipe)
1552 {
1553         int reg;
1554         u32 val;
1555
1556         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_B, TRANS_DP_PORT_SEL_B);
1557         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_C, TRANS_DP_PORT_SEL_C);
1558         assert_pch_dp_disabled(dev_priv, pipe, PCH_DP_D, TRANS_DP_PORT_SEL_D);
1559
1560         reg = PCH_ADPA;
1561         val = I915_READ(reg);
1562         I915_STATE_WARN(adpa_pipe_enabled(dev_priv, pipe, val),
1563              "PCH VGA enabled on transcoder %c, should be disabled\n",
1564              pipe_name(pipe));
1565
1566         reg = PCH_LVDS;
1567         val = I915_READ(reg);
1568         I915_STATE_WARN(lvds_pipe_enabled(dev_priv, pipe, val),
1569              "PCH LVDS enabled on transcoder %c, should be disabled\n",
1570              pipe_name(pipe));
1571
1572         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMIB);
1573         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMIC);
1574         assert_pch_hdmi_disabled(dev_priv, pipe, PCH_HDMID);
1575 }
1576
1577 static void intel_init_dpio(struct drm_device *dev)
1578 {
1579         struct drm_i915_private *dev_priv = dev->dev_private;
1580
1581         if (!IS_VALLEYVIEW(dev))
1582                 return;
1583
1584         /*
1585          * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
1586          * CHV x1 PHY (DP/HDMI D)
1587          * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
1588          */
1589         if (IS_CHERRYVIEW(dev)) {
1590                 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
1591                 DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
1592         } else {
1593                 DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
1594         }
1595 }
1596
1597 static void vlv_enable_pll(struct intel_crtc *crtc,
1598                            const struct intel_crtc_state *pipe_config)
1599 {
1600         struct drm_device *dev = crtc->base.dev;
1601         struct drm_i915_private *dev_priv = dev->dev_private;
1602         int reg = DPLL(crtc->pipe);
1603         u32 dpll = pipe_config->dpll_hw_state.dpll;
1604
1605         assert_pipe_disabled(dev_priv, crtc->pipe);
1606
1607         /* No really, not for ILK+ */
1608         BUG_ON(!IS_VALLEYVIEW(dev_priv->dev));
1609
1610         /* PLL is protected by panel, make sure we can write it */
1611         if (IS_MOBILE(dev_priv->dev))
1612                 assert_panel_unlocked(dev_priv, crtc->pipe);
1613
1614         I915_WRITE(reg, dpll);
1615         POSTING_READ(reg);
1616         udelay(150);
1617
1618         if (wait_for(((I915_READ(reg) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1))
1619                 DRM_ERROR("DPLL %d failed to lock\n", crtc->pipe);
1620
1621         I915_WRITE(DPLL_MD(crtc->pipe), pipe_config->dpll_hw_state.dpll_md);
1622         POSTING_READ(DPLL_MD(crtc->pipe));
1623
1624         /* We do this three times for luck */
1625         I915_WRITE(reg, dpll);
1626         POSTING_READ(reg);
1627         udelay(150); /* wait for warmup */
1628         I915_WRITE(reg, dpll);
1629         POSTING_READ(reg);
1630         udelay(150); /* wait for warmup */
1631         I915_WRITE(reg, dpll);
1632         POSTING_READ(reg);
1633         udelay(150); /* wait for warmup */
1634 }
1635
1636 static void chv_enable_pll(struct intel_crtc *crtc,
1637                            const struct intel_crtc_state *pipe_config)
1638 {
1639         struct drm_device *dev = crtc->base.dev;
1640         struct drm_i915_private *dev_priv = dev->dev_private;
1641         int pipe = crtc->pipe;
1642         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1643         u32 tmp;
1644
1645         assert_pipe_disabled(dev_priv, crtc->pipe);
1646
1647         BUG_ON(!IS_CHERRYVIEW(dev_priv->dev));
1648
1649         mutex_lock(&dev_priv->sb_lock);
1650
1651         /* Enable back the 10bit clock to display controller */
1652         tmp = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1653         tmp |= DPIO_DCLKP_EN;
1654         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), tmp);
1655
1656         mutex_unlock(&dev_priv->sb_lock);
1657
1658         /*
1659          * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
1660          */
1661         udelay(1);
1662
1663         /* Enable PLL */
1664         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
1665
1666         /* Check PLL is locked */
1667         if (wait_for(((I915_READ(DPLL(pipe)) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1))
1668                 DRM_ERROR("PLL %d failed to lock\n", pipe);
1669
1670         /* not sure when this should be written */
1671         I915_WRITE(DPLL_MD(pipe), pipe_config->dpll_hw_state.dpll_md);
1672         POSTING_READ(DPLL_MD(pipe));
1673 }
1674
1675 static int intel_num_dvo_pipes(struct drm_device *dev)
1676 {
1677         struct intel_crtc *crtc;
1678         int count = 0;
1679
1680         for_each_intel_crtc(dev, crtc)
1681                 count += crtc->base.state->active &&
1682                         intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO);
1683
1684         return count;
1685 }
1686
1687 static void i9xx_enable_pll(struct intel_crtc *crtc)
1688 {
1689         struct drm_device *dev = crtc->base.dev;
1690         struct drm_i915_private *dev_priv = dev->dev_private;
1691         int reg = DPLL(crtc->pipe);
1692         u32 dpll = crtc->config->dpll_hw_state.dpll;
1693
1694         assert_pipe_disabled(dev_priv, crtc->pipe);
1695
1696         /* No really, not for ILK+ */
1697         BUG_ON(INTEL_INFO(dev)->gen >= 5);
1698
1699         /* PLL is protected by panel, make sure we can write it */
1700         if (IS_MOBILE(dev) && !IS_I830(dev))
1701                 assert_panel_unlocked(dev_priv, crtc->pipe);
1702
1703         /* Enable DVO 2x clock on both PLLs if necessary */
1704         if (IS_I830(dev) && intel_num_dvo_pipes(dev) > 0) {
1705                 /*
1706                  * It appears to be important that we don't enable this
1707                  * for the current pipe before otherwise configuring the
1708                  * PLL. No idea how this should be handled if multiple
1709                  * DVO outputs are enabled simultaneosly.
1710                  */
1711                 dpll |= DPLL_DVO_2X_MODE;
1712                 I915_WRITE(DPLL(!crtc->pipe),
1713                            I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
1714         }
1715
1716         /* Wait for the clocks to stabilize. */
1717         POSTING_READ(reg);
1718         udelay(150);
1719
1720         if (INTEL_INFO(dev)->gen >= 4) {
1721                 I915_WRITE(DPLL_MD(crtc->pipe),
1722                            crtc->config->dpll_hw_state.dpll_md);
1723         } else {
1724                 /* The pixel multiplier can only be updated once the
1725                  * DPLL is enabled and the clocks are stable.
1726                  *
1727                  * So write it again.
1728                  */
1729                 I915_WRITE(reg, dpll);
1730         }
1731
1732         /* We do this three times for luck */
1733         I915_WRITE(reg, dpll);
1734         POSTING_READ(reg);
1735         udelay(150); /* wait for warmup */
1736         I915_WRITE(reg, dpll);
1737         POSTING_READ(reg);
1738         udelay(150); /* wait for warmup */
1739         I915_WRITE(reg, dpll);
1740         POSTING_READ(reg);
1741         udelay(150); /* wait for warmup */
1742 }
1743
1744 /**
1745  * i9xx_disable_pll - disable a PLL
1746  * @dev_priv: i915 private structure
1747  * @pipe: pipe PLL to disable
1748  *
1749  * Disable the PLL for @pipe, making sure the pipe is off first.
1750  *
1751  * Note!  This is for pre-ILK only.
1752  */
1753 static void i9xx_disable_pll(struct intel_crtc *crtc)
1754 {
1755         struct drm_device *dev = crtc->base.dev;
1756         struct drm_i915_private *dev_priv = dev->dev_private;
1757         enum pipe pipe = crtc->pipe;
1758
1759         /* Disable DVO 2x clock on both PLLs if necessary */
1760         if (IS_I830(dev) &&
1761             intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO) &&
1762             !intel_num_dvo_pipes(dev)) {
1763                 I915_WRITE(DPLL(PIPE_B),
1764                            I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
1765                 I915_WRITE(DPLL(PIPE_A),
1766                            I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
1767         }
1768
1769         /* Don't disable pipe or pipe PLLs if needed */
1770         if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
1771             (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
1772                 return;
1773
1774         /* Make sure the pipe isn't still relying on us */
1775         assert_pipe_disabled(dev_priv, pipe);
1776
1777         I915_WRITE(DPLL(pipe), 0);
1778         POSTING_READ(DPLL(pipe));
1779 }
1780
1781 static void vlv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1782 {
1783         u32 val = 0;
1784
1785         /* Make sure the pipe isn't still relying on us */
1786         assert_pipe_disabled(dev_priv, pipe);
1787
1788         /*
1789          * Leave integrated clock source and reference clock enabled for pipe B.
1790          * The latter is needed for VGA hotplug / manual detection.
1791          */
1792         if (pipe == PIPE_B)
1793                 val = DPLL_INTEGRATED_CRI_CLK_VLV | DPLL_REFA_CLK_ENABLE_VLV;
1794         I915_WRITE(DPLL(pipe), val);
1795         POSTING_READ(DPLL(pipe));
1796
1797 }
1798
1799 static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
1800 {
1801         enum dpio_channel port = vlv_pipe_to_channel(pipe);
1802         u32 val;
1803
1804         /* Make sure the pipe isn't still relying on us */
1805         assert_pipe_disabled(dev_priv, pipe);
1806
1807         /* Set PLL en = 0 */
1808         val = DPLL_SSC_REF_CLOCK_CHV | DPLL_REFA_CLK_ENABLE_VLV;
1809         if (pipe != PIPE_A)
1810                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
1811         I915_WRITE(DPLL(pipe), val);
1812         POSTING_READ(DPLL(pipe));
1813
1814         mutex_lock(&dev_priv->sb_lock);
1815
1816         /* Disable 10bit clock to display controller */
1817         val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port));
1818         val &= ~DPIO_DCLKP_EN;
1819         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
1820
1821         /* disable left/right clock distribution */
1822         if (pipe != PIPE_B) {
1823                 val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
1824                 val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
1825                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
1826         } else {
1827                 val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
1828                 val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
1829                 vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
1830         }
1831
1832         mutex_unlock(&dev_priv->sb_lock);
1833 }
1834
1835 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
1836                          struct intel_digital_port *dport,
1837                          unsigned int expected_mask)
1838 {
1839         u32 port_mask;
1840         int dpll_reg;
1841
1842         switch (dport->port) {
1843         case PORT_B:
1844                 port_mask = DPLL_PORTB_READY_MASK;
1845                 dpll_reg = DPLL(0);
1846                 break;
1847         case PORT_C:
1848                 port_mask = DPLL_PORTC_READY_MASK;
1849                 dpll_reg = DPLL(0);
1850                 expected_mask <<= 4;
1851                 break;
1852         case PORT_D:
1853                 port_mask = DPLL_PORTD_READY_MASK;
1854                 dpll_reg = DPIO_PHY_STATUS;
1855                 break;
1856         default:
1857                 BUG();
1858         }
1859
1860         if (wait_for((I915_READ(dpll_reg) & port_mask) == expected_mask, 1000))
1861                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
1862                      port_name(dport->port), I915_READ(dpll_reg) & port_mask, expected_mask);
1863 }
1864
1865 static void intel_prepare_shared_dpll(struct intel_crtc *crtc)
1866 {
1867         struct drm_device *dev = crtc->base.dev;
1868         struct drm_i915_private *dev_priv = dev->dev_private;
1869         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
1870
1871         if (WARN_ON(pll == NULL))
1872                 return;
1873
1874         WARN_ON(!pll->config.crtc_mask);
1875         if (pll->active == 0) {
1876                 DRM_DEBUG_DRIVER("setting up %s\n", pll->name);
1877                 WARN_ON(pll->on);
1878                 assert_shared_dpll_disabled(dev_priv, pll);
1879
1880                 pll->mode_set(dev_priv, pll);
1881         }
1882 }
1883
1884 /**
1885  * intel_enable_shared_dpll - enable PCH PLL
1886  * @dev_priv: i915 private structure
1887  * @pipe: pipe PLL to enable
1888  *
1889  * The PCH PLL needs to be enabled before the PCH transcoder, since it
1890  * drives the transcoder clock.
1891  */
1892 static void intel_enable_shared_dpll(struct intel_crtc *crtc)
1893 {
1894         struct drm_device *dev = crtc->base.dev;
1895         struct drm_i915_private *dev_priv = dev->dev_private;
1896         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
1897
1898         if (WARN_ON(pll == NULL))
1899                 return;
1900
1901         if (WARN_ON(pll->config.crtc_mask == 0))
1902                 return;
1903
1904         DRM_DEBUG_KMS("enable %s (active %d, on? %d) for crtc %d\n",
1905                       pll->name, pll->active, pll->on,
1906                       crtc->base.base.id);
1907
1908         if (pll->active++) {
1909                 WARN_ON(!pll->on);
1910                 assert_shared_dpll_enabled(dev_priv, pll);
1911                 return;
1912         }
1913         WARN_ON(pll->on);
1914
1915         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
1916
1917         DRM_DEBUG_KMS("enabling %s\n", pll->name);
1918         pll->enable(dev_priv, pll);
1919         pll->on = true;
1920 }
1921
1922 static void intel_disable_shared_dpll(struct intel_crtc *crtc)
1923 {
1924         struct drm_device *dev = crtc->base.dev;
1925         struct drm_i915_private *dev_priv = dev->dev_private;
1926         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
1927
1928         /* PCH only available on ILK+ */
1929         BUG_ON(INTEL_INFO(dev)->gen < 5);
1930         if (pll == NULL)
1931                 return;
1932
1933         if (WARN_ON(!(pll->config.crtc_mask & (1 << drm_crtc_index(&crtc->base)))))
1934                 return;
1935
1936         DRM_DEBUG_KMS("disable %s (active %d, on? %d) for crtc %d\n",
1937                       pll->name, pll->active, pll->on,
1938                       crtc->base.base.id);
1939
1940         if (WARN_ON(pll->active == 0)) {
1941                 assert_shared_dpll_disabled(dev_priv, pll);
1942                 return;
1943         }
1944
1945         assert_shared_dpll_enabled(dev_priv, pll);
1946         WARN_ON(!pll->on);
1947         if (--pll->active)
1948                 return;
1949
1950         DRM_DEBUG_KMS("disabling %s\n", pll->name);
1951         pll->disable(dev_priv, pll);
1952         pll->on = false;
1953
1954         intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
1955 }
1956
1957 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
1958                                            enum pipe pipe)
1959 {
1960         struct drm_device *dev = dev_priv->dev;
1961         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
1962         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
1963         uint32_t reg, val, pipeconf_val;
1964
1965         /* PCH only available on ILK+ */
1966         BUG_ON(!HAS_PCH_SPLIT(dev));
1967
1968         /* Make sure PCH DPLL is enabled */
1969         assert_shared_dpll_enabled(dev_priv,
1970                                    intel_crtc_to_shared_dpll(intel_crtc));
1971
1972         /* FDI must be feeding us bits for PCH ports */
1973         assert_fdi_tx_enabled(dev_priv, pipe);
1974         assert_fdi_rx_enabled(dev_priv, pipe);
1975
1976         if (HAS_PCH_CPT(dev)) {
1977                 /* Workaround: Set the timing override bit before enabling the
1978                  * pch transcoder. */
1979                 reg = TRANS_CHICKEN2(pipe);
1980                 val = I915_READ(reg);
1981                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
1982                 I915_WRITE(reg, val);
1983         }
1984
1985         reg = PCH_TRANSCONF(pipe);
1986         val = I915_READ(reg);
1987         pipeconf_val = I915_READ(PIPECONF(pipe));
1988
1989         if (HAS_PCH_IBX(dev_priv->dev)) {
1990                 /*
1991                  * Make the BPC in transcoder be consistent with
1992                  * that in pipeconf reg. For HDMI we must use 8bpc
1993                  * here for both 8bpc and 12bpc.
1994                  */
1995                 val &= ~PIPECONF_BPC_MASK;
1996                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_HDMI))
1997                         val |= PIPECONF_8BPC;
1998                 else
1999                         val |= pipeconf_val & PIPECONF_BPC_MASK;
2000         }
2001
2002         val &= ~TRANS_INTERLACE_MASK;
2003         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
2004                 if (HAS_PCH_IBX(dev_priv->dev) &&
2005                     intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
2006                         val |= TRANS_LEGACY_INTERLACED_ILK;
2007                 else
2008                         val |= TRANS_INTERLACED;
2009         else
2010                 val |= TRANS_PROGRESSIVE;
2011
2012         I915_WRITE(reg, val | TRANS_ENABLE);
2013         if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
2014                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
2015 }
2016
2017 static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
2018                                       enum transcoder cpu_transcoder)
2019 {
2020         u32 val, pipeconf_val;
2021
2022         /* PCH only available on ILK+ */
2023         BUG_ON(!HAS_PCH_SPLIT(dev_priv->dev));
2024
2025         /* FDI must be feeding us bits for PCH ports */
2026         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
2027         assert_fdi_rx_enabled(dev_priv, TRANSCODER_A);
2028
2029         /* Workaround: set timing override bit. */
2030         val = I915_READ(_TRANSA_CHICKEN2);
2031         val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
2032         I915_WRITE(_TRANSA_CHICKEN2, val);
2033
2034         val = TRANS_ENABLE;
2035         pipeconf_val = I915_READ(PIPECONF(cpu_transcoder));
2036
2037         if ((pipeconf_val & PIPECONF_INTERLACE_MASK_HSW) ==
2038             PIPECONF_INTERLACED_ILK)
2039                 val |= TRANS_INTERLACED;
2040         else
2041                 val |= TRANS_PROGRESSIVE;
2042
2043         I915_WRITE(LPT_TRANSCONF, val);
2044         if (wait_for(I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE, 100))
2045                 DRM_ERROR("Failed to enable PCH transcoder\n");
2046 }
2047
2048 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
2049                                             enum pipe pipe)
2050 {
2051         struct drm_device *dev = dev_priv->dev;
2052         uint32_t reg, val;
2053
2054         /* FDI relies on the transcoder */
2055         assert_fdi_tx_disabled(dev_priv, pipe);
2056         assert_fdi_rx_disabled(dev_priv, pipe);
2057
2058         /* Ports must be off as well */
2059         assert_pch_ports_disabled(dev_priv, pipe);
2060
2061         reg = PCH_TRANSCONF(pipe);
2062         val = I915_READ(reg);
2063         val &= ~TRANS_ENABLE;
2064         I915_WRITE(reg, val);
2065         /* wait for PCH transcoder off, transcoder state */
2066         if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
2067                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
2068
2069         if (!HAS_PCH_IBX(dev)) {
2070                 /* Workaround: Clear the timing override chicken bit again. */
2071                 reg = TRANS_CHICKEN2(pipe);
2072                 val = I915_READ(reg);
2073                 val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
2074                 I915_WRITE(reg, val);
2075         }
2076 }
2077
2078 static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
2079 {
2080         u32 val;
2081
2082         val = I915_READ(LPT_TRANSCONF);
2083         val &= ~TRANS_ENABLE;
2084         I915_WRITE(LPT_TRANSCONF, val);
2085         /* wait for PCH transcoder off, transcoder state */
2086         if (wait_for((I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE) == 0, 50))
2087                 DRM_ERROR("Failed to disable PCH transcoder\n");
2088
2089         /* Workaround: clear timing override bit. */
2090         val = I915_READ(_TRANSA_CHICKEN2);
2091         val &= ~TRANS_CHICKEN2_TIMING_OVERRIDE;
2092         I915_WRITE(_TRANSA_CHICKEN2, val);
2093 }
2094
2095 /**
2096  * intel_enable_pipe - enable a pipe, asserting requirements
2097  * @crtc: crtc responsible for the pipe
2098  *
2099  * Enable @crtc's pipe, making sure that various hardware specific requirements
2100  * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
2101  */
2102 static void intel_enable_pipe(struct intel_crtc *crtc)
2103 {
2104         struct drm_device *dev = crtc->base.dev;
2105         struct drm_i915_private *dev_priv = dev->dev_private;
2106         enum pipe pipe = crtc->pipe;
2107         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
2108                                                                       pipe);
2109         enum pipe pch_transcoder;
2110         int reg;
2111         u32 val;
2112
2113         assert_planes_disabled(dev_priv, pipe);
2114         assert_cursor_disabled(dev_priv, pipe);
2115         assert_sprites_disabled(dev_priv, pipe);
2116
2117         if (HAS_PCH_LPT(dev_priv->dev))
2118                 pch_transcoder = TRANSCODER_A;
2119         else
2120                 pch_transcoder = pipe;
2121
2122         /*
2123          * A pipe without a PLL won't actually be able to drive bits from
2124          * a plane.  On ILK+ the pipe PLLs are integrated, so we don't
2125          * need the check.
2126          */
2127         if (HAS_GMCH_DISPLAY(dev_priv->dev))
2128                 if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI))
2129                         assert_dsi_pll_enabled(dev_priv);
2130                 else
2131                         assert_pll_enabled(dev_priv, pipe);
2132         else {
2133                 if (crtc->config->has_pch_encoder) {
2134                         /* if driving the PCH, we need FDI enabled */
2135                         assert_fdi_rx_pll_enabled(dev_priv, pch_transcoder);
2136                         assert_fdi_tx_pll_enabled(dev_priv,
2137                                                   (enum pipe) cpu_transcoder);
2138                 }
2139                 /* FIXME: assert CPU port conditions for SNB+ */
2140         }
2141
2142         reg = PIPECONF(cpu_transcoder);
2143         val = I915_READ(reg);
2144         if (val & PIPECONF_ENABLE) {
2145                 WARN_ON(!((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
2146                           (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)));
2147                 return;
2148         }
2149
2150         I915_WRITE(reg, val | PIPECONF_ENABLE);
2151         POSTING_READ(reg);
2152 }
2153
2154 /**
2155  * intel_disable_pipe - disable a pipe, asserting requirements
2156  * @crtc: crtc whose pipes is to be disabled
2157  *
2158  * Disable the pipe of @crtc, making sure that various hardware
2159  * specific requirements are met, if applicable, e.g. plane
2160  * disabled, panel fitter off, etc.
2161  *
2162  * Will wait until the pipe has shut down before returning.
2163  */
2164 static void intel_disable_pipe(struct intel_crtc *crtc)
2165 {
2166         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
2167         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
2168         enum pipe pipe = crtc->pipe;
2169         int reg;
2170         u32 val;
2171
2172         /*
2173          * Make sure planes won't keep trying to pump pixels to us,
2174          * or we might hang the display.
2175          */
2176         assert_planes_disabled(dev_priv, pipe);
2177         assert_cursor_disabled(dev_priv, pipe);
2178         assert_sprites_disabled(dev_priv, pipe);
2179
2180         reg = PIPECONF(cpu_transcoder);
2181         val = I915_READ(reg);
2182         if ((val & PIPECONF_ENABLE) == 0)
2183                 return;
2184
2185         /*
2186          * Double wide has implications for planes
2187          * so best keep it disabled when not needed.
2188          */
2189         if (crtc->config->double_wide)
2190                 val &= ~PIPECONF_DOUBLE_WIDE;
2191
2192         /* Don't disable pipe or pipe PLLs if needed */
2193         if (!(pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) &&
2194             !(pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
2195                 val &= ~PIPECONF_ENABLE;
2196
2197         I915_WRITE(reg, val);
2198         if ((val & PIPECONF_ENABLE) == 0)
2199                 intel_wait_for_pipe_off(crtc);
2200 }
2201
2202 static bool need_vtd_wa(struct drm_device *dev)
2203 {
2204 #ifdef CONFIG_INTEL_IOMMU
2205         if (INTEL_INFO(dev)->gen >= 6 && intel_iommu_gfx_mapped)
2206                 return true;
2207 #endif
2208         return false;
2209 }
2210
2211 unsigned int
2212 intel_tile_height(struct drm_device *dev, uint32_t pixel_format,
2213                   uint64_t fb_format_modifier)
2214 {
2215         unsigned int tile_height;
2216         uint32_t pixel_bytes;
2217
2218         switch (fb_format_modifier) {
2219         case DRM_FORMAT_MOD_NONE:
2220                 tile_height = 1;
2221                 break;
2222         case I915_FORMAT_MOD_X_TILED:
2223                 tile_height = IS_GEN2(dev) ? 16 : 8;
2224                 break;
2225         case I915_FORMAT_MOD_Y_TILED:
2226                 tile_height = 32;
2227                 break;
2228         case I915_FORMAT_MOD_Yf_TILED:
2229                 pixel_bytes = drm_format_plane_cpp(pixel_format, 0);
2230                 switch (pixel_bytes) {
2231                 default:
2232                 case 1:
2233                         tile_height = 64;
2234                         break;
2235                 case 2:
2236                 case 4:
2237                         tile_height = 32;
2238                         break;
2239                 case 8:
2240                         tile_height = 16;
2241                         break;
2242                 case 16:
2243                         WARN_ONCE(1,
2244                                   "128-bit pixels are not supported for display!");
2245                         tile_height = 16;
2246                         break;
2247                 }
2248                 break;
2249         default:
2250                 MISSING_CASE(fb_format_modifier);
2251                 tile_height = 1;
2252                 break;
2253         }
2254
2255         return tile_height;
2256 }
2257
2258 unsigned int
2259 intel_fb_align_height(struct drm_device *dev, unsigned int height,
2260                       uint32_t pixel_format, uint64_t fb_format_modifier)
2261 {
2262         return ALIGN(height, intel_tile_height(dev, pixel_format,
2263                                                fb_format_modifier));
2264 }
2265
2266 static int
2267 intel_fill_fb_ggtt_view(struct i915_ggtt_view *view, struct drm_framebuffer *fb,
2268                         const struct drm_plane_state *plane_state)
2269 {
2270         struct intel_rotation_info *info = &view->rotation_info;
2271         unsigned int tile_height, tile_pitch;
2272
2273         *view = i915_ggtt_view_normal;
2274
2275         if (!plane_state)
2276                 return 0;
2277
2278         if (!intel_rotation_90_or_270(plane_state->rotation))
2279                 return 0;
2280
2281         *view = i915_ggtt_view_rotated;
2282
2283         info->height = fb->height;
2284         info->pixel_format = fb->pixel_format;
2285         info->pitch = fb->pitches[0];
2286         info->fb_modifier = fb->modifier[0];
2287
2288         tile_height = intel_tile_height(fb->dev, fb->pixel_format,
2289                                         fb->modifier[0]);
2290         tile_pitch = PAGE_SIZE / tile_height;
2291         info->width_pages = DIV_ROUND_UP(fb->pitches[0], tile_pitch);
2292         info->height_pages = DIV_ROUND_UP(fb->height, tile_height);
2293         info->size = info->width_pages * info->height_pages * PAGE_SIZE;
2294
2295         return 0;
2296 }
2297
2298 static unsigned int intel_linear_alignment(struct drm_i915_private *dev_priv)
2299 {
2300         if (INTEL_INFO(dev_priv)->gen >= 9)
2301                 return 256 * 1024;
2302         else if (IS_BROADWATER(dev_priv) || IS_CRESTLINE(dev_priv) ||
2303                  IS_VALLEYVIEW(dev_priv))
2304                 return 128 * 1024;
2305         else if (INTEL_INFO(dev_priv)->gen >= 4)
2306                 return 4 * 1024;
2307         else
2308                 return 0;
2309 }
2310
2311 int
2312 intel_pin_and_fence_fb_obj(struct drm_plane *plane,
2313                            struct drm_framebuffer *fb,
2314                            const struct drm_plane_state *plane_state,
2315                            struct intel_engine_cs *pipelined,
2316                            struct drm_i915_gem_request **pipelined_request)
2317 {
2318         struct drm_device *dev = fb->dev;
2319         struct drm_i915_private *dev_priv = dev->dev_private;
2320         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2321         struct i915_ggtt_view view;
2322         u32 alignment;
2323         int ret;
2324
2325         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
2326
2327         switch (fb->modifier[0]) {
2328         case DRM_FORMAT_MOD_NONE:
2329                 alignment = intel_linear_alignment(dev_priv);
2330                 break;
2331         case I915_FORMAT_MOD_X_TILED:
2332                 if (INTEL_INFO(dev)->gen >= 9)
2333                         alignment = 256 * 1024;
2334                 else {
2335                         /* pin() will align the object as required by fence */
2336                         alignment = 0;
2337                 }
2338                 break;
2339         case I915_FORMAT_MOD_Y_TILED:
2340         case I915_FORMAT_MOD_Yf_TILED:
2341                 if (WARN_ONCE(INTEL_INFO(dev)->gen < 9,
2342                           "Y tiling bo slipped through, driver bug!\n"))
2343                         return -EINVAL;
2344                 alignment = 1 * 1024 * 1024;
2345                 break;
2346         default:
2347                 MISSING_CASE(fb->modifier[0]);
2348                 return -EINVAL;
2349         }
2350
2351         ret = intel_fill_fb_ggtt_view(&view, fb, plane_state);
2352         if (ret)
2353                 return ret;
2354
2355         /* Note that the w/a also requires 64 PTE of padding following the
2356          * bo. We currently fill all unused PTE with the shadow page and so
2357          * we should always have valid PTE following the scanout preventing
2358          * the VT-d warning.
2359          */
2360         if (need_vtd_wa(dev) && alignment < 256 * 1024)
2361                 alignment = 256 * 1024;
2362
2363         /*
2364          * Global gtt pte registers are special registers which actually forward
2365          * writes to a chunk of system memory. Which means that there is no risk
2366          * that the register values disappear as soon as we call
2367          * intel_runtime_pm_put(), so it is correct to wrap only the
2368          * pin/unpin/fence and not more.
2369          */
2370         intel_runtime_pm_get(dev_priv);
2371
2372         dev_priv->mm.interruptible = false;
2373         ret = i915_gem_object_pin_to_display_plane(obj, alignment, pipelined,
2374                                                    pipelined_request, &view);
2375         if (ret)
2376                 goto err_interruptible;
2377
2378         /* Install a fence for tiled scan-out. Pre-i965 always needs a
2379          * fence, whereas 965+ only requires a fence if using
2380          * framebuffer compression.  For simplicity, we always install
2381          * a fence as the cost is not that onerous.
2382          */
2383         ret = i915_gem_object_get_fence(obj);
2384         if (ret)
2385                 goto err_unpin;
2386
2387         i915_gem_object_pin_fence(obj);
2388
2389         dev_priv->mm.interruptible = true;
2390         intel_runtime_pm_put(dev_priv);
2391         return 0;
2392
2393 err_unpin:
2394         i915_gem_object_unpin_from_display_plane(obj, &view);
2395 err_interruptible:
2396         dev_priv->mm.interruptible = true;
2397         intel_runtime_pm_put(dev_priv);
2398         return ret;
2399 }
2400
2401 static void intel_unpin_fb_obj(struct drm_framebuffer *fb,
2402                                const struct drm_plane_state *plane_state)
2403 {
2404         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
2405         struct i915_ggtt_view view;
2406         int ret;
2407
2408         WARN_ON(!mutex_is_locked(&obj->base.dev->struct_mutex));
2409
2410         ret = intel_fill_fb_ggtt_view(&view, fb, plane_state);
2411         WARN_ONCE(ret, "Couldn't get view from plane state!");
2412
2413         i915_gem_object_unpin_fence(obj);
2414         i915_gem_object_unpin_from_display_plane(obj, &view);
2415 }
2416
2417 /* Computes the linear offset to the base tile and adjusts x, y. bytes per pixel
2418  * is assumed to be a power-of-two. */
2419 unsigned long intel_gen4_compute_page_offset(struct drm_i915_private *dev_priv,
2420                                              int *x, int *y,
2421                                              unsigned int tiling_mode,
2422                                              unsigned int cpp,
2423                                              unsigned int pitch)
2424 {
2425         if (tiling_mode != I915_TILING_NONE) {
2426                 unsigned int tile_rows, tiles;
2427
2428                 tile_rows = *y / 8;
2429                 *y %= 8;
2430
2431                 tiles = *x / (512/cpp);
2432                 *x %= 512/cpp;
2433
2434                 return tile_rows * pitch * 8 + tiles * 4096;
2435         } else {
2436                 unsigned int alignment = intel_linear_alignment(dev_priv) - 1;
2437                 unsigned int offset;
2438
2439                 offset = *y * pitch + *x * cpp;
2440                 *y = (offset & alignment) / pitch;
2441                 *x = ((offset & alignment) - *y * pitch) / cpp;
2442                 return offset & ~alignment;
2443         }
2444 }
2445
2446 static int i9xx_format_to_fourcc(int format)
2447 {
2448         switch (format) {
2449         case DISPPLANE_8BPP:
2450                 return DRM_FORMAT_C8;
2451         case DISPPLANE_BGRX555:
2452                 return DRM_FORMAT_XRGB1555;
2453         case DISPPLANE_BGRX565:
2454                 return DRM_FORMAT_RGB565;
2455         default:
2456         case DISPPLANE_BGRX888:
2457                 return DRM_FORMAT_XRGB8888;
2458         case DISPPLANE_RGBX888:
2459                 return DRM_FORMAT_XBGR8888;
2460         case DISPPLANE_BGRX101010:
2461                 return DRM_FORMAT_XRGB2101010;
2462         case DISPPLANE_RGBX101010:
2463                 return DRM_FORMAT_XBGR2101010;
2464         }
2465 }
2466
2467 static int skl_format_to_fourcc(int format, bool rgb_order, bool alpha)
2468 {
2469         switch (format) {
2470         case PLANE_CTL_FORMAT_RGB_565:
2471                 return DRM_FORMAT_RGB565;
2472         default:
2473         case PLANE_CTL_FORMAT_XRGB_8888:
2474                 if (rgb_order) {
2475                         if (alpha)
2476                                 return DRM_FORMAT_ABGR8888;
2477                         else
2478                                 return DRM_FORMAT_XBGR8888;
2479                 } else {
2480                         if (alpha)
2481                                 return DRM_FORMAT_ARGB8888;
2482                         else
2483                                 return DRM_FORMAT_XRGB8888;
2484                 }
2485         case PLANE_CTL_FORMAT_XRGB_2101010:
2486                 if (rgb_order)
2487                         return DRM_FORMAT_XBGR2101010;
2488                 else
2489                         return DRM_FORMAT_XRGB2101010;
2490         }
2491 }
2492
2493 static bool
2494 intel_alloc_initial_plane_obj(struct intel_crtc *crtc,
2495                               struct intel_initial_plane_config *plane_config)
2496 {
2497         struct drm_device *dev = crtc->base.dev;
2498         struct drm_i915_gem_object *obj = NULL;
2499         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
2500         struct drm_framebuffer *fb = &plane_config->fb->base;
2501         u32 base_aligned = round_down(plane_config->base, PAGE_SIZE);
2502         u32 size_aligned = round_up(plane_config->base + plane_config->size,
2503                                     PAGE_SIZE);
2504
2505         size_aligned -= base_aligned;
2506
2507         if (plane_config->size == 0)
2508                 return false;
2509
2510         obj = i915_gem_object_create_stolen_for_preallocated(dev,
2511                                                              base_aligned,
2512                                                              base_aligned,
2513                                                              size_aligned);
2514         if (!obj)
2515                 return false;
2516
2517         obj->tiling_mode = plane_config->tiling;
2518         if (obj->tiling_mode == I915_TILING_X)
2519                 obj->stride = fb->pitches[0];
2520
2521         mode_cmd.pixel_format = fb->pixel_format;
2522         mode_cmd.width = fb->width;
2523         mode_cmd.height = fb->height;
2524         mode_cmd.pitches[0] = fb->pitches[0];
2525         mode_cmd.modifier[0] = fb->modifier[0];
2526         mode_cmd.flags = DRM_MODE_FB_MODIFIERS;
2527
2528         mutex_lock(&dev->struct_mutex);
2529         if (intel_framebuffer_init(dev, to_intel_framebuffer(fb),
2530                                    &mode_cmd, obj)) {
2531                 DRM_DEBUG_KMS("intel fb init failed\n");
2532                 goto out_unref_obj;
2533         }
2534         mutex_unlock(&dev->struct_mutex);
2535
2536         DRM_DEBUG_KMS("initial plane fb obj %p\n", obj);
2537         return true;
2538
2539 out_unref_obj:
2540         drm_gem_object_unreference(&obj->base);
2541         mutex_unlock(&dev->struct_mutex);
2542         return false;
2543 }
2544
2545 /* Update plane->state->fb to match plane->fb after driver-internal updates */
2546 static void
2547 update_state_fb(struct drm_plane *plane)
2548 {
2549         if (plane->fb == plane->state->fb)
2550                 return;
2551
2552         if (plane->state->fb)
2553                 drm_framebuffer_unreference(plane->state->fb);
2554         plane->state->fb = plane->fb;
2555         if (plane->state->fb)
2556                 drm_framebuffer_reference(plane->state->fb);
2557 }
2558
2559 static void
2560 intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
2561                              struct intel_initial_plane_config *plane_config)
2562 {
2563         struct drm_device *dev = intel_crtc->base.dev;
2564         struct drm_i915_private *dev_priv = dev->dev_private;
2565         struct drm_crtc *c;
2566         struct intel_crtc *i;
2567         struct drm_i915_gem_object *obj;
2568         struct drm_plane *primary = intel_crtc->base.primary;
2569         struct drm_framebuffer *fb;
2570
2571         if (!plane_config->fb)
2572                 return;
2573
2574         if (intel_alloc_initial_plane_obj(intel_crtc, plane_config)) {
2575                 fb = &plane_config->fb->base;
2576                 goto valid_fb;
2577         }
2578
2579         kfree(plane_config->fb);
2580
2581         /*
2582          * Failed to alloc the obj, check to see if we should share
2583          * an fb with another CRTC instead
2584          */
2585         for_each_crtc(dev, c) {
2586                 i = to_intel_crtc(c);
2587
2588                 if (c == &intel_crtc->base)
2589                         continue;
2590
2591                 if (!i->active)
2592                         continue;
2593
2594                 fb = c->primary->fb;
2595                 if (!fb)
2596                         continue;
2597
2598                 obj = intel_fb_obj(fb);
2599                 if (i915_gem_obj_ggtt_offset(obj) == plane_config->base) {
2600                         drm_framebuffer_reference(fb);
2601                         goto valid_fb;
2602                 }
2603         }
2604
2605         return;
2606
2607 valid_fb:
2608         obj = intel_fb_obj(fb);
2609         if (obj->tiling_mode != I915_TILING_NONE)
2610                 dev_priv->preserve_bios_swizzle = true;
2611
2612         primary->fb = fb;
2613         primary->crtc = primary->state->crtc = &intel_crtc->base;
2614         update_state_fb(primary);
2615         intel_crtc->base.state->plane_mask |= (1 << drm_plane_index(primary));
2616         obj->frontbuffer_bits |= INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe);
2617 }
2618
2619 static void i9xx_update_primary_plane(struct drm_crtc *crtc,
2620                                       struct drm_framebuffer *fb,
2621                                       int x, int y)
2622 {
2623         struct drm_device *dev = crtc->dev;
2624         struct drm_i915_private *dev_priv = dev->dev_private;
2625         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2626         struct drm_plane *primary = crtc->primary;
2627         bool visible = to_intel_plane_state(primary->state)->visible;
2628         struct drm_i915_gem_object *obj;
2629         int plane = intel_crtc->plane;
2630         unsigned long linear_offset;
2631         u32 dspcntr;
2632         u32 reg = DSPCNTR(plane);
2633         int pixel_size;
2634
2635         if (!visible || !fb) {
2636                 I915_WRITE(reg, 0);
2637                 if (INTEL_INFO(dev)->gen >= 4)
2638                         I915_WRITE(DSPSURF(plane), 0);
2639                 else
2640                         I915_WRITE(DSPADDR(plane), 0);
2641                 POSTING_READ(reg);
2642                 return;
2643         }
2644
2645         obj = intel_fb_obj(fb);
2646         if (WARN_ON(obj == NULL))
2647                 return;
2648
2649         pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
2650
2651         dspcntr = DISPPLANE_GAMMA_ENABLE;
2652
2653         dspcntr |= DISPLAY_PLANE_ENABLE;
2654
2655         if (INTEL_INFO(dev)->gen < 4) {
2656                 if (intel_crtc->pipe == PIPE_B)
2657                         dspcntr |= DISPPLANE_SEL_PIPE_B;
2658
2659                 /* pipesrc and dspsize control the size that is scaled from,
2660                  * which should always be the user's requested size.
2661                  */
2662                 I915_WRITE(DSPSIZE(plane),
2663                            ((intel_crtc->config->pipe_src_h - 1) << 16) |
2664                            (intel_crtc->config->pipe_src_w - 1));
2665                 I915_WRITE(DSPPOS(plane), 0);
2666         } else if (IS_CHERRYVIEW(dev) && plane == PLANE_B) {
2667                 I915_WRITE(PRIMSIZE(plane),
2668                            ((intel_crtc->config->pipe_src_h - 1) << 16) |
2669                            (intel_crtc->config->pipe_src_w - 1));
2670                 I915_WRITE(PRIMPOS(plane), 0);
2671                 I915_WRITE(PRIMCNSTALPHA(plane), 0);
2672         }
2673
2674         switch (fb->pixel_format) {
2675         case DRM_FORMAT_C8:
2676                 dspcntr |= DISPPLANE_8BPP;
2677                 break;
2678         case DRM_FORMAT_XRGB1555:
2679                 dspcntr |= DISPPLANE_BGRX555;
2680                 break;
2681         case DRM_FORMAT_RGB565:
2682                 dspcntr |= DISPPLANE_BGRX565;
2683                 break;
2684         case DRM_FORMAT_XRGB8888:
2685                 dspcntr |= DISPPLANE_BGRX888;
2686                 break;
2687         case DRM_FORMAT_XBGR8888:
2688                 dspcntr |= DISPPLANE_RGBX888;
2689                 break;
2690         case DRM_FORMAT_XRGB2101010:
2691                 dspcntr |= DISPPLANE_BGRX101010;
2692                 break;
2693         case DRM_FORMAT_XBGR2101010:
2694                 dspcntr |= DISPPLANE_RGBX101010;
2695                 break;
2696         default:
2697                 BUG();
2698         }
2699
2700         if (INTEL_INFO(dev)->gen >= 4 &&
2701             obj->tiling_mode != I915_TILING_NONE)
2702                 dspcntr |= DISPPLANE_TILED;
2703
2704         if (IS_G4X(dev))
2705                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2706
2707         linear_offset = y * fb->pitches[0] + x * pixel_size;
2708
2709         if (INTEL_INFO(dev)->gen >= 4) {
2710                 intel_crtc->dspaddr_offset =
2711                         intel_gen4_compute_page_offset(dev_priv,
2712                                                        &x, &y, obj->tiling_mode,
2713                                                        pixel_size,
2714                                                        fb->pitches[0]);
2715                 linear_offset -= intel_crtc->dspaddr_offset;
2716         } else {
2717                 intel_crtc->dspaddr_offset = linear_offset;
2718         }
2719
2720         if (crtc->primary->state->rotation == BIT(DRM_ROTATE_180)) {
2721                 dspcntr |= DISPPLANE_ROTATE_180;
2722
2723                 x += (intel_crtc->config->pipe_src_w - 1);
2724                 y += (intel_crtc->config->pipe_src_h - 1);
2725
2726                 /* Finding the last pixel of the last line of the display
2727                 data and adding to linear_offset*/
2728                 linear_offset +=
2729                         (intel_crtc->config->pipe_src_h - 1) * fb->pitches[0] +
2730                         (intel_crtc->config->pipe_src_w - 1) * pixel_size;
2731         }
2732
2733         I915_WRITE(reg, dspcntr);
2734
2735         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2736         if (INTEL_INFO(dev)->gen >= 4) {
2737                 I915_WRITE(DSPSURF(plane),
2738                            i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
2739                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2740                 I915_WRITE(DSPLINOFF(plane), linear_offset);
2741         } else
2742                 I915_WRITE(DSPADDR(plane), i915_gem_obj_ggtt_offset(obj) + linear_offset);
2743         POSTING_READ(reg);
2744 }
2745
2746 static void ironlake_update_primary_plane(struct drm_crtc *crtc,
2747                                           struct drm_framebuffer *fb,
2748                                           int x, int y)
2749 {
2750         struct drm_device *dev = crtc->dev;
2751         struct drm_i915_private *dev_priv = dev->dev_private;
2752         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
2753         struct drm_plane *primary = crtc->primary;
2754         bool visible = to_intel_plane_state(primary->state)->visible;
2755         struct drm_i915_gem_object *obj;
2756         int plane = intel_crtc->plane;
2757         unsigned long linear_offset;
2758         u32 dspcntr;
2759         u32 reg = DSPCNTR(plane);
2760         int pixel_size;
2761
2762         if (!visible || !fb) {
2763                 I915_WRITE(reg, 0);
2764                 I915_WRITE(DSPSURF(plane), 0);
2765                 POSTING_READ(reg);
2766                 return;
2767         }
2768
2769         obj = intel_fb_obj(fb);
2770         if (WARN_ON(obj == NULL))
2771                 return;
2772
2773         pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
2774
2775         dspcntr = DISPPLANE_GAMMA_ENABLE;
2776
2777         dspcntr |= DISPLAY_PLANE_ENABLE;
2778
2779         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
2780                 dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
2781
2782         switch (fb->pixel_format) {
2783         case DRM_FORMAT_C8:
2784                 dspcntr |= DISPPLANE_8BPP;
2785                 break;
2786         case DRM_FORMAT_RGB565:
2787                 dspcntr |= DISPPLANE_BGRX565;
2788                 break;
2789         case DRM_FORMAT_XRGB8888:
2790                 dspcntr |= DISPPLANE_BGRX888;
2791                 break;
2792         case DRM_FORMAT_XBGR8888:
2793                 dspcntr |= DISPPLANE_RGBX888;
2794                 break;
2795         case DRM_FORMAT_XRGB2101010:
2796                 dspcntr |= DISPPLANE_BGRX101010;
2797                 break;
2798         case DRM_FORMAT_XBGR2101010:
2799                 dspcntr |= DISPPLANE_RGBX101010;
2800                 break;
2801         default:
2802                 BUG();
2803         }
2804
2805         if (obj->tiling_mode != I915_TILING_NONE)
2806                 dspcntr |= DISPPLANE_TILED;
2807
2808         if (!IS_HASWELL(dev) && !IS_BROADWELL(dev))
2809                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
2810
2811         linear_offset = y * fb->pitches[0] + x * pixel_size;
2812         intel_crtc->dspaddr_offset =
2813                 intel_gen4_compute_page_offset(dev_priv,
2814                                                &x, &y, obj->tiling_mode,
2815                                                pixel_size,
2816                                                fb->pitches[0]);
2817         linear_offset -= intel_crtc->dspaddr_offset;
2818         if (crtc->primary->state->rotation == BIT(DRM_ROTATE_180)) {
2819                 dspcntr |= DISPPLANE_ROTATE_180;
2820
2821                 if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
2822                         x += (intel_crtc->config->pipe_src_w - 1);
2823                         y += (intel_crtc->config->pipe_src_h - 1);
2824
2825                         /* Finding the last pixel of the last line of the display
2826                         data and adding to linear_offset*/
2827                         linear_offset +=
2828                                 (intel_crtc->config->pipe_src_h - 1) * fb->pitches[0] +
2829                                 (intel_crtc->config->pipe_src_w - 1) * pixel_size;
2830                 }
2831         }
2832
2833         I915_WRITE(reg, dspcntr);
2834
2835         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
2836         I915_WRITE(DSPSURF(plane),
2837                    i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
2838         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
2839                 I915_WRITE(DSPOFFSET(plane), (y << 16) | x);
2840         } else {
2841                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
2842                 I915_WRITE(DSPLINOFF(plane), linear_offset);
2843         }
2844         POSTING_READ(reg);
2845 }
2846
2847 u32 intel_fb_stride_alignment(struct drm_device *dev, uint64_t fb_modifier,
2848                               uint32_t pixel_format)
2849 {
2850         u32 bits_per_pixel = drm_format_plane_cpp(pixel_format, 0) * 8;
2851
2852         /*
2853          * The stride is either expressed as a multiple of 64 bytes
2854          * chunks for linear buffers or in number of tiles for tiled
2855          * buffers.
2856          */
2857         switch (fb_modifier) {
2858         case DRM_FORMAT_MOD_NONE:
2859                 return 64;
2860         case I915_FORMAT_MOD_X_TILED:
2861                 if (INTEL_INFO(dev)->gen == 2)
2862                         return 128;
2863                 return 512;
2864         case I915_FORMAT_MOD_Y_TILED:
2865                 /* No need to check for old gens and Y tiling since this is
2866                  * about the display engine and those will be blocked before
2867                  * we get here.
2868                  */
2869                 return 128;
2870         case I915_FORMAT_MOD_Yf_TILED:
2871                 if (bits_per_pixel == 8)
2872                         return 64;
2873                 else
2874                         return 128;
2875         default:
2876                 MISSING_CASE(fb_modifier);
2877                 return 64;
2878         }
2879 }
2880
2881 unsigned long intel_plane_obj_offset(struct intel_plane *intel_plane,
2882                                      struct drm_i915_gem_object *obj)
2883 {
2884         const struct i915_ggtt_view *view = &i915_ggtt_view_normal;
2885
2886         if (intel_rotation_90_or_270(intel_plane->base.state->rotation))
2887                 view = &i915_ggtt_view_rotated;
2888
2889         return i915_gem_obj_ggtt_offset_view(obj, view);
2890 }
2891
2892 /*
2893  * This function detaches (aka. unbinds) unused scalers in hardware
2894  */
2895 static void skl_detach_scalers(struct intel_crtc *intel_crtc)
2896 {
2897         struct drm_device *dev;
2898         struct drm_i915_private *dev_priv;
2899         struct intel_crtc_scaler_state *scaler_state;
2900         int i;
2901
2902         dev = intel_crtc->base.dev;
2903         dev_priv = dev->dev_private;
2904         scaler_state = &intel_crtc->config->scaler_state;
2905
2906         /* loop through and disable scalers that aren't in use */
2907         for (i = 0; i < intel_crtc->num_scalers; i++) {
2908                 if (!scaler_state->scalers[i].in_use) {
2909                         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, i), 0);
2910                         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, i), 0);
2911                         I915_WRITE(SKL_PS_WIN_SZ(intel_crtc->pipe, i), 0);
2912                         DRM_DEBUG_KMS("CRTC:%d Disabled scaler id %u.%u\n",
2913                                 intel_crtc->base.base.id, intel_crtc->pipe, i);
2914                 }
2915         }
2916 }
2917
2918 u32 skl_plane_ctl_format(uint32_t pixel_format)
2919 {
2920         switch (pixel_format) {
2921         case DRM_FORMAT_C8:
2922                 return PLANE_CTL_FORMAT_INDEXED;
2923         case DRM_FORMAT_RGB565:
2924                 return PLANE_CTL_FORMAT_RGB_565;
2925         case DRM_FORMAT_XBGR8888:
2926                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
2927         case DRM_FORMAT_XRGB8888:
2928                 return PLANE_CTL_FORMAT_XRGB_8888;
2929         /*
2930          * XXX: For ARBG/ABGR formats we default to expecting scanout buffers
2931          * to be already pre-multiplied. We need to add a knob (or a different
2932          * DRM_FORMAT) for user-space to configure that.
2933          */
2934         case DRM_FORMAT_ABGR8888:
2935                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX |
2936                         PLANE_CTL_ALPHA_SW_PREMULTIPLY;
2937         case DRM_FORMAT_ARGB8888:
2938                 return PLANE_CTL_FORMAT_XRGB_8888 |
2939                         PLANE_CTL_ALPHA_SW_PREMULTIPLY;
2940         case DRM_FORMAT_XRGB2101010:
2941                 return PLANE_CTL_FORMAT_XRGB_2101010;
2942         case DRM_FORMAT_XBGR2101010:
2943                 return PLANE_CTL_ORDER_RGBX | PLANE_CTL_FORMAT_XRGB_2101010;
2944         case DRM_FORMAT_YUYV:
2945                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YUYV;
2946         case DRM_FORMAT_YVYU:
2947                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_YVYU;
2948         case DRM_FORMAT_UYVY:
2949                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_UYVY;
2950         case DRM_FORMAT_VYUY:
2951                 return PLANE_CTL_FORMAT_YUV422 | PLANE_CTL_YUV422_VYUY;
2952         default:
2953                 MISSING_CASE(pixel_format);
2954         }
2955
2956         return 0;
2957 }
2958
2959 u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
2960 {
2961         switch (fb_modifier) {
2962         case DRM_FORMAT_MOD_NONE:
2963                 break;
2964         case I915_FORMAT_MOD_X_TILED:
2965                 return PLANE_CTL_TILED_X;
2966         case I915_FORMAT_MOD_Y_TILED:
2967                 return PLANE_CTL_TILED_Y;
2968         case I915_FORMAT_MOD_Yf_TILED:
2969                 return PLANE_CTL_TILED_YF;
2970         default:
2971                 MISSING_CASE(fb_modifier);
2972         }
2973
2974         return 0;
2975 }
2976
2977 u32 skl_plane_ctl_rotation(unsigned int rotation)
2978 {
2979         switch (rotation) {
2980         case BIT(DRM_ROTATE_0):
2981                 break;
2982         /*
2983          * DRM_ROTATE_ is counter clockwise to stay compatible with Xrandr
2984          * while i915 HW rotation is clockwise, thats why this swapping.
2985          */
2986         case BIT(DRM_ROTATE_90):
2987                 return PLANE_CTL_ROTATE_270;
2988         case BIT(DRM_ROTATE_180):
2989                 return PLANE_CTL_ROTATE_180;
2990         case BIT(DRM_ROTATE_270):
2991                 return PLANE_CTL_ROTATE_90;
2992         default:
2993                 MISSING_CASE(rotation);
2994         }
2995
2996         return 0;
2997 }
2998
2999 static void skylake_update_primary_plane(struct drm_crtc *crtc,
3000                                          struct drm_framebuffer *fb,
3001                                          int x, int y)
3002 {
3003         struct drm_device *dev = crtc->dev;
3004         struct drm_i915_private *dev_priv = dev->dev_private;
3005         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3006         struct drm_plane *plane = crtc->primary;
3007         bool visible = to_intel_plane_state(plane->state)->visible;
3008         struct drm_i915_gem_object *obj;
3009         int pipe = intel_crtc->pipe;
3010         u32 plane_ctl, stride_div, stride;
3011         u32 tile_height, plane_offset, plane_size;
3012         unsigned int rotation;
3013         int x_offset, y_offset;
3014         unsigned long surf_addr;
3015         struct intel_crtc_state *crtc_state = intel_crtc->config;
3016         struct intel_plane_state *plane_state;
3017         int src_x = 0, src_y = 0, src_w = 0, src_h = 0;
3018         int dst_x = 0, dst_y = 0, dst_w = 0, dst_h = 0;
3019         int scaler_id = -1;
3020
3021         plane_state = to_intel_plane_state(plane->state);
3022
3023         if (!visible || !fb) {
3024                 I915_WRITE(PLANE_CTL(pipe, 0), 0);
3025                 I915_WRITE(PLANE_SURF(pipe, 0), 0);
3026                 POSTING_READ(PLANE_CTL(pipe, 0));
3027                 return;
3028         }
3029
3030         plane_ctl = PLANE_CTL_ENABLE |
3031                     PLANE_CTL_PIPE_GAMMA_ENABLE |
3032                     PLANE_CTL_PIPE_CSC_ENABLE;
3033
3034         plane_ctl |= skl_plane_ctl_format(fb->pixel_format);
3035         plane_ctl |= skl_plane_ctl_tiling(fb->modifier[0]);
3036         plane_ctl |= PLANE_CTL_PLANE_GAMMA_DISABLE;
3037
3038         rotation = plane->state->rotation;
3039         plane_ctl |= skl_plane_ctl_rotation(rotation);
3040
3041         obj = intel_fb_obj(fb);
3042         stride_div = intel_fb_stride_alignment(dev, fb->modifier[0],
3043                                                fb->pixel_format);
3044         surf_addr = intel_plane_obj_offset(to_intel_plane(plane), obj);
3045
3046         /*
3047          * FIXME: intel_plane_state->src, dst aren't set when transitional
3048          * update_plane helpers are called from legacy paths.
3049          * Once full atomic crtc is available, below check can be avoided.
3050          */
3051         if (drm_rect_width(&plane_state->src)) {
3052                 scaler_id = plane_state->scaler_id;
3053                 src_x = plane_state->src.x1 >> 16;
3054                 src_y = plane_state->src.y1 >> 16;
3055                 src_w = drm_rect_width(&plane_state->src) >> 16;
3056                 src_h = drm_rect_height(&plane_state->src) >> 16;
3057                 dst_x = plane_state->dst.x1;
3058                 dst_y = plane_state->dst.y1;
3059                 dst_w = drm_rect_width(&plane_state->dst);
3060                 dst_h = drm_rect_height(&plane_state->dst);
3061
3062                 WARN_ON(x != src_x || y != src_y);
3063         } else {
3064                 src_w = intel_crtc->config->pipe_src_w;
3065                 src_h = intel_crtc->config->pipe_src_h;
3066         }
3067
3068         if (intel_rotation_90_or_270(rotation)) {
3069                 /* stride = Surface height in tiles */
3070                 tile_height = intel_tile_height(dev, fb->pixel_format,
3071                                                 fb->modifier[0]);
3072                 stride = DIV_ROUND_UP(fb->height, tile_height);
3073                 x_offset = stride * tile_height - y - src_h;
3074                 y_offset = x;
3075                 plane_size = (src_w - 1) << 16 | (src_h - 1);
3076         } else {
3077                 stride = fb->pitches[0] / stride_div;
3078                 x_offset = x;
3079                 y_offset = y;
3080                 plane_size = (src_h - 1) << 16 | (src_w - 1);
3081         }
3082         plane_offset = y_offset << 16 | x_offset;
3083
3084         I915_WRITE(PLANE_CTL(pipe, 0), plane_ctl);
3085         I915_WRITE(PLANE_OFFSET(pipe, 0), plane_offset);
3086         I915_WRITE(PLANE_SIZE(pipe, 0), plane_size);
3087         I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
3088
3089         if (scaler_id >= 0) {
3090                 uint32_t ps_ctrl = 0;
3091
3092                 WARN_ON(!dst_w || !dst_h);
3093                 ps_ctrl = PS_SCALER_EN | PS_PLANE_SEL(0) |
3094                         crtc_state->scaler_state.scalers[scaler_id].mode;
3095                 I915_WRITE(SKL_PS_CTRL(pipe, scaler_id), ps_ctrl);
3096                 I915_WRITE(SKL_PS_PWR_GATE(pipe, scaler_id), 0);
3097                 I915_WRITE(SKL_PS_WIN_POS(pipe, scaler_id), (dst_x << 16) | dst_y);
3098                 I915_WRITE(SKL_PS_WIN_SZ(pipe, scaler_id), (dst_w << 16) | dst_h);
3099                 I915_WRITE(PLANE_POS(pipe, 0), 0);
3100         } else {
3101                 I915_WRITE(PLANE_POS(pipe, 0), (dst_y << 16) | dst_x);
3102         }
3103
3104         I915_WRITE(PLANE_SURF(pipe, 0), surf_addr);
3105
3106         POSTING_READ(PLANE_SURF(pipe, 0));
3107 }
3108
3109 /* Assume fb object is pinned & idle & fenced and just update base pointers */
3110 static int
3111 intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
3112                            int x, int y, enum mode_set_atomic state)
3113 {
3114         struct drm_device *dev = crtc->dev;
3115         struct drm_i915_private *dev_priv = dev->dev_private;
3116
3117         if (dev_priv->display.disable_fbc)
3118                 dev_priv->display.disable_fbc(dev);
3119
3120         dev_priv->display.update_primary_plane(crtc, fb, x, y);
3121
3122         return 0;
3123 }
3124
3125 static void intel_complete_page_flips(struct drm_device *dev)
3126 {
3127         struct drm_crtc *crtc;
3128
3129         for_each_crtc(dev, crtc) {
3130                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3131                 enum plane plane = intel_crtc->plane;
3132
3133                 intel_prepare_page_flip(dev, plane);
3134                 intel_finish_page_flip_plane(dev, plane);
3135         }
3136 }
3137
3138 static void intel_update_primary_planes(struct drm_device *dev)
3139 {
3140         struct drm_i915_private *dev_priv = dev->dev_private;
3141         struct drm_crtc *crtc;
3142
3143         for_each_crtc(dev, crtc) {
3144                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3145
3146                 drm_modeset_lock(&crtc->mutex, NULL);
3147                 /*
3148                  * FIXME: Once we have proper support for primary planes (and
3149                  * disabling them without disabling the entire crtc) allow again
3150                  * a NULL crtc->primary->fb.
3151                  */
3152                 if (intel_crtc->active && crtc->primary->fb)
3153                         dev_priv->display.update_primary_plane(crtc,
3154                                                                crtc->primary->fb,
3155                                                                crtc->x,
3156                                                                crtc->y);
3157                 drm_modeset_unlock(&crtc->mutex);
3158         }
3159 }
3160
3161 void intel_prepare_reset(struct drm_device *dev)
3162 {
3163         /* no reset support for gen2 */
3164         if (IS_GEN2(dev))
3165                 return;
3166
3167         /* reset doesn't touch the display */
3168         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
3169                 return;
3170
3171         drm_modeset_lock_all(dev);
3172         /*
3173          * Disabling the crtcs gracefully seems nicer. Also the
3174          * g33 docs say we should at least disable all the planes.
3175          */
3176         intel_display_suspend(dev);
3177 }
3178
3179 void intel_finish_reset(struct drm_device *dev)
3180 {
3181         struct drm_i915_private *dev_priv = to_i915(dev);
3182
3183         /*
3184          * Flips in the rings will be nuked by the reset,
3185          * so complete all pending flips so that user space
3186          * will get its events and not get stuck.
3187          */
3188         intel_complete_page_flips(dev);
3189
3190         /* no reset support for gen2 */
3191         if (IS_GEN2(dev))
3192                 return;
3193
3194         /* reset doesn't touch the display */
3195         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) {
3196                 /*
3197                  * Flips in the rings have been nuked by the reset,
3198                  * so update the base address of all primary
3199                  * planes to the the last fb to make sure we're
3200                  * showing the correct fb after a reset.
3201                  */
3202                 intel_update_primary_planes(dev);
3203                 return;
3204         }
3205
3206         /*
3207          * The display has been reset as well,
3208          * so need a full re-initialization.
3209          */
3210         intel_runtime_pm_disable_interrupts(dev_priv);
3211         intel_runtime_pm_enable_interrupts(dev_priv);
3212
3213         intel_modeset_init_hw(dev);
3214
3215         spin_lock_irq(&dev_priv->irq_lock);
3216         if (dev_priv->display.hpd_irq_setup)
3217                 dev_priv->display.hpd_irq_setup(dev);
3218         spin_unlock_irq(&dev_priv->irq_lock);
3219
3220         intel_modeset_setup_hw_state(dev, true);
3221
3222         intel_hpd_init(dev_priv);
3223
3224         drm_modeset_unlock_all(dev);
3225 }
3226
3227 static void
3228 intel_finish_fb(struct drm_framebuffer *old_fb)
3229 {
3230         struct drm_i915_gem_object *obj = intel_fb_obj(old_fb);
3231         struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
3232         bool was_interruptible = dev_priv->mm.interruptible;
3233         int ret;
3234
3235         /* Big Hammer, we also need to ensure that any pending
3236          * MI_WAIT_FOR_EVENT inside a user batch buffer on the
3237          * current scanout is retired before unpinning the old
3238          * framebuffer. Note that we rely on userspace rendering
3239          * into the buffer attached to the pipe they are waiting
3240          * on. If not, userspace generates a GPU hang with IPEHR
3241          * point to the MI_WAIT_FOR_EVENT.
3242          *
3243          * This should only fail upon a hung GPU, in which case we
3244          * can safely continue.
3245          */
3246         dev_priv->mm.interruptible = false;
3247         ret = i915_gem_object_wait_rendering(obj, true);
3248         dev_priv->mm.interruptible = was_interruptible;
3249
3250         WARN_ON(ret);
3251 }
3252
3253 static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
3254 {
3255         struct drm_device *dev = crtc->dev;
3256         struct drm_i915_private *dev_priv = dev->dev_private;
3257         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3258         bool pending;
3259
3260         if (i915_reset_in_progress(&dev_priv->gpu_error) ||
3261             intel_crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
3262                 return false;
3263
3264         spin_lock_irq(&dev->event_lock);
3265         pending = to_intel_crtc(crtc)->unpin_work != NULL;
3266         spin_unlock_irq(&dev->event_lock);
3267
3268         return pending;
3269 }
3270
3271 static void intel_update_pipe_size(struct intel_crtc *crtc)
3272 {
3273         struct drm_device *dev = crtc->base.dev;
3274         struct drm_i915_private *dev_priv = dev->dev_private;
3275         const struct drm_display_mode *adjusted_mode;
3276
3277         if (!i915.fastboot)
3278                 return;
3279
3280         /*
3281          * Update pipe size and adjust fitter if needed: the reason for this is
3282          * that in compute_mode_changes we check the native mode (not the pfit
3283          * mode) to see if we can flip rather than do a full mode set. In the
3284          * fastboot case, we'll flip, but if we don't update the pipesrc and
3285          * pfit state, we'll end up with a big fb scanned out into the wrong
3286          * sized surface.
3287          *
3288          * To fix this properly, we need to hoist the checks up into
3289          * compute_mode_changes (or above), check the actual pfit state and
3290          * whether the platform allows pfit disable with pipe active, and only
3291          * then update the pipesrc and pfit state, even on the flip path.
3292          */
3293
3294         adjusted_mode = &crtc->config->base.adjusted_mode;
3295
3296         I915_WRITE(PIPESRC(crtc->pipe),
3297                    ((adjusted_mode->crtc_hdisplay - 1) << 16) |
3298                    (adjusted_mode->crtc_vdisplay - 1));
3299         if (!crtc->config->pch_pfit.enabled &&
3300             (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) ||
3301              intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))) {
3302                 I915_WRITE(PF_CTL(crtc->pipe), 0);
3303                 I915_WRITE(PF_WIN_POS(crtc->pipe), 0);
3304                 I915_WRITE(PF_WIN_SZ(crtc->pipe), 0);
3305         }
3306         crtc->config->pipe_src_w = adjusted_mode->crtc_hdisplay;
3307         crtc->config->pipe_src_h = adjusted_mode->crtc_vdisplay;
3308 }
3309
3310 static void intel_fdi_normal_train(struct drm_crtc *crtc)
3311 {
3312         struct drm_device *dev = crtc->dev;
3313         struct drm_i915_private *dev_priv = dev->dev_private;
3314         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3315         int pipe = intel_crtc->pipe;
3316         u32 reg, temp;
3317
3318         /* enable normal train */
3319         reg = FDI_TX_CTL(pipe);
3320         temp = I915_READ(reg);
3321         if (IS_IVYBRIDGE(dev)) {
3322                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3323                 temp |= FDI_LINK_TRAIN_NONE_IVB | FDI_TX_ENHANCE_FRAME_ENABLE;
3324         } else {
3325                 temp &= ~FDI_LINK_TRAIN_NONE;
3326                 temp |= FDI_LINK_TRAIN_NONE | FDI_TX_ENHANCE_FRAME_ENABLE;
3327         }
3328         I915_WRITE(reg, temp);
3329
3330         reg = FDI_RX_CTL(pipe);
3331         temp = I915_READ(reg);
3332         if (HAS_PCH_CPT(dev)) {
3333                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3334                 temp |= FDI_LINK_TRAIN_NORMAL_CPT;
3335         } else {
3336                 temp &= ~FDI_LINK_TRAIN_NONE;
3337                 temp |= FDI_LINK_TRAIN_NONE;
3338         }
3339         I915_WRITE(reg, temp | FDI_RX_ENHANCE_FRAME_ENABLE);
3340
3341         /* wait one idle pattern time */
3342         POSTING_READ(reg);
3343         udelay(1000);
3344
3345         /* IVB wants error correction enabled */
3346         if (IS_IVYBRIDGE(dev))
3347                 I915_WRITE(reg, I915_READ(reg) | FDI_FS_ERRC_ENABLE |
3348                            FDI_FE_ERRC_ENABLE);
3349 }
3350
3351 /* The FDI link training functions for ILK/Ibexpeak. */
3352 static void ironlake_fdi_link_train(struct drm_crtc *crtc)
3353 {
3354         struct drm_device *dev = crtc->dev;
3355         struct drm_i915_private *dev_priv = dev->dev_private;
3356         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3357         int pipe = intel_crtc->pipe;
3358         u32 reg, temp, tries;
3359
3360         /* FDI needs bits from pipe first */
3361         assert_pipe_enabled(dev_priv, pipe);
3362
3363         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3364            for train result */
3365         reg = FDI_RX_IMR(pipe);
3366         temp = I915_READ(reg);
3367         temp &= ~FDI_RX_SYMBOL_LOCK;
3368         temp &= ~FDI_RX_BIT_LOCK;
3369         I915_WRITE(reg, temp);
3370         I915_READ(reg);
3371         udelay(150);
3372
3373         /* enable CPU FDI TX and PCH FDI RX */
3374         reg = FDI_TX_CTL(pipe);
3375         temp = I915_READ(reg);
3376         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3377         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3378         temp &= ~FDI_LINK_TRAIN_NONE;
3379         temp |= FDI_LINK_TRAIN_PATTERN_1;
3380         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3381
3382         reg = FDI_RX_CTL(pipe);
3383         temp = I915_READ(reg);
3384         temp &= ~FDI_LINK_TRAIN_NONE;
3385         temp |= FDI_LINK_TRAIN_PATTERN_1;
3386         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3387
3388         POSTING_READ(reg);
3389         udelay(150);
3390
3391         /* Ironlake workaround, enable clock pointer after FDI enable*/
3392         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3393         I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR |
3394                    FDI_RX_PHASE_SYNC_POINTER_EN);
3395
3396         reg = FDI_RX_IIR(pipe);
3397         for (tries = 0; tries < 5; tries++) {
3398                 temp = I915_READ(reg);
3399                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3400
3401                 if ((temp & FDI_RX_BIT_LOCK)) {
3402                         DRM_DEBUG_KMS("FDI train 1 done.\n");
3403                         I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3404                         break;
3405                 }
3406         }
3407         if (tries == 5)
3408                 DRM_ERROR("FDI train 1 fail!\n");
3409
3410         /* Train 2 */
3411         reg = FDI_TX_CTL(pipe);
3412         temp = I915_READ(reg);
3413         temp &= ~FDI_LINK_TRAIN_NONE;
3414         temp |= FDI_LINK_TRAIN_PATTERN_2;
3415         I915_WRITE(reg, temp);
3416
3417         reg = FDI_RX_CTL(pipe);
3418         temp = I915_READ(reg);
3419         temp &= ~FDI_LINK_TRAIN_NONE;
3420         temp |= FDI_LINK_TRAIN_PATTERN_2;
3421         I915_WRITE(reg, temp);
3422
3423         POSTING_READ(reg);
3424         udelay(150);
3425
3426         reg = FDI_RX_IIR(pipe);
3427         for (tries = 0; tries < 5; tries++) {
3428                 temp = I915_READ(reg);
3429                 DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3430
3431                 if (temp & FDI_RX_SYMBOL_LOCK) {
3432                         I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3433                         DRM_DEBUG_KMS("FDI train 2 done.\n");
3434                         break;
3435                 }
3436         }
3437         if (tries == 5)
3438                 DRM_ERROR("FDI train 2 fail!\n");
3439
3440         DRM_DEBUG_KMS("FDI train done\n");
3441
3442 }
3443
3444 static const int snb_b_fdi_train_param[] = {
3445         FDI_LINK_TRAIN_400MV_0DB_SNB_B,
3446         FDI_LINK_TRAIN_400MV_6DB_SNB_B,
3447         FDI_LINK_TRAIN_600MV_3_5DB_SNB_B,
3448         FDI_LINK_TRAIN_800MV_0DB_SNB_B,
3449 };
3450
3451 /* The FDI link training functions for SNB/Cougarpoint. */
3452 static void gen6_fdi_link_train(struct drm_crtc *crtc)
3453 {
3454         struct drm_device *dev = crtc->dev;
3455         struct drm_i915_private *dev_priv = dev->dev_private;
3456         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3457         int pipe = intel_crtc->pipe;
3458         u32 reg, temp, i, retry;
3459
3460         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3461            for train result */
3462         reg = FDI_RX_IMR(pipe);
3463         temp = I915_READ(reg);
3464         temp &= ~FDI_RX_SYMBOL_LOCK;
3465         temp &= ~FDI_RX_BIT_LOCK;
3466         I915_WRITE(reg, temp);
3467
3468         POSTING_READ(reg);
3469         udelay(150);
3470
3471         /* enable CPU FDI TX and PCH FDI RX */
3472         reg = FDI_TX_CTL(pipe);
3473         temp = I915_READ(reg);
3474         temp &= ~FDI_DP_PORT_WIDTH_MASK;
3475         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3476         temp &= ~FDI_LINK_TRAIN_NONE;
3477         temp |= FDI_LINK_TRAIN_PATTERN_1;
3478         temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3479         /* SNB-B */
3480         temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
3481         I915_WRITE(reg, temp | FDI_TX_ENABLE);
3482
3483         I915_WRITE(FDI_RX_MISC(pipe),
3484                    FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
3485
3486         reg = FDI_RX_CTL(pipe);
3487         temp = I915_READ(reg);
3488         if (HAS_PCH_CPT(dev)) {
3489                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3490                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3491         } else {
3492                 temp &= ~FDI_LINK_TRAIN_NONE;
3493                 temp |= FDI_LINK_TRAIN_PATTERN_1;
3494         }
3495         I915_WRITE(reg, temp | FDI_RX_ENABLE);
3496
3497         POSTING_READ(reg);
3498         udelay(150);
3499
3500         for (i = 0; i < 4; i++) {
3501                 reg = FDI_TX_CTL(pipe);
3502                 temp = I915_READ(reg);
3503                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3504                 temp |= snb_b_fdi_train_param[i];
3505                 I915_WRITE(reg, temp);
3506
3507                 POSTING_READ(reg);
3508                 udelay(500);
3509
3510                 for (retry = 0; retry < 5; retry++) {
3511                         reg = FDI_RX_IIR(pipe);
3512                         temp = I915_READ(reg);
3513                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3514                         if (temp & FDI_RX_BIT_LOCK) {
3515                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3516                                 DRM_DEBUG_KMS("FDI train 1 done.\n");
3517                                 break;
3518                         }
3519                         udelay(50);
3520                 }
3521                 if (retry < 5)
3522                         break;
3523         }
3524         if (i == 4)
3525                 DRM_ERROR("FDI train 1 fail!\n");
3526
3527         /* Train 2 */
3528         reg = FDI_TX_CTL(pipe);
3529         temp = I915_READ(reg);
3530         temp &= ~FDI_LINK_TRAIN_NONE;
3531         temp |= FDI_LINK_TRAIN_PATTERN_2;
3532         if (IS_GEN6(dev)) {
3533                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3534                 /* SNB-B */
3535                 temp |= FDI_LINK_TRAIN_400MV_0DB_SNB_B;
3536         }
3537         I915_WRITE(reg, temp);
3538
3539         reg = FDI_RX_CTL(pipe);
3540         temp = I915_READ(reg);
3541         if (HAS_PCH_CPT(dev)) {
3542                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3543                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
3544         } else {
3545                 temp &= ~FDI_LINK_TRAIN_NONE;
3546                 temp |= FDI_LINK_TRAIN_PATTERN_2;
3547         }
3548         I915_WRITE(reg, temp);
3549
3550         POSTING_READ(reg);
3551         udelay(150);
3552
3553         for (i = 0; i < 4; i++) {
3554                 reg = FDI_TX_CTL(pipe);
3555                 temp = I915_READ(reg);
3556                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3557                 temp |= snb_b_fdi_train_param[i];
3558                 I915_WRITE(reg, temp);
3559
3560                 POSTING_READ(reg);
3561                 udelay(500);
3562
3563                 for (retry = 0; retry < 5; retry++) {
3564                         reg = FDI_RX_IIR(pipe);
3565                         temp = I915_READ(reg);
3566                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3567                         if (temp & FDI_RX_SYMBOL_LOCK) {
3568                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3569                                 DRM_DEBUG_KMS("FDI train 2 done.\n");
3570                                 break;
3571                         }
3572                         udelay(50);
3573                 }
3574                 if (retry < 5)
3575                         break;
3576         }
3577         if (i == 4)
3578                 DRM_ERROR("FDI train 2 fail!\n");
3579
3580         DRM_DEBUG_KMS("FDI train done.\n");
3581 }
3582
3583 /* Manual link training for Ivy Bridge A0 parts */
3584 static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
3585 {
3586         struct drm_device *dev = crtc->dev;
3587         struct drm_i915_private *dev_priv = dev->dev_private;
3588         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3589         int pipe = intel_crtc->pipe;
3590         u32 reg, temp, i, j;
3591
3592         /* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
3593            for train result */
3594         reg = FDI_RX_IMR(pipe);
3595         temp = I915_READ(reg);
3596         temp &= ~FDI_RX_SYMBOL_LOCK;
3597         temp &= ~FDI_RX_BIT_LOCK;
3598         I915_WRITE(reg, temp);
3599
3600         POSTING_READ(reg);
3601         udelay(150);
3602
3603         DRM_DEBUG_KMS("FDI_RX_IIR before link train 0x%x\n",
3604                       I915_READ(FDI_RX_IIR(pipe)));
3605
3606         /* Try each vswing and preemphasis setting twice before moving on */
3607         for (j = 0; j < ARRAY_SIZE(snb_b_fdi_train_param) * 2; j++) {
3608                 /* disable first in case we need to retry */
3609                 reg = FDI_TX_CTL(pipe);
3610                 temp = I915_READ(reg);
3611                 temp &= ~(FDI_LINK_TRAIN_AUTO | FDI_LINK_TRAIN_NONE_IVB);
3612                 temp &= ~FDI_TX_ENABLE;
3613                 I915_WRITE(reg, temp);
3614
3615                 reg = FDI_RX_CTL(pipe);
3616                 temp = I915_READ(reg);
3617                 temp &= ~FDI_LINK_TRAIN_AUTO;
3618                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3619                 temp &= ~FDI_RX_ENABLE;
3620                 I915_WRITE(reg, temp);
3621
3622                 /* enable CPU FDI TX and PCH FDI RX */
3623                 reg = FDI_TX_CTL(pipe);
3624                 temp = I915_READ(reg);
3625                 temp &= ~FDI_DP_PORT_WIDTH_MASK;
3626                 temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3627                 temp |= FDI_LINK_TRAIN_PATTERN_1_IVB;
3628                 temp &= ~FDI_LINK_TRAIN_VOL_EMP_MASK;
3629                 temp |= snb_b_fdi_train_param[j/2];
3630                 temp |= FDI_COMPOSITE_SYNC;
3631                 I915_WRITE(reg, temp | FDI_TX_ENABLE);
3632
3633                 I915_WRITE(FDI_RX_MISC(pipe),
3634                            FDI_RX_TP1_TO_TP2_48 | FDI_RX_FDI_DELAY_90);
3635
3636                 reg = FDI_RX_CTL(pipe);
3637                 temp = I915_READ(reg);
3638                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3639                 temp |= FDI_COMPOSITE_SYNC;
3640                 I915_WRITE(reg, temp | FDI_RX_ENABLE);
3641
3642                 POSTING_READ(reg);
3643                 udelay(1); /* should be 0.5us */
3644
3645                 for (i = 0; i < 4; i++) {
3646                         reg = FDI_RX_IIR(pipe);
3647                         temp = I915_READ(reg);
3648                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3649
3650                         if (temp & FDI_RX_BIT_LOCK ||
3651                             (I915_READ(reg) & FDI_RX_BIT_LOCK)) {
3652                                 I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
3653                                 DRM_DEBUG_KMS("FDI train 1 done, level %i.\n",
3654                                               i);
3655                                 break;
3656                         }
3657                         udelay(1); /* should be 0.5us */
3658                 }
3659                 if (i == 4) {
3660                         DRM_DEBUG_KMS("FDI train 1 fail on vswing %d\n", j / 2);
3661                         continue;
3662                 }
3663
3664                 /* Train 2 */
3665                 reg = FDI_TX_CTL(pipe);
3666                 temp = I915_READ(reg);
3667                 temp &= ~FDI_LINK_TRAIN_NONE_IVB;
3668                 temp |= FDI_LINK_TRAIN_PATTERN_2_IVB;
3669                 I915_WRITE(reg, temp);
3670
3671                 reg = FDI_RX_CTL(pipe);
3672                 temp = I915_READ(reg);
3673                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3674                 temp |= FDI_LINK_TRAIN_PATTERN_2_CPT;
3675                 I915_WRITE(reg, temp);
3676
3677                 POSTING_READ(reg);
3678                 udelay(2); /* should be 1.5us */
3679
3680                 for (i = 0; i < 4; i++) {
3681                         reg = FDI_RX_IIR(pipe);
3682                         temp = I915_READ(reg);
3683                         DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
3684
3685                         if (temp & FDI_RX_SYMBOL_LOCK ||
3686                             (I915_READ(reg) & FDI_RX_SYMBOL_LOCK)) {
3687                                 I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
3688                                 DRM_DEBUG_KMS("FDI train 2 done, level %i.\n",
3689                                               i);
3690                                 goto train_done;
3691                         }
3692                         udelay(2); /* should be 1.5us */
3693                 }
3694                 if (i == 4)
3695                         DRM_DEBUG_KMS("FDI train 2 fail on vswing %d\n", j / 2);
3696         }
3697
3698 train_done:
3699         DRM_DEBUG_KMS("FDI train done.\n");
3700 }
3701
3702 static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
3703 {
3704         struct drm_device *dev = intel_crtc->base.dev;
3705         struct drm_i915_private *dev_priv = dev->dev_private;
3706         int pipe = intel_crtc->pipe;
3707         u32 reg, temp;
3708
3709
3710         /* enable PCH FDI RX PLL, wait warmup plus DMI latency */
3711         reg = FDI_RX_CTL(pipe);
3712         temp = I915_READ(reg);
3713         temp &= ~(FDI_DP_PORT_WIDTH_MASK | (0x7 << 16));
3714         temp |= FDI_DP_PORT_WIDTH(intel_crtc->config->fdi_lanes);
3715         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3716         I915_WRITE(reg, temp | FDI_RX_PLL_ENABLE);
3717
3718         POSTING_READ(reg);
3719         udelay(200);
3720
3721         /* Switch from Rawclk to PCDclk */
3722         temp = I915_READ(reg);
3723         I915_WRITE(reg, temp | FDI_PCDCLK);
3724
3725         POSTING_READ(reg);
3726         udelay(200);
3727
3728         /* Enable CPU FDI TX PLL, always on for Ironlake */
3729         reg = FDI_TX_CTL(pipe);
3730         temp = I915_READ(reg);
3731         if ((temp & FDI_TX_PLL_ENABLE) == 0) {
3732                 I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
3733
3734                 POSTING_READ(reg);
3735                 udelay(100);
3736         }
3737 }
3738
3739 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
3740 {
3741         struct drm_device *dev = intel_crtc->base.dev;
3742         struct drm_i915_private *dev_priv = dev->dev_private;
3743         int pipe = intel_crtc->pipe;
3744         u32 reg, temp;
3745
3746         /* Switch from PCDclk to Rawclk */
3747         reg = FDI_RX_CTL(pipe);
3748         temp = I915_READ(reg);
3749         I915_WRITE(reg, temp & ~FDI_PCDCLK);
3750
3751         /* Disable CPU FDI TX PLL */
3752         reg = FDI_TX_CTL(pipe);
3753         temp = I915_READ(reg);
3754         I915_WRITE(reg, temp & ~FDI_TX_PLL_ENABLE);
3755
3756         POSTING_READ(reg);
3757         udelay(100);
3758
3759         reg = FDI_RX_CTL(pipe);
3760         temp = I915_READ(reg);
3761         I915_WRITE(reg, temp & ~FDI_RX_PLL_ENABLE);
3762
3763         /* Wait for the clocks to turn off. */
3764         POSTING_READ(reg);
3765         udelay(100);
3766 }
3767
3768 static void ironlake_fdi_disable(struct drm_crtc *crtc)
3769 {
3770         struct drm_device *dev = crtc->dev;
3771         struct drm_i915_private *dev_priv = dev->dev_private;
3772         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3773         int pipe = intel_crtc->pipe;
3774         u32 reg, temp;
3775
3776         /* disable CPU FDI tx and PCH FDI rx */
3777         reg = FDI_TX_CTL(pipe);
3778         temp = I915_READ(reg);
3779         I915_WRITE(reg, temp & ~FDI_TX_ENABLE);
3780         POSTING_READ(reg);
3781
3782         reg = FDI_RX_CTL(pipe);
3783         temp = I915_READ(reg);
3784         temp &= ~(0x7 << 16);
3785         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3786         I915_WRITE(reg, temp & ~FDI_RX_ENABLE);
3787
3788         POSTING_READ(reg);
3789         udelay(100);
3790
3791         /* Ironlake workaround, disable clock pointer after downing FDI */
3792         if (HAS_PCH_IBX(dev))
3793                 I915_WRITE(FDI_RX_CHICKEN(pipe), FDI_RX_PHASE_SYNC_POINTER_OVR);
3794
3795         /* still set train pattern 1 */
3796         reg = FDI_TX_CTL(pipe);
3797         temp = I915_READ(reg);
3798         temp &= ~FDI_LINK_TRAIN_NONE;
3799         temp |= FDI_LINK_TRAIN_PATTERN_1;
3800         I915_WRITE(reg, temp);
3801
3802         reg = FDI_RX_CTL(pipe);
3803         temp = I915_READ(reg);
3804         if (HAS_PCH_CPT(dev)) {
3805                 temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT;
3806                 temp |= FDI_LINK_TRAIN_PATTERN_1_CPT;
3807         } else {
3808                 temp &= ~FDI_LINK_TRAIN_NONE;
3809                 temp |= FDI_LINK_TRAIN_PATTERN_1;
3810         }
3811         /* BPC in FDI rx is consistent with that in PIPECONF */
3812         temp &= ~(0x07 << 16);
3813         temp |= (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) << 11;
3814         I915_WRITE(reg, temp);
3815
3816         POSTING_READ(reg);
3817         udelay(100);
3818 }
3819
3820 bool intel_has_pending_fb_unpin(struct drm_device *dev)
3821 {
3822         struct intel_crtc *crtc;
3823
3824         /* Note that we don't need to be called with mode_config.lock here
3825          * as our list of CRTC objects is static for the lifetime of the
3826          * device and so cannot disappear as we iterate. Similarly, we can
3827          * happily treat the predicates as racy, atomic checks as userspace
3828          * cannot claim and pin a new fb without at least acquring the
3829          * struct_mutex and so serialising with us.
3830          */
3831         for_each_intel_crtc(dev, crtc) {
3832                 if (atomic_read(&crtc->unpin_work_count) == 0)
3833                         continue;
3834
3835                 if (crtc->unpin_work)
3836                         intel_wait_for_vblank(dev, crtc->pipe);
3837
3838                 return true;
3839         }
3840
3841         return false;
3842 }
3843
3844 static void page_flip_completed(struct intel_crtc *intel_crtc)
3845 {
3846         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
3847         struct intel_unpin_work *work = intel_crtc->unpin_work;
3848
3849         /* ensure that the unpin work is consistent wrt ->pending. */
3850         smp_rmb();
3851         intel_crtc->unpin_work = NULL;
3852
3853         if (work->event)
3854                 drm_send_vblank_event(intel_crtc->base.dev,
3855                                       intel_crtc->pipe,
3856                                       work->event);
3857
3858         drm_crtc_vblank_put(&intel_crtc->base);
3859
3860         wake_up_all(&dev_priv->pending_flip_queue);
3861         queue_work(dev_priv->wq, &work->work);
3862
3863         trace_i915_flip_complete(intel_crtc->plane,
3864                                  work->pending_flip_obj);
3865 }
3866
3867 void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
3868 {
3869         struct drm_device *dev = crtc->dev;
3870         struct drm_i915_private *dev_priv = dev->dev_private;
3871
3872         WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
3873         if (WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue,
3874                                        !intel_crtc_has_pending_flip(crtc),
3875                                        60*HZ) == 0)) {
3876                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
3877
3878                 spin_lock_irq(&dev->event_lock);
3879                 if (intel_crtc->unpin_work) {
3880                         WARN_ONCE(1, "Removing stuck page flip\n");
3881                         page_flip_completed(intel_crtc);
3882                 }
3883                 spin_unlock_irq(&dev->event_lock);
3884         }
3885
3886         if (crtc->primary->fb) {
3887                 mutex_lock(&dev->struct_mutex);
3888                 intel_finish_fb(crtc->primary->fb);
3889                 mutex_unlock(&dev->struct_mutex);
3890         }
3891 }
3892
3893 /* Program iCLKIP clock to the desired frequency */
3894 static void lpt_program_iclkip(struct drm_crtc *crtc)
3895 {
3896         struct drm_device *dev = crtc->dev;
3897         struct drm_i915_private *dev_priv = dev->dev_private;
3898         int clock = to_intel_crtc(crtc)->config->base.adjusted_mode.crtc_clock;
3899         u32 divsel, phaseinc, auxdiv, phasedir = 0;
3900         u32 temp;
3901
3902         mutex_lock(&dev_priv->sb_lock);
3903
3904         /* It is necessary to ungate the pixclk gate prior to programming
3905          * the divisors, and gate it back when it is done.
3906          */
3907         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
3908
3909         /* Disable SSCCTL */
3910         intel_sbi_write(dev_priv, SBI_SSCCTL6,
3911                         intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK) |
3912                                 SBI_SSCCTL_DISABLE,
3913                         SBI_ICLK);
3914
3915         /* 20MHz is a corner case which is out of range for the 7-bit divisor */
3916         if (clock == 20000) {
3917                 auxdiv = 1;
3918                 divsel = 0x41;
3919                 phaseinc = 0x20;
3920         } else {
3921                 /* The iCLK virtual clock root frequency is in MHz,
3922                  * but the adjusted_mode->crtc_clock in in KHz. To get the
3923                  * divisors, it is necessary to divide one by another, so we
3924                  * convert the virtual clock precision to KHz here for higher
3925                  * precision.
3926                  */
3927                 u32 iclk_virtual_root_freq = 172800 * 1000;
3928                 u32 iclk_pi_range = 64;
3929                 u32 desired_divisor, msb_divisor_value, pi_value;
3930
3931                 desired_divisor = (iclk_virtual_root_freq / clock);
3932                 msb_divisor_value = desired_divisor / iclk_pi_range;
3933                 pi_value = desired_divisor % iclk_pi_range;
3934
3935                 auxdiv = 0;
3936                 divsel = msb_divisor_value - 2;
3937                 phaseinc = pi_value;
3938         }
3939
3940         /* This should not happen with any sane values */
3941         WARN_ON(SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
3942                 ~SBI_SSCDIVINTPHASE_DIVSEL_MASK);
3943         WARN_ON(SBI_SSCDIVINTPHASE_DIR(phasedir) &
3944                 ~SBI_SSCDIVINTPHASE_INCVAL_MASK);
3945
3946         DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
3947                         clock,
3948                         auxdiv,
3949                         divsel,
3950                         phasedir,
3951                         phaseinc);
3952
3953         /* Program SSCDIVINTPHASE6 */
3954         temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6, SBI_ICLK);
3955         temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
3956         temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
3957         temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
3958         temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
3959         temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
3960         temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
3961         intel_sbi_write(dev_priv, SBI_SSCDIVINTPHASE6, temp, SBI_ICLK);
3962
3963         /* Program SSCAUXDIV */
3964         temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6, SBI_ICLK);
3965         temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
3966         temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
3967         intel_sbi_write(dev_priv, SBI_SSCAUXDIV6, temp, SBI_ICLK);
3968
3969         /* Enable modulator and associated divider */
3970         temp = intel_sbi_read(dev_priv, SBI_SSCCTL6, SBI_ICLK);
3971         temp &= ~SBI_SSCCTL_DISABLE;
3972         intel_sbi_write(dev_priv, SBI_SSCCTL6, temp, SBI_ICLK);
3973
3974         /* Wait for initialization time */
3975         udelay(24);
3976
3977         I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
3978
3979         mutex_unlock(&dev_priv->sb_lock);
3980 }
3981
3982 static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
3983                                                 enum pipe pch_transcoder)
3984 {
3985         struct drm_device *dev = crtc->base.dev;
3986         struct drm_i915_private *dev_priv = dev->dev_private;
3987         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
3988
3989         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
3990                    I915_READ(HTOTAL(cpu_transcoder)));
3991         I915_WRITE(PCH_TRANS_HBLANK(pch_transcoder),
3992                    I915_READ(HBLANK(cpu_transcoder)));
3993         I915_WRITE(PCH_TRANS_HSYNC(pch_transcoder),
3994                    I915_READ(HSYNC(cpu_transcoder)));
3995
3996         I915_WRITE(PCH_TRANS_VTOTAL(pch_transcoder),
3997                    I915_READ(VTOTAL(cpu_transcoder)));
3998         I915_WRITE(PCH_TRANS_VBLANK(pch_transcoder),
3999                    I915_READ(VBLANK(cpu_transcoder)));
4000         I915_WRITE(PCH_TRANS_VSYNC(pch_transcoder),
4001                    I915_READ(VSYNC(cpu_transcoder)));
4002         I915_WRITE(PCH_TRANS_VSYNCSHIFT(pch_transcoder),
4003                    I915_READ(VSYNCSHIFT(cpu_transcoder)));
4004 }
4005
4006 static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable)
4007 {
4008         struct drm_i915_private *dev_priv = dev->dev_private;
4009         uint32_t temp;
4010
4011         temp = I915_READ(SOUTH_CHICKEN1);
4012         if (!!(temp & FDI_BC_BIFURCATION_SELECT) == enable)
4013                 return;
4014
4015         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_B)) & FDI_RX_ENABLE);
4016         WARN_ON(I915_READ(FDI_RX_CTL(PIPE_C)) & FDI_RX_ENABLE);
4017
4018         temp &= ~FDI_BC_BIFURCATION_SELECT;
4019         if (enable)
4020                 temp |= FDI_BC_BIFURCATION_SELECT;
4021
4022         DRM_DEBUG_KMS("%sabling fdi C rx\n", enable ? "en" : "dis");
4023         I915_WRITE(SOUTH_CHICKEN1, temp);
4024         POSTING_READ(SOUTH_CHICKEN1);
4025 }
4026
4027 static void ivybridge_update_fdi_bc_bifurcation(struct intel_crtc *intel_crtc)
4028 {
4029         struct drm_device *dev = intel_crtc->base.dev;
4030
4031         switch (intel_crtc->pipe) {
4032         case PIPE_A:
4033                 break;
4034         case PIPE_B:
4035                 if (intel_crtc->config->fdi_lanes > 2)
4036                         cpt_set_fdi_bc_bifurcation(dev, false);
4037                 else
4038                         cpt_set_fdi_bc_bifurcation(dev, true);
4039
4040                 break;
4041         case PIPE_C:
4042                 cpt_set_fdi_bc_bifurcation(dev, true);
4043
4044                 break;
4045         default:
4046                 BUG();
4047         }
4048 }
4049
4050 /*
4051  * Enable PCH resources required for PCH ports:
4052  *   - PCH PLLs
4053  *   - FDI training & RX/TX
4054  *   - update transcoder timings
4055  *   - DP transcoding bits
4056  *   - transcoder
4057  */
4058 static void ironlake_pch_enable(struct drm_crtc *crtc)
4059 {
4060         struct drm_device *dev = crtc->dev;
4061         struct drm_i915_private *dev_priv = dev->dev_private;
4062         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4063         int pipe = intel_crtc->pipe;
4064         u32 reg, temp;
4065
4066         assert_pch_transcoder_disabled(dev_priv, pipe);
4067
4068         if (IS_IVYBRIDGE(dev))
4069                 ivybridge_update_fdi_bc_bifurcation(intel_crtc);
4070
4071         /* Write the TU size bits before fdi link training, so that error
4072          * detection works. */
4073         I915_WRITE(FDI_RX_TUSIZE1(pipe),
4074                    I915_READ(PIPE_DATA_M1(pipe)) & TU_SIZE_MASK);
4075
4076         /* For PCH output, training FDI link */
4077         dev_priv->display.fdi_link_train(crtc);
4078
4079         /* We need to program the right clock selection before writing the pixel
4080          * mutliplier into the DPLL. */
4081         if (HAS_PCH_CPT(dev)) {
4082                 u32 sel;
4083
4084                 temp = I915_READ(PCH_DPLL_SEL);
4085                 temp |= TRANS_DPLL_ENABLE(pipe);
4086                 sel = TRANS_DPLLB_SEL(pipe);
4087                 if (intel_crtc->config->shared_dpll == DPLL_ID_PCH_PLL_B)
4088                         temp |= sel;
4089                 else
4090                         temp &= ~sel;
4091                 I915_WRITE(PCH_DPLL_SEL, temp);
4092         }
4093
4094         /* XXX: pch pll's can be enabled any time before we enable the PCH
4095          * transcoder, and we actually should do this to not upset any PCH
4096          * transcoder that already use the clock when we share it.
4097          *
4098          * Note that enable_shared_dpll tries to do the right thing, but
4099          * get_shared_dpll unconditionally resets the pll - we need that to have
4100          * the right LVDS enable sequence. */
4101         intel_enable_shared_dpll(intel_crtc);
4102
4103         /* set transcoder timing, panel must allow it */
4104         assert_panel_unlocked(dev_priv, pipe);
4105         ironlake_pch_transcoder_set_timings(intel_crtc, pipe);
4106
4107         intel_fdi_normal_train(crtc);
4108
4109         /* For PCH DP, enable TRANS_DP_CTL */
4110         if (HAS_PCH_CPT(dev) && intel_crtc->config->has_dp_encoder) {
4111                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
4112                 reg = TRANS_DP_CTL(pipe);
4113                 temp = I915_READ(reg);
4114                 temp &= ~(TRANS_DP_PORT_SEL_MASK |
4115                           TRANS_DP_SYNC_MASK |
4116                           TRANS_DP_BPC_MASK);
4117                 temp |= TRANS_DP_OUTPUT_ENABLE;
4118                 temp |= bpc << 9; /* same format but at 11:9 */
4119
4120                 if (crtc->mode.flags & DRM_MODE_FLAG_PHSYNC)
4121                         temp |= TRANS_DP_HSYNC_ACTIVE_HIGH;
4122                 if (crtc->mode.flags & DRM_MODE_FLAG_PVSYNC)
4123                         temp |= TRANS_DP_VSYNC_ACTIVE_HIGH;
4124
4125                 switch (intel_trans_dp_port_sel(crtc)) {
4126                 case PCH_DP_B:
4127                         temp |= TRANS_DP_PORT_SEL_B;
4128                         break;
4129                 case PCH_DP_C:
4130                         temp |= TRANS_DP_PORT_SEL_C;
4131                         break;
4132                 case PCH_DP_D:
4133                         temp |= TRANS_DP_PORT_SEL_D;
4134                         break;
4135                 default:
4136                         BUG();
4137                 }
4138
4139                 I915_WRITE(reg, temp);
4140         }
4141
4142         ironlake_enable_pch_transcoder(dev_priv, pipe);
4143 }
4144
4145 static void lpt_pch_enable(struct drm_crtc *crtc)
4146 {
4147         struct drm_device *dev = crtc->dev;
4148         struct drm_i915_private *dev_priv = dev->dev_private;
4149         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4150         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
4151
4152         assert_pch_transcoder_disabled(dev_priv, TRANSCODER_A);
4153
4154         lpt_program_iclkip(crtc);
4155
4156         /* Set transcoder timing. */
4157         ironlake_pch_transcoder_set_timings(intel_crtc, PIPE_A);
4158
4159         lpt_enable_pch_transcoder(dev_priv, cpu_transcoder);
4160 }
4161
4162 struct intel_shared_dpll *intel_get_shared_dpll(struct intel_crtc *crtc,
4163                                                 struct intel_crtc_state *crtc_state)
4164 {
4165         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
4166         struct intel_shared_dpll *pll;
4167         struct intel_shared_dpll_config *shared_dpll;
4168         enum intel_dpll_id i;
4169
4170         shared_dpll = intel_atomic_get_shared_dpll_state(crtc_state->base.state);
4171
4172         if (HAS_PCH_IBX(dev_priv->dev)) {
4173                 /* Ironlake PCH has a fixed PLL->PCH pipe mapping. */
4174                 i = (enum intel_dpll_id) crtc->pipe;
4175                 pll = &dev_priv->shared_dplls[i];
4176
4177                 DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n",
4178                               crtc->base.base.id, pll->name);
4179
4180                 WARN_ON(shared_dpll[i].crtc_mask);
4181
4182                 goto found;
4183         }
4184
4185         if (IS_BROXTON(dev_priv->dev)) {
4186                 /* PLL is attached to port in bxt */
4187                 struct intel_encoder *encoder;
4188                 struct intel_digital_port *intel_dig_port;
4189
4190                 encoder = intel_ddi_get_crtc_new_encoder(crtc_state);
4191                 if (WARN_ON(!encoder))
4192                         return NULL;
4193
4194                 intel_dig_port = enc_to_dig_port(&encoder->base);
4195                 /* 1:1 mapping between ports and PLLs */
4196                 i = (enum intel_dpll_id)intel_dig_port->port;
4197                 pll = &dev_priv->shared_dplls[i];
4198                 DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n",
4199                         crtc->base.base.id, pll->name);
4200                 WARN_ON(shared_dpll[i].crtc_mask);
4201
4202                 goto found;
4203         }
4204
4205         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
4206                 pll = &dev_priv->shared_dplls[i];
4207
4208                 /* Only want to check enabled timings first */
4209                 if (shared_dpll[i].crtc_mask == 0)
4210                         continue;
4211
4212                 if (memcmp(&crtc_state->dpll_hw_state,
4213                            &shared_dpll[i].hw_state,
4214                            sizeof(crtc_state->dpll_hw_state)) == 0) {
4215                         DRM_DEBUG_KMS("CRTC:%d sharing existing %s (crtc mask 0x%08x, ative %d)\n",
4216                                       crtc->base.base.id, pll->name,
4217                                       shared_dpll[i].crtc_mask,
4218                                       pll->active);
4219                         goto found;
4220                 }
4221         }
4222
4223         /* Ok no matching timings, maybe there's a free one? */
4224         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
4225                 pll = &dev_priv->shared_dplls[i];
4226                 if (shared_dpll[i].crtc_mask == 0) {
4227                         DRM_DEBUG_KMS("CRTC:%d allocated %s\n",
4228                                       crtc->base.base.id, pll->name);
4229                         goto found;
4230                 }
4231         }
4232
4233         return NULL;
4234
4235 found:
4236         if (shared_dpll[i].crtc_mask == 0)
4237                 shared_dpll[i].hw_state =
4238                         crtc_state->dpll_hw_state;
4239
4240         crtc_state->shared_dpll = i;
4241         DRM_DEBUG_DRIVER("using %s for pipe %c\n", pll->name,
4242                          pipe_name(crtc->pipe));
4243
4244         shared_dpll[i].crtc_mask |= 1 << crtc->pipe;
4245
4246         return pll;
4247 }
4248
4249 static void intel_shared_dpll_commit(struct drm_atomic_state *state)
4250 {
4251         struct drm_i915_private *dev_priv = to_i915(state->dev);
4252         struct intel_shared_dpll_config *shared_dpll;
4253         struct intel_shared_dpll *pll;
4254         enum intel_dpll_id i;
4255
4256         if (!to_intel_atomic_state(state)->dpll_set)
4257                 return;
4258
4259         shared_dpll = to_intel_atomic_state(state)->shared_dpll;
4260         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
4261                 pll = &dev_priv->shared_dplls[i];
4262                 pll->config = shared_dpll[i];
4263         }
4264 }
4265
4266 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
4267 {
4268         struct drm_i915_private *dev_priv = dev->dev_private;
4269         int dslreg = PIPEDSL(pipe);
4270         u32 temp;
4271
4272         temp = I915_READ(dslreg);
4273         udelay(500);
4274         if (wait_for(I915_READ(dslreg) != temp, 5)) {
4275                 if (wait_for(I915_READ(dslreg) != temp, 5))
4276                         DRM_ERROR("mode set failed: pipe %c stuck\n", pipe_name(pipe));
4277         }
4278 }
4279
4280 static int
4281 skl_update_scaler(struct intel_crtc_state *crtc_state, bool force_detach,
4282                   unsigned scaler_user, int *scaler_id, unsigned int rotation,
4283                   int src_w, int src_h, int dst_w, int dst_h)
4284 {
4285         struct intel_crtc_scaler_state *scaler_state =
4286                 &crtc_state->scaler_state;
4287         struct intel_crtc *intel_crtc =
4288                 to_intel_crtc(crtc_state->base.crtc);
4289         int need_scaling;
4290
4291         need_scaling = intel_rotation_90_or_270(rotation) ?
4292                 (src_h != dst_w || src_w != dst_h):
4293                 (src_w != dst_w || src_h != dst_h);
4294
4295         /*
4296          * if plane is being disabled or scaler is no more required or force detach
4297          *  - free scaler binded to this plane/crtc
4298          *  - in order to do this, update crtc->scaler_usage
4299          *
4300          * Here scaler state in crtc_state is set free so that
4301          * scaler can be assigned to other user. Actual register
4302          * update to free the scaler is done in plane/panel-fit programming.
4303          * For this purpose crtc/plane_state->scaler_id isn't reset here.
4304          */
4305         if (force_detach || !need_scaling) {
4306                 if (*scaler_id >= 0) {
4307                         scaler_state->scaler_users &= ~(1 << scaler_user);
4308                         scaler_state->scalers[*scaler_id].in_use = 0;
4309
4310                         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4311                                 "Staged freeing scaler id %d scaler_users = 0x%x\n",
4312                                 intel_crtc->pipe, scaler_user, *scaler_id,
4313                                 scaler_state->scaler_users);
4314                         *scaler_id = -1;
4315                 }
4316                 return 0;
4317         }
4318
4319         /* range checks */
4320         if (src_w < SKL_MIN_SRC_W || src_h < SKL_MIN_SRC_H ||
4321                 dst_w < SKL_MIN_DST_W || dst_h < SKL_MIN_DST_H ||
4322
4323                 src_w > SKL_MAX_SRC_W || src_h > SKL_MAX_SRC_H ||
4324                 dst_w > SKL_MAX_DST_W || dst_h > SKL_MAX_DST_H) {
4325                 DRM_DEBUG_KMS("scaler_user index %u.%u: src %ux%u dst %ux%u "
4326                         "size is out of scaler range\n",
4327                         intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h);
4328                 return -EINVAL;
4329         }
4330
4331         /* mark this plane as a scaler user in crtc_state */
4332         scaler_state->scaler_users |= (1 << scaler_user);
4333         DRM_DEBUG_KMS("scaler_user index %u.%u: "
4334                 "staged scaling request for %ux%u->%ux%u scaler_users = 0x%x\n",
4335                 intel_crtc->pipe, scaler_user, src_w, src_h, dst_w, dst_h,
4336                 scaler_state->scaler_users);
4337
4338         return 0;
4339 }
4340
4341 /**
4342  * skl_update_scaler_crtc - Stages update to scaler state for a given crtc.
4343  *
4344  * @state: crtc's scaler state
4345  * @force_detach: whether to forcibly disable scaler
4346  *
4347  * Return
4348  *     0 - scaler_usage updated successfully
4349  *    error - requested scaling cannot be supported or other error condition
4350  */
4351 int skl_update_scaler_crtc(struct intel_crtc_state *state, int force_detach)
4352 {
4353         struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc);
4354         struct drm_display_mode *adjusted_mode =
4355                 &state->base.adjusted_mode;
4356
4357         DRM_DEBUG_KMS("Updating scaler for [CRTC:%i] scaler_user index %u.%u\n",
4358                       intel_crtc->base.base.id, intel_crtc->pipe, SKL_CRTC_INDEX);
4359
4360         return skl_update_scaler(state, force_detach, SKL_CRTC_INDEX,
4361                 &state->scaler_state.scaler_id, DRM_ROTATE_0,
4362                 state->pipe_src_w, state->pipe_src_h,
4363                 adjusted_mode->hdisplay, adjusted_mode->vdisplay);
4364 }
4365
4366 /**
4367  * skl_update_scaler_plane - Stages update to scaler state for a given plane.
4368  *
4369  * @state: crtc's scaler state
4370  * @plane_state: atomic plane state to update
4371  *
4372  * Return
4373  *     0 - scaler_usage updated successfully
4374  *    error - requested scaling cannot be supported or other error condition
4375  */
4376 static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
4377                                    struct intel_plane_state *plane_state)
4378 {
4379
4380         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
4381         struct intel_plane *intel_plane =
4382                 to_intel_plane(plane_state->base.plane);
4383         struct drm_framebuffer *fb = plane_state->base.fb;
4384         int ret;
4385
4386         bool force_detach = !fb || !plane_state->visible;
4387
4388         DRM_DEBUG_KMS("Updating scaler for [PLANE:%d] scaler_user index %u.%u\n",
4389                       intel_plane->base.base.id, intel_crtc->pipe,
4390                       drm_plane_index(&intel_plane->base));
4391
4392         ret = skl_update_scaler(crtc_state, force_detach,
4393                                 drm_plane_index(&intel_plane->base),
4394                                 &plane_state->scaler_id,
4395                                 plane_state->base.rotation,
4396                                 drm_rect_width(&plane_state->src) >> 16,
4397                                 drm_rect_height(&plane_state->src) >> 16,
4398                                 drm_rect_width(&plane_state->dst),
4399                                 drm_rect_height(&plane_state->dst));
4400
4401         if (ret || plane_state->scaler_id < 0)
4402                 return ret;
4403
4404         /* check colorkey */
4405         if (plane_state->ckey.flags != I915_SET_COLORKEY_NONE) {
4406                 DRM_DEBUG_KMS("[PLANE:%d] scaling with color key not allowed",
4407                               intel_plane->base.base.id);
4408                 return -EINVAL;
4409         }
4410
4411         /* Check src format */
4412         switch (fb->pixel_format) {
4413         case DRM_FORMAT_RGB565:
4414         case DRM_FORMAT_XBGR8888:
4415         case DRM_FORMAT_XRGB8888:
4416         case DRM_FORMAT_ABGR8888:
4417         case DRM_FORMAT_ARGB8888:
4418         case DRM_FORMAT_XRGB2101010:
4419         case DRM_FORMAT_XBGR2101010:
4420         case DRM_FORMAT_YUYV:
4421         case DRM_FORMAT_YVYU:
4422         case DRM_FORMAT_UYVY:
4423         case DRM_FORMAT_VYUY:
4424                 break;
4425         default:
4426                 DRM_DEBUG_KMS("[PLANE:%d] FB:%d unsupported scaling format 0x%x\n",
4427                         intel_plane->base.base.id, fb->base.id, fb->pixel_format);
4428                 return -EINVAL;
4429         }
4430
4431         return 0;
4432 }
4433
4434 static void skylake_pfit_update(struct intel_crtc *crtc, int enable)
4435 {
4436         struct drm_device *dev = crtc->base.dev;
4437         struct drm_i915_private *dev_priv = dev->dev_private;
4438         int pipe = crtc->pipe;
4439         struct intel_crtc_scaler_state *scaler_state =
4440                 &crtc->config->scaler_state;
4441
4442         DRM_DEBUG_KMS("for crtc_state = %p\n", crtc->config);
4443
4444         /* To update pfit, first update scaler state */
4445         skl_update_scaler_crtc(crtc->config, !enable);
4446         intel_atomic_setup_scalers(crtc->base.dev, crtc, crtc->config);
4447         skl_detach_scalers(crtc);
4448         if (!enable)
4449                 return;
4450
4451         if (crtc->config->pch_pfit.enabled) {
4452                 int id;
4453
4454                 if (WARN_ON(crtc->config->scaler_state.scaler_id < 0)) {
4455                         DRM_ERROR("Requesting pfit without getting a scaler first\n");
4456                         return;
4457                 }
4458
4459                 id = scaler_state->scaler_id;
4460                 I915_WRITE(SKL_PS_CTRL(pipe, id), PS_SCALER_EN |
4461                         PS_FILTER_MEDIUM | scaler_state->scalers[id].mode);
4462                 I915_WRITE(SKL_PS_WIN_POS(pipe, id), crtc->config->pch_pfit.pos);
4463                 I915_WRITE(SKL_PS_WIN_SZ(pipe, id), crtc->config->pch_pfit.size);
4464
4465                 DRM_DEBUG_KMS("for crtc_state = %p scaler_id = %d\n", crtc->config, id);
4466         }
4467 }
4468
4469 static void ironlake_pfit_enable(struct intel_crtc *crtc)
4470 {
4471         struct drm_device *dev = crtc->base.dev;
4472         struct drm_i915_private *dev_priv = dev->dev_private;
4473         int pipe = crtc->pipe;
4474
4475         if (crtc->config->pch_pfit.enabled) {
4476                 /* Force use of hard-coded filter coefficients
4477                  * as some pre-programmed values are broken,
4478                  * e.g. x201.
4479                  */
4480                 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
4481                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3 |
4482                                                  PF_PIPE_SEL_IVB(pipe));
4483                 else
4484                         I915_WRITE(PF_CTL(pipe), PF_ENABLE | PF_FILTER_MED_3x3);
4485                 I915_WRITE(PF_WIN_POS(pipe), crtc->config->pch_pfit.pos);
4486                 I915_WRITE(PF_WIN_SZ(pipe), crtc->config->pch_pfit.size);
4487         }
4488 }
4489
4490 void hsw_enable_ips(struct intel_crtc *crtc)
4491 {
4492         struct drm_device *dev = crtc->base.dev;
4493         struct drm_i915_private *dev_priv = dev->dev_private;
4494
4495         if (!crtc->config->ips_enabled)
4496                 return;
4497
4498         /* We can only enable IPS after we enable a plane and wait for a vblank */
4499         intel_wait_for_vblank(dev, crtc->pipe);
4500
4501         assert_plane_enabled(dev_priv, crtc->plane);
4502         if (IS_BROADWELL(dev)) {
4503                 mutex_lock(&dev_priv->rps.hw_lock);
4504                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0xc0000000));
4505                 mutex_unlock(&dev_priv->rps.hw_lock);
4506                 /* Quoting Art Runyan: "its not safe to expect any particular
4507                  * value in IPS_CTL bit 31 after enabling IPS through the
4508                  * mailbox." Moreover, the mailbox may return a bogus state,
4509                  * so we need to just enable it and continue on.
4510                  */
4511         } else {
4512                 I915_WRITE(IPS_CTL, IPS_ENABLE);
4513                 /* The bit only becomes 1 in the next vblank, so this wait here
4514                  * is essentially intel_wait_for_vblank. If we don't have this
4515                  * and don't wait for vblanks until the end of crtc_enable, then
4516                  * the HW state readout code will complain that the expected
4517                  * IPS_CTL value is not the one we read. */
4518                 if (wait_for(I915_READ_NOTRACE(IPS_CTL) & IPS_ENABLE, 50))
4519                         DRM_ERROR("Timed out waiting for IPS enable\n");
4520         }
4521 }
4522
4523 void hsw_disable_ips(struct intel_crtc *crtc)
4524 {
4525         struct drm_device *dev = crtc->base.dev;
4526         struct drm_i915_private *dev_priv = dev->dev_private;
4527
4528         if (!crtc->config->ips_enabled)
4529                 return;
4530
4531         assert_plane_enabled(dev_priv, crtc->plane);
4532         if (IS_BROADWELL(dev)) {
4533                 mutex_lock(&dev_priv->rps.hw_lock);
4534                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
4535                 mutex_unlock(&dev_priv->rps.hw_lock);
4536                 /* wait for pcode to finish disabling IPS, which may take up to 42ms */
4537                 if (wait_for((I915_READ(IPS_CTL) & IPS_ENABLE) == 0, 42))
4538                         DRM_ERROR("Timed out waiting for IPS disable\n");
4539         } else {
4540                 I915_WRITE(IPS_CTL, 0);
4541                 POSTING_READ(IPS_CTL);
4542         }
4543
4544         /* We need to wait for a vblank before we can disable the plane. */
4545         intel_wait_for_vblank(dev, crtc->pipe);
4546 }
4547
4548 /** Loads the palette/gamma unit for the CRTC with the prepared values */
4549 static void intel_crtc_load_lut(struct drm_crtc *crtc)
4550 {
4551         struct drm_device *dev = crtc->dev;
4552         struct drm_i915_private *dev_priv = dev->dev_private;
4553         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4554         enum pipe pipe = intel_crtc->pipe;
4555         int palreg = PALETTE(pipe);
4556         int i;
4557         bool reenable_ips = false;
4558
4559         /* The clocks have to be on to load the palette. */
4560         if (!crtc->state->active)
4561                 return;
4562
4563         if (HAS_GMCH_DISPLAY(dev_priv->dev)) {
4564                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI))
4565                         assert_dsi_pll_enabled(dev_priv);
4566                 else
4567                         assert_pll_enabled(dev_priv, pipe);
4568         }
4569
4570         /* use legacy palette for Ironlake */
4571         if (!HAS_GMCH_DISPLAY(dev))
4572                 palreg = LGC_PALETTE(pipe);
4573
4574         /* Workaround : Do not read or write the pipe palette/gamma data while
4575          * GAMMA_MODE is configured for split gamma and IPS_CTL has IPS enabled.
4576          */
4577         if (IS_HASWELL(dev) && intel_crtc->config->ips_enabled &&
4578             ((I915_READ(GAMMA_MODE(pipe)) & GAMMA_MODE_MODE_MASK) ==
4579              GAMMA_MODE_MODE_SPLIT)) {
4580                 hsw_disable_ips(intel_crtc);
4581                 reenable_ips = true;
4582         }
4583
4584         for (i = 0; i < 256; i++) {
4585                 I915_WRITE(palreg + 4 * i,
4586                            (intel_crtc->lut_r[i] << 16) |
4587                            (intel_crtc->lut_g[i] << 8) |
4588                            intel_crtc->lut_b[i]);
4589         }
4590
4591         if (reenable_ips)
4592                 hsw_enable_ips(intel_crtc);
4593 }
4594
4595 static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
4596 {
4597         if (intel_crtc->overlay) {
4598                 struct drm_device *dev = intel_crtc->base.dev;
4599                 struct drm_i915_private *dev_priv = dev->dev_private;
4600
4601                 mutex_lock(&dev->struct_mutex);
4602                 dev_priv->mm.interruptible = false;
4603                 (void) intel_overlay_switch_off(intel_crtc->overlay);
4604                 dev_priv->mm.interruptible = true;
4605                 mutex_unlock(&dev->struct_mutex);
4606         }
4607
4608         /* Let userspace switch the overlay on again. In most cases userspace
4609          * has to recompute where to put it anyway.
4610          */
4611 }
4612
4613 /**
4614  * intel_post_enable_primary - Perform operations after enabling primary plane
4615  * @crtc: the CRTC whose primary plane was just enabled
4616  *
4617  * Performs potentially sleeping operations that must be done after the primary
4618  * plane is enabled, such as updating FBC and IPS.  Note that this may be
4619  * called due to an explicit primary plane update, or due to an implicit
4620  * re-enable that is caused when a sprite plane is updated to no longer
4621  * completely hide the primary plane.
4622  */
4623 static void
4624 intel_post_enable_primary(struct drm_crtc *crtc)
4625 {
4626         struct drm_device *dev = crtc->dev;
4627         struct drm_i915_private *dev_priv = dev->dev_private;
4628         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4629         int pipe = intel_crtc->pipe;
4630
4631         /*
4632          * BDW signals flip done immediately if the plane
4633          * is disabled, even if the plane enable is already
4634          * armed to occur at the next vblank :(
4635          */
4636         if (IS_BROADWELL(dev))
4637                 intel_wait_for_vblank(dev, pipe);
4638
4639         /*
4640          * FIXME IPS should be fine as long as one plane is
4641          * enabled, but in practice it seems to have problems
4642          * when going from primary only to sprite only and vice
4643          * versa.
4644          */
4645         hsw_enable_ips(intel_crtc);
4646
4647         /*
4648          * Gen2 reports pipe underruns whenever all planes are disabled.
4649          * So don't enable underrun reporting before at least some planes
4650          * are enabled.
4651          * FIXME: Need to fix the logic to work when we turn off all planes
4652          * but leave the pipe running.
4653          */
4654         if (IS_GEN2(dev))
4655                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4656
4657         /* Underruns don't raise interrupts, so check manually. */
4658         if (HAS_GMCH_DISPLAY(dev))
4659                 i9xx_check_fifo_underruns(dev_priv);
4660 }
4661
4662 /**
4663  * intel_pre_disable_primary - Perform operations before disabling primary plane
4664  * @crtc: the CRTC whose primary plane is to be disabled
4665  *
4666  * Performs potentially sleeping operations that must be done before the
4667  * primary plane is disabled, such as updating FBC and IPS.  Note that this may
4668  * be called due to an explicit primary plane update, or due to an implicit
4669  * disable that is caused when a sprite plane completely hides the primary
4670  * plane.
4671  */
4672 static void
4673 intel_pre_disable_primary(struct drm_crtc *crtc)
4674 {
4675         struct drm_device *dev = crtc->dev;
4676         struct drm_i915_private *dev_priv = dev->dev_private;
4677         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4678         int pipe = intel_crtc->pipe;
4679
4680         /*
4681          * Gen2 reports pipe underruns whenever all planes are disabled.
4682          * So diasble underrun reporting before all the planes get disabled.
4683          * FIXME: Need to fix the logic to work when we turn off all planes
4684          * but leave the pipe running.
4685          */
4686         if (IS_GEN2(dev))
4687                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
4688
4689         /*
4690          * Vblank time updates from the shadow to live plane control register
4691          * are blocked if the memory self-refresh mode is active at that
4692          * moment. So to make sure the plane gets truly disabled, disable
4693          * first the self-refresh mode. The self-refresh enable bit in turn
4694          * will be checked/applied by the HW only at the next frame start
4695          * event which is after the vblank start event, so we need to have a
4696          * wait-for-vblank between disabling the plane and the pipe.
4697          */
4698         if (HAS_GMCH_DISPLAY(dev))
4699                 intel_set_memory_cxsr(dev_priv, false);
4700
4701         /*
4702          * FIXME IPS should be fine as long as one plane is
4703          * enabled, but in practice it seems to have problems
4704          * when going from primary only to sprite only and vice
4705          * versa.
4706          */
4707         hsw_disable_ips(intel_crtc);
4708 }
4709
4710 static void intel_post_plane_update(struct intel_crtc *crtc)
4711 {
4712         struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
4713         struct drm_device *dev = crtc->base.dev;
4714         struct drm_plane *plane;
4715
4716         if (atomic->wait_vblank)
4717                 intel_wait_for_vblank(dev, crtc->pipe);
4718
4719         intel_frontbuffer_flip(dev, atomic->fb_bits);
4720
4721         if (atomic->update_fbc) {
4722                 mutex_lock(&dev->struct_mutex);
4723                 intel_fbc_update(dev);
4724                 mutex_unlock(&dev->struct_mutex);
4725         }
4726
4727         if (atomic->post_enable_primary)
4728                 intel_post_enable_primary(&crtc->base);
4729
4730         drm_for_each_plane_mask(plane, dev, atomic->update_sprite_watermarks)
4731                 intel_update_sprite_watermarks(plane, &crtc->base,
4732                                                0, 0, 0, false, false);
4733
4734         memset(atomic, 0, sizeof(*atomic));
4735 }
4736
4737 static void intel_pre_plane_update(struct intel_crtc *crtc)
4738 {
4739         struct drm_device *dev = crtc->base.dev;
4740         struct drm_i915_private *dev_priv = dev->dev_private;
4741         struct intel_crtc_atomic_commit *atomic = &crtc->atomic;
4742         struct drm_plane *p;
4743
4744         /* Track fb's for any planes being disabled */
4745
4746         drm_for_each_plane_mask(p, dev, atomic->disabled_planes) {
4747                 struct intel_plane *plane = to_intel_plane(p);
4748                 unsigned fb_bits = 0;
4749
4750                 switch (p->type) {
4751                 case DRM_PLANE_TYPE_PRIMARY:
4752                         fb_bits = INTEL_FRONTBUFFER_PRIMARY(plane->pipe);
4753                         break;
4754                 case DRM_PLANE_TYPE_CURSOR:
4755                         fb_bits = INTEL_FRONTBUFFER_CURSOR(plane->pipe);
4756                         break;
4757                 case DRM_PLANE_TYPE_OVERLAY:
4758                         fb_bits = INTEL_FRONTBUFFER_SPRITE(plane->pipe);
4759                         break;
4760                 }
4761
4762                 mutex_lock(&dev->struct_mutex);
4763                 i915_gem_track_fb(intel_fb_obj(plane->base.fb), NULL, fb_bits);
4764                 mutex_unlock(&dev->struct_mutex);
4765         }
4766
4767         if (atomic->wait_for_flips)
4768                 intel_crtc_wait_for_pending_flips(&crtc->base);
4769
4770         if (atomic->disable_fbc &&
4771             dev_priv->fbc.crtc == crtc) {
4772                 mutex_lock(&dev->struct_mutex);
4773                 if (dev_priv->fbc.crtc == crtc)
4774                         intel_fbc_disable(dev);
4775                 mutex_unlock(&dev->struct_mutex);
4776         }
4777
4778         if (atomic->pre_disable_primary)
4779                 intel_pre_disable_primary(&crtc->base);
4780 }
4781
4782 static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask)
4783 {
4784         struct drm_device *dev = crtc->dev;
4785         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4786         struct drm_plane *p;
4787         int pipe = intel_crtc->pipe;
4788
4789         intel_crtc_dpms_overlay_disable(intel_crtc);
4790
4791         drm_for_each_plane_mask(p, dev, plane_mask)
4792                 to_intel_plane(p)->disable_plane(p, crtc);
4793
4794         /*
4795          * FIXME: Once we grow proper nuclear flip support out of this we need
4796          * to compute the mask of flip planes precisely. For the time being
4797          * consider this a flip to a NULL plane.
4798          */
4799         intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe));
4800 }
4801
4802 static void ironlake_crtc_enable(struct drm_crtc *crtc)
4803 {
4804         struct drm_device *dev = crtc->dev;
4805         struct drm_i915_private *dev_priv = dev->dev_private;
4806         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4807         struct intel_encoder *encoder;
4808         int pipe = intel_crtc->pipe;
4809
4810         if (WARN_ON(intel_crtc->active))
4811                 return;
4812
4813         if (intel_crtc->config->has_pch_encoder)
4814                 intel_prepare_shared_dpll(intel_crtc);
4815
4816         if (intel_crtc->config->has_dp_encoder)
4817                 intel_dp_set_m_n(intel_crtc, M1_N1);
4818
4819         intel_set_pipe_timings(intel_crtc);
4820
4821         if (intel_crtc->config->has_pch_encoder) {
4822                 intel_cpu_transcoder_set_m_n(intel_crtc,
4823                                      &intel_crtc->config->fdi_m_n, NULL);
4824         }
4825
4826         ironlake_set_pipeconf(crtc);
4827
4828         intel_crtc->active = true;
4829
4830         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4831         intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, true);
4832
4833         for_each_encoder_on_crtc(dev, crtc, encoder)
4834                 if (encoder->pre_enable)
4835                         encoder->pre_enable(encoder);
4836
4837         if (intel_crtc->config->has_pch_encoder) {
4838                 /* Note: FDI PLL enabling _must_ be done before we enable the
4839                  * cpu pipes, hence this is separate from all the other fdi/pch
4840                  * enabling. */
4841                 ironlake_fdi_pll_enable(intel_crtc);
4842         } else {
4843                 assert_fdi_tx_disabled(dev_priv, pipe);
4844                 assert_fdi_rx_disabled(dev_priv, pipe);
4845         }
4846
4847         ironlake_pfit_enable(intel_crtc);
4848
4849         /*
4850          * On ILK+ LUT must be loaded before the pipe is running but with
4851          * clocks enabled
4852          */
4853         intel_crtc_load_lut(crtc);
4854
4855         intel_update_watermarks(crtc);
4856         intel_enable_pipe(intel_crtc);
4857
4858         if (intel_crtc->config->has_pch_encoder)
4859                 ironlake_pch_enable(crtc);
4860
4861         assert_vblank_disabled(crtc);
4862         drm_crtc_vblank_on(crtc);
4863
4864         for_each_encoder_on_crtc(dev, crtc, encoder)
4865                 encoder->enable(encoder);
4866
4867         if (HAS_PCH_CPT(dev))
4868                 cpt_verify_modeset(dev, intel_crtc->pipe);
4869 }
4870
4871 /* IPS only exists on ULT machines and is tied to pipe A. */
4872 static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
4873 {
4874         return HAS_IPS(crtc->base.dev) && crtc->pipe == PIPE_A;
4875 }
4876
4877 static void haswell_crtc_enable(struct drm_crtc *crtc)
4878 {
4879         struct drm_device *dev = crtc->dev;
4880         struct drm_i915_private *dev_priv = dev->dev_private;
4881         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4882         struct intel_encoder *encoder;
4883         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
4884         struct intel_crtc_state *pipe_config =
4885                 to_intel_crtc_state(crtc->state);
4886
4887         if (WARN_ON(intel_crtc->active))
4888                 return;
4889
4890         if (intel_crtc_to_shared_dpll(intel_crtc))
4891                 intel_enable_shared_dpll(intel_crtc);
4892
4893         if (intel_crtc->config->has_dp_encoder)
4894                 intel_dp_set_m_n(intel_crtc, M1_N1);
4895
4896         intel_set_pipe_timings(intel_crtc);
4897
4898         if (intel_crtc->config->cpu_transcoder != TRANSCODER_EDP) {
4899                 I915_WRITE(PIPE_MULT(intel_crtc->config->cpu_transcoder),
4900                            intel_crtc->config->pixel_multiplier - 1);
4901         }
4902
4903         if (intel_crtc->config->has_pch_encoder) {
4904                 intel_cpu_transcoder_set_m_n(intel_crtc,
4905                                      &intel_crtc->config->fdi_m_n, NULL);
4906         }
4907
4908         haswell_set_pipeconf(crtc);
4909
4910         intel_set_pipe_csc(crtc);
4911
4912         intel_crtc->active = true;
4913
4914         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
4915         for_each_encoder_on_crtc(dev, crtc, encoder)
4916                 if (encoder->pre_enable)
4917                         encoder->pre_enable(encoder);
4918
4919         if (intel_crtc->config->has_pch_encoder) {
4920                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
4921                                                       true);
4922                 dev_priv->display.fdi_link_train(crtc);
4923         }
4924
4925         intel_ddi_enable_pipe_clock(intel_crtc);
4926
4927         if (INTEL_INFO(dev)->gen == 9)
4928                 skylake_pfit_update(intel_crtc, 1);
4929         else if (INTEL_INFO(dev)->gen < 9)
4930                 ironlake_pfit_enable(intel_crtc);
4931         else
4932                 MISSING_CASE(INTEL_INFO(dev)->gen);
4933
4934         /*
4935          * On ILK+ LUT must be loaded before the pipe is running but with
4936          * clocks enabled
4937          */
4938         intel_crtc_load_lut(crtc);
4939
4940         intel_ddi_set_pipe_settings(crtc);
4941         intel_ddi_enable_transcoder_func(crtc);
4942
4943         intel_update_watermarks(crtc);
4944         intel_enable_pipe(intel_crtc);
4945
4946         if (intel_crtc->config->has_pch_encoder)
4947                 lpt_pch_enable(crtc);
4948
4949         if (intel_crtc->config->dp_encoder_is_mst)
4950                 intel_ddi_set_vc_payload_alloc(crtc, true);
4951
4952         assert_vblank_disabled(crtc);
4953         drm_crtc_vblank_on(crtc);
4954
4955         for_each_encoder_on_crtc(dev, crtc, encoder) {
4956                 encoder->enable(encoder);
4957                 intel_opregion_notify_encoder(encoder, true);
4958         }
4959
4960         /* If we change the relative order between pipe/planes enabling, we need
4961          * to change the workaround. */
4962         hsw_workaround_pipe = pipe_config->hsw_workaround_pipe;
4963         if (IS_HASWELL(dev) && hsw_workaround_pipe != INVALID_PIPE) {
4964                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
4965                 intel_wait_for_vblank(dev, hsw_workaround_pipe);
4966         }
4967 }
4968
4969 static void ironlake_pfit_disable(struct intel_crtc *crtc)
4970 {
4971         struct drm_device *dev = crtc->base.dev;
4972         struct drm_i915_private *dev_priv = dev->dev_private;
4973         int pipe = crtc->pipe;
4974
4975         /* To avoid upsetting the power well on haswell only disable the pfit if
4976          * it's in use. The hw state code will make sure we get this right. */
4977         if (crtc->config->pch_pfit.enabled) {
4978                 I915_WRITE(PF_CTL(pipe), 0);
4979                 I915_WRITE(PF_WIN_POS(pipe), 0);
4980                 I915_WRITE(PF_WIN_SZ(pipe), 0);
4981         }
4982 }
4983
4984 static void ironlake_crtc_disable(struct drm_crtc *crtc)
4985 {
4986         struct drm_device *dev = crtc->dev;
4987         struct drm_i915_private *dev_priv = dev->dev_private;
4988         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
4989         struct intel_encoder *encoder;
4990         int pipe = intel_crtc->pipe;
4991         u32 reg, temp;
4992
4993         for_each_encoder_on_crtc(dev, crtc, encoder)
4994                 encoder->disable(encoder);
4995
4996         drm_crtc_vblank_off(crtc);
4997         assert_vblank_disabled(crtc);
4998
4999         if (intel_crtc->config->has_pch_encoder)
5000                 intel_set_pch_fifo_underrun_reporting(dev_priv, pipe, false);
5001
5002         intel_disable_pipe(intel_crtc);
5003
5004         ironlake_pfit_disable(intel_crtc);
5005
5006         if (intel_crtc->config->has_pch_encoder)
5007                 ironlake_fdi_disable(crtc);
5008
5009         for_each_encoder_on_crtc(dev, crtc, encoder)
5010                 if (encoder->post_disable)
5011                         encoder->post_disable(encoder);
5012
5013         if (intel_crtc->config->has_pch_encoder) {
5014                 ironlake_disable_pch_transcoder(dev_priv, pipe);
5015
5016                 if (HAS_PCH_CPT(dev)) {
5017                         /* disable TRANS_DP_CTL */
5018                         reg = TRANS_DP_CTL(pipe);
5019                         temp = I915_READ(reg);
5020                         temp &= ~(TRANS_DP_OUTPUT_ENABLE |
5021                                   TRANS_DP_PORT_SEL_MASK);
5022                         temp |= TRANS_DP_PORT_SEL_NONE;
5023                         I915_WRITE(reg, temp);
5024
5025                         /* disable DPLL_SEL */
5026                         temp = I915_READ(PCH_DPLL_SEL);
5027                         temp &= ~(TRANS_DPLL_ENABLE(pipe) | TRANS_DPLLB_SEL(pipe));
5028                         I915_WRITE(PCH_DPLL_SEL, temp);
5029                 }
5030
5031                 ironlake_fdi_pll_disable(intel_crtc);
5032         }
5033 }
5034
5035 static void haswell_crtc_disable(struct drm_crtc *crtc)
5036 {
5037         struct drm_device *dev = crtc->dev;
5038         struct drm_i915_private *dev_priv = dev->dev_private;
5039         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5040         struct intel_encoder *encoder;
5041         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
5042
5043         for_each_encoder_on_crtc(dev, crtc, encoder) {
5044                 intel_opregion_notify_encoder(encoder, false);
5045                 encoder->disable(encoder);
5046         }
5047
5048         drm_crtc_vblank_off(crtc);
5049         assert_vblank_disabled(crtc);
5050
5051         if (intel_crtc->config->has_pch_encoder)
5052                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
5053                                                       false);
5054         intel_disable_pipe(intel_crtc);
5055
5056         if (intel_crtc->config->dp_encoder_is_mst)
5057                 intel_ddi_set_vc_payload_alloc(crtc, false);
5058
5059         intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
5060
5061         if (INTEL_INFO(dev)->gen == 9)
5062                 skylake_pfit_update(intel_crtc, 0);
5063         else if (INTEL_INFO(dev)->gen < 9)
5064                 ironlake_pfit_disable(intel_crtc);
5065         else
5066                 MISSING_CASE(INTEL_INFO(dev)->gen);
5067
5068         intel_ddi_disable_pipe_clock(intel_crtc);
5069
5070         if (intel_crtc->config->has_pch_encoder) {
5071                 lpt_disable_pch_transcoder(dev_priv);
5072                 intel_ddi_fdi_disable(crtc);
5073         }
5074
5075         for_each_encoder_on_crtc(dev, crtc, encoder)
5076                 if (encoder->post_disable)
5077                         encoder->post_disable(encoder);
5078 }
5079
5080 static void i9xx_pfit_enable(struct intel_crtc *crtc)
5081 {
5082         struct drm_device *dev = crtc->base.dev;
5083         struct drm_i915_private *dev_priv = dev->dev_private;
5084         struct intel_crtc_state *pipe_config = crtc->config;
5085
5086         if (!pipe_config->gmch_pfit.control)
5087                 return;
5088
5089         /*
5090          * The panel fitter should only be adjusted whilst the pipe is disabled,
5091          * according to register description and PRM.
5092          */
5093         WARN_ON(I915_READ(PFIT_CONTROL) & PFIT_ENABLE);
5094         assert_pipe_disabled(dev_priv, crtc->pipe);
5095
5096         I915_WRITE(PFIT_PGM_RATIOS, pipe_config->gmch_pfit.pgm_ratios);
5097         I915_WRITE(PFIT_CONTROL, pipe_config->gmch_pfit.control);
5098
5099         /* Border color in case we don't scale up to the full screen. Black by
5100          * default, change to something else for debugging. */
5101         I915_WRITE(BCLRPAT(crtc->pipe), 0);
5102 }
5103
5104 static enum intel_display_power_domain port_to_power_domain(enum port port)
5105 {
5106         switch (port) {
5107         case PORT_A:
5108                 return POWER_DOMAIN_PORT_DDI_A_4_LANES;
5109         case PORT_B:
5110                 return POWER_DOMAIN_PORT_DDI_B_4_LANES;
5111         case PORT_C:
5112                 return POWER_DOMAIN_PORT_DDI_C_4_LANES;
5113         case PORT_D:
5114                 return POWER_DOMAIN_PORT_DDI_D_4_LANES;
5115         default:
5116                 WARN_ON_ONCE(1);
5117                 return POWER_DOMAIN_PORT_OTHER;
5118         }
5119 }
5120
5121 #define for_each_power_domain(domain, mask)                             \
5122         for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)     \
5123                 if ((1 << (domain)) & (mask))
5124
5125 enum intel_display_power_domain
5126 intel_display_port_power_domain(struct intel_encoder *intel_encoder)
5127 {
5128         struct drm_device *dev = intel_encoder->base.dev;
5129         struct intel_digital_port *intel_dig_port;
5130
5131         switch (intel_encoder->type) {
5132         case INTEL_OUTPUT_UNKNOWN:
5133                 /* Only DDI platforms should ever use this output type */
5134                 WARN_ON_ONCE(!HAS_DDI(dev));
5135         case INTEL_OUTPUT_DISPLAYPORT:
5136         case INTEL_OUTPUT_HDMI:
5137         case INTEL_OUTPUT_EDP:
5138                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
5139                 return port_to_power_domain(intel_dig_port->port);
5140         case INTEL_OUTPUT_DP_MST:
5141                 intel_dig_port = enc_to_mst(&intel_encoder->base)->primary;
5142                 return port_to_power_domain(intel_dig_port->port);
5143         case INTEL_OUTPUT_ANALOG:
5144                 return POWER_DOMAIN_PORT_CRT;
5145         case INTEL_OUTPUT_DSI:
5146                 return POWER_DOMAIN_PORT_DSI;
5147         default:
5148                 return POWER_DOMAIN_PORT_OTHER;
5149         }
5150 }
5151
5152 static unsigned long get_crtc_power_domains(struct drm_crtc *crtc)
5153 {
5154         struct drm_device *dev = crtc->dev;
5155         struct intel_encoder *intel_encoder;
5156         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5157         enum pipe pipe = intel_crtc->pipe;
5158         unsigned long mask;
5159         enum transcoder transcoder;
5160
5161         transcoder = intel_pipe_to_cpu_transcoder(dev->dev_private, pipe);
5162
5163         mask = BIT(POWER_DOMAIN_PIPE(pipe));
5164         mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder));
5165         if (intel_crtc->config->pch_pfit.enabled ||
5166             intel_crtc->config->pch_pfit.force_thru)
5167                 mask |= BIT(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
5168
5169         for_each_encoder_on_crtc(dev, crtc, intel_encoder)
5170                 mask |= BIT(intel_display_port_power_domain(intel_encoder));
5171
5172         return mask;
5173 }
5174
5175 static void modeset_update_crtc_power_domains(struct drm_atomic_state *state)
5176 {
5177         struct drm_device *dev = state->dev;
5178         struct drm_i915_private *dev_priv = dev->dev_private;
5179         unsigned long pipe_domains[I915_MAX_PIPES] = { 0, };
5180         struct intel_crtc *crtc;
5181
5182         /*
5183          * First get all needed power domains, then put all unneeded, to avoid
5184          * any unnecessary toggling of the power wells.
5185          */
5186         for_each_intel_crtc(dev, crtc) {
5187                 enum intel_display_power_domain domain;
5188
5189                 if (!crtc->base.state->enable)
5190                         continue;
5191
5192                 pipe_domains[crtc->pipe] = get_crtc_power_domains(&crtc->base);
5193
5194                 for_each_power_domain(domain, pipe_domains[crtc->pipe])
5195                         intel_display_power_get(dev_priv, domain);
5196         }
5197
5198         if (dev_priv->display.modeset_commit_cdclk) {
5199                 unsigned int cdclk = to_intel_atomic_state(state)->cdclk;
5200
5201                 if (cdclk != dev_priv->cdclk_freq &&
5202                     !WARN_ON(!state->allow_modeset))
5203                         dev_priv->display.modeset_commit_cdclk(state);
5204         }
5205
5206         for_each_intel_crtc(dev, crtc) {
5207                 enum intel_display_power_domain domain;
5208
5209                 for_each_power_domain(domain, crtc->enabled_power_domains)
5210                         intel_display_power_put(dev_priv, domain);
5211
5212                 crtc->enabled_power_domains = pipe_domains[crtc->pipe];
5213         }
5214
5215         intel_display_set_init_power(dev_priv, false);
5216 }
5217
5218 static void intel_update_max_cdclk(struct drm_device *dev)
5219 {
5220         struct drm_i915_private *dev_priv = dev->dev_private;
5221
5222         if (IS_SKYLAKE(dev)) {
5223                 u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
5224
5225                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
5226                         dev_priv->max_cdclk_freq = 675000;
5227                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
5228                         dev_priv->max_cdclk_freq = 540000;
5229                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
5230                         dev_priv->max_cdclk_freq = 450000;
5231                 else
5232                         dev_priv->max_cdclk_freq = 337500;
5233         } else if (IS_BROADWELL(dev))  {
5234                 /*
5235                  * FIXME with extra cooling we can allow
5236                  * 540 MHz for ULX and 675 Mhz for ULT.
5237                  * How can we know if extra cooling is
5238                  * available? PCI ID, VTB, something else?
5239                  */
5240                 if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
5241                         dev_priv->max_cdclk_freq = 450000;
5242                 else if (IS_BDW_ULX(dev))
5243                         dev_priv->max_cdclk_freq = 450000;
5244                 else if (IS_BDW_ULT(dev))
5245                         dev_priv->max_cdclk_freq = 540000;
5246                 else
5247                         dev_priv->max_cdclk_freq = 675000;
5248         } else if (IS_CHERRYVIEW(dev)) {
5249                 dev_priv->max_cdclk_freq = 320000;
5250         } else if (IS_VALLEYVIEW(dev)) {
5251                 dev_priv->max_cdclk_freq = 400000;
5252         } else {
5253                 /* otherwise assume cdclk is fixed */
5254                 dev_priv->max_cdclk_freq = dev_priv->cdclk_freq;
5255         }
5256
5257         DRM_DEBUG_DRIVER("Max CD clock rate: %d kHz\n",
5258                          dev_priv->max_cdclk_freq);
5259 }
5260
5261 static void intel_update_cdclk(struct drm_device *dev)
5262 {
5263         struct drm_i915_private *dev_priv = dev->dev_private;
5264
5265         dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev);
5266         DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n",
5267                          dev_priv->cdclk_freq);
5268
5269         /*
5270          * Program the gmbus_freq based on the cdclk frequency.
5271          * BSpec erroneously claims we should aim for 4MHz, but
5272          * in fact 1MHz is the correct frequency.
5273          */
5274         if (IS_VALLEYVIEW(dev)) {
5275                 /*
5276                  * Program the gmbus_freq based on the cdclk frequency.
5277                  * BSpec erroneously claims we should aim for 4MHz, but
5278                  * in fact 1MHz is the correct frequency.
5279                  */
5280                 I915_WRITE(GMBUSFREQ_VLV, DIV_ROUND_UP(dev_priv->cdclk_freq, 1000));
5281         }
5282
5283         if (dev_priv->max_cdclk_freq == 0)
5284                 intel_update_max_cdclk(dev);
5285 }
5286
5287 static void broxton_set_cdclk(struct drm_device *dev, int frequency)
5288 {
5289         struct drm_i915_private *dev_priv = dev->dev_private;
5290         uint32_t divider;
5291         uint32_t ratio;
5292         uint32_t current_freq;
5293         int ret;
5294
5295         /* frequency = 19.2MHz * ratio / 2 / div{1,1.5,2,4} */
5296         switch (frequency) {
5297         case 144000:
5298                 divider = BXT_CDCLK_CD2X_DIV_SEL_4;
5299                 ratio = BXT_DE_PLL_RATIO(60);
5300                 break;
5301         case 288000:
5302                 divider = BXT_CDCLK_CD2X_DIV_SEL_2;
5303                 ratio = BXT_DE_PLL_RATIO(60);
5304                 break;
5305         case 384000:
5306                 divider = BXT_CDCLK_CD2X_DIV_SEL_1_5;
5307                 ratio = BXT_DE_PLL_RATIO(60);
5308                 break;
5309         case 576000:
5310                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
5311                 ratio = BXT_DE_PLL_RATIO(60);
5312                 break;
5313         case 624000:
5314                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
5315                 ratio = BXT_DE_PLL_RATIO(65);
5316                 break;
5317         case 19200:
5318                 /*
5319                  * Bypass frequency with DE PLL disabled. Init ratio, divider
5320                  * to suppress GCC warning.
5321                  */
5322                 ratio = 0;
5323                 divider = 0;
5324                 break;
5325         default:
5326                 DRM_ERROR("unsupported CDCLK freq %d", frequency);
5327
5328                 return;
5329         }
5330
5331         mutex_lock(&dev_priv->rps.hw_lock);
5332         /* Inform power controller of upcoming frequency change */
5333         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
5334                                       0x80000000);
5335         mutex_unlock(&dev_priv->rps.hw_lock);
5336
5337         if (ret) {
5338                 DRM_ERROR("PCode CDCLK freq change notify failed (err %d, freq %d)\n",
5339                           ret, frequency);
5340                 return;
5341         }
5342
5343         current_freq = I915_READ(CDCLK_CTL) & CDCLK_FREQ_DECIMAL_MASK;
5344         /* convert from .1 fixpoint MHz with -1MHz offset to kHz */
5345         current_freq = current_freq * 500 + 1000;
5346
5347         /*
5348          * DE PLL has to be disabled when
5349          * - setting to 19.2MHz (bypass, PLL isn't used)
5350          * - before setting to 624MHz (PLL needs toggling)
5351          * - before setting to any frequency from 624MHz (PLL needs toggling)
5352          */
5353         if (frequency == 19200 || frequency == 624000 ||
5354             current_freq == 624000) {
5355                 I915_WRITE(BXT_DE_PLL_ENABLE, ~BXT_DE_PLL_PLL_ENABLE);
5356                 /* Timeout 200us */
5357                 if (wait_for(!(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK),
5358                              1))
5359                         DRM_ERROR("timout waiting for DE PLL unlock\n");
5360         }
5361
5362         if (frequency != 19200) {
5363                 uint32_t val;
5364
5365                 val = I915_READ(BXT_DE_PLL_CTL);
5366                 val &= ~BXT_DE_PLL_RATIO_MASK;
5367                 val |= ratio;
5368                 I915_WRITE(BXT_DE_PLL_CTL, val);
5369
5370                 I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
5371                 /* Timeout 200us */
5372                 if (wait_for(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK, 1))
5373                         DRM_ERROR("timeout waiting for DE PLL lock\n");
5374
5375                 val = I915_READ(CDCLK_CTL);
5376                 val &= ~BXT_CDCLK_CD2X_DIV_SEL_MASK;
5377                 val |= divider;
5378                 /*
5379                  * Disable SSA Precharge when CD clock frequency < 500 MHz,
5380                  * enable otherwise.
5381                  */
5382                 val &= ~BXT_CDCLK_SSA_PRECHARGE_ENABLE;
5383                 if (frequency >= 500000)
5384                         val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
5385
5386                 val &= ~CDCLK_FREQ_DECIMAL_MASK;
5387                 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
5388                 val |= (frequency - 1000) / 500;
5389                 I915_WRITE(CDCLK_CTL, val);
5390         }
5391
5392         mutex_lock(&dev_priv->rps.hw_lock);
5393         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
5394                                       DIV_ROUND_UP(frequency, 25000));
5395         mutex_unlock(&dev_priv->rps.hw_lock);
5396
5397         if (ret) {
5398                 DRM_ERROR("PCode CDCLK freq set failed, (err %d, freq %d)\n",
5399                           ret, frequency);
5400                 return;
5401         }
5402
5403         intel_update_cdclk(dev);
5404 }
5405
5406 void broxton_init_cdclk(struct drm_device *dev)
5407 {
5408         struct drm_i915_private *dev_priv = dev->dev_private;
5409         uint32_t val;
5410
5411         /*
5412          * NDE_RSTWRN_OPT RST PCH Handshake En must always be 0b on BXT
5413          * or else the reset will hang because there is no PCH to respond.
5414          * Move the handshake programming to initialization sequence.
5415          * Previously was left up to BIOS.
5416          */
5417         val = I915_READ(HSW_NDE_RSTWRN_OPT);
5418         val &= ~RESET_PCH_HANDSHAKE_ENABLE;
5419         I915_WRITE(HSW_NDE_RSTWRN_OPT, val);
5420
5421         /* Enable PG1 for cdclk */
5422         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
5423
5424         /* check if cd clock is enabled */
5425         if (I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_PLL_ENABLE) {
5426                 DRM_DEBUG_KMS("Display already initialized\n");
5427                 return;
5428         }
5429
5430         /*
5431          * FIXME:
5432          * - The initial CDCLK needs to be read from VBT.
5433          *   Need to make this change after VBT has changes for BXT.
5434          * - check if setting the max (or any) cdclk freq is really necessary
5435          *   here, it belongs to modeset time
5436          */
5437         broxton_set_cdclk(dev, 624000);
5438
5439         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
5440         POSTING_READ(DBUF_CTL);
5441
5442         udelay(10);
5443
5444         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
5445                 DRM_ERROR("DBuf power enable timeout!\n");
5446 }
5447
5448 void broxton_uninit_cdclk(struct drm_device *dev)
5449 {
5450         struct drm_i915_private *dev_priv = dev->dev_private;
5451
5452         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
5453         POSTING_READ(DBUF_CTL);
5454
5455         udelay(10);
5456
5457         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
5458                 DRM_ERROR("DBuf power disable timeout!\n");
5459
5460         /* Set minimum (bypass) frequency, in effect turning off the DE PLL */
5461         broxton_set_cdclk(dev, 19200);
5462
5463         intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
5464 }
5465
5466 static const struct skl_cdclk_entry {
5467         unsigned int freq;
5468         unsigned int vco;
5469 } skl_cdclk_frequencies[] = {
5470         { .freq = 308570, .vco = 8640 },
5471         { .freq = 337500, .vco = 8100 },
5472         { .freq = 432000, .vco = 8640 },
5473         { .freq = 450000, .vco = 8100 },
5474         { .freq = 540000, .vco = 8100 },
5475         { .freq = 617140, .vco = 8640 },
5476         { .freq = 675000, .vco = 8100 },
5477 };
5478
5479 static unsigned int skl_cdclk_decimal(unsigned int freq)
5480 {
5481         return (freq - 1000) / 500;
5482 }
5483
5484 static unsigned int skl_cdclk_get_vco(unsigned int freq)
5485 {
5486         unsigned int i;
5487
5488         for (i = 0; i < ARRAY_SIZE(skl_cdclk_frequencies); i++) {
5489                 const struct skl_cdclk_entry *e = &skl_cdclk_frequencies[i];
5490
5491                 if (e->freq == freq)
5492                         return e->vco;
5493         }
5494
5495         return 8100;
5496 }
5497
5498 static void
5499 skl_dpll0_enable(struct drm_i915_private *dev_priv, unsigned int required_vco)
5500 {
5501         unsigned int min_freq;
5502         u32 val;
5503
5504         /* select the minimum CDCLK before enabling DPLL 0 */
5505         val = I915_READ(CDCLK_CTL);
5506         val &= ~CDCLK_FREQ_SEL_MASK | ~CDCLK_FREQ_DECIMAL_MASK;
5507         val |= CDCLK_FREQ_337_308;
5508
5509         if (required_vco == 8640)
5510                 min_freq = 308570;
5511         else
5512                 min_freq = 337500;
5513
5514         val = CDCLK_FREQ_337_308 | skl_cdclk_decimal(min_freq);
5515
5516         I915_WRITE(CDCLK_CTL, val);
5517         POSTING_READ(CDCLK_CTL);
5518
5519         /*
5520          * We always enable DPLL0 with the lowest link rate possible, but still
5521          * taking into account the VCO required to operate the eDP panel at the
5522          * desired frequency. The usual DP link rates operate with a VCO of
5523          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
5524          * The modeset code is responsible for the selection of the exact link
5525          * rate later on, with the constraint of choosing a frequency that
5526          * works with required_vco.
5527          */
5528         val = I915_READ(DPLL_CTRL1);
5529
5530         val &= ~(DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | DPLL_CTRL1_SSC(SKL_DPLL0) |
5531                  DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
5532         val |= DPLL_CTRL1_OVERRIDE(SKL_DPLL0);
5533         if (required_vco == 8640)
5534                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080,
5535                                             SKL_DPLL0);
5536         else
5537                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810,
5538                                             SKL_DPLL0);
5539
5540         I915_WRITE(DPLL_CTRL1, val);
5541         POSTING_READ(DPLL_CTRL1);
5542
5543         I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) | LCPLL_PLL_ENABLE);
5544
5545         if (wait_for(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK, 5))
5546                 DRM_ERROR("DPLL0 not locked\n");
5547 }
5548
5549 static bool skl_cdclk_pcu_ready(struct drm_i915_private *dev_priv)
5550 {
5551         int ret;
5552         u32 val;
5553
5554         /* inform PCU we want to change CDCLK */
5555         val = SKL_CDCLK_PREPARE_FOR_CHANGE;
5556         mutex_lock(&dev_priv->rps.hw_lock);
5557         ret = sandybridge_pcode_read(dev_priv, SKL_PCODE_CDCLK_CONTROL, &val);
5558         mutex_unlock(&dev_priv->rps.hw_lock);
5559
5560         return ret == 0 && (val & SKL_CDCLK_READY_FOR_CHANGE);
5561 }
5562
5563 static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv)
5564 {
5565         unsigned int i;
5566
5567         for (i = 0; i < 15; i++) {
5568                 if (skl_cdclk_pcu_ready(dev_priv))
5569                         return true;
5570                 udelay(10);
5571         }
5572
5573         return false;
5574 }
5575
5576 static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq)
5577 {
5578         struct drm_device *dev = dev_priv->dev;
5579         u32 freq_select, pcu_ack;
5580
5581         DRM_DEBUG_DRIVER("Changing CDCLK to %dKHz\n", freq);
5582
5583         if (!skl_cdclk_wait_for_pcu_ready(dev_priv)) {
5584                 DRM_ERROR("failed to inform PCU about cdclk change\n");
5585                 return;
5586         }
5587
5588         /* set CDCLK_CTL */
5589         switch(freq) {
5590         case 450000:
5591         case 432000:
5592                 freq_select = CDCLK_FREQ_450_432;
5593                 pcu_ack = 1;
5594                 break;
5595         case 540000:
5596                 freq_select = CDCLK_FREQ_540;
5597                 pcu_ack = 2;
5598                 break;
5599         case 308570:
5600         case 337500:
5601         default:
5602                 freq_select = CDCLK_FREQ_337_308;
5603                 pcu_ack = 0;
5604                 break;
5605         case 617140:
5606         case 675000:
5607                 freq_select = CDCLK_FREQ_675_617;
5608                 pcu_ack = 3;
5609                 break;
5610         }
5611
5612         I915_WRITE(CDCLK_CTL, freq_select | skl_cdclk_decimal(freq));
5613         POSTING_READ(CDCLK_CTL);
5614
5615         /* inform PCU of the change */
5616         mutex_lock(&dev_priv->rps.hw_lock);
5617         sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
5618         mutex_unlock(&dev_priv->rps.hw_lock);
5619
5620         intel_update_cdclk(dev);
5621 }
5622
5623 void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
5624 {
5625         /* disable DBUF power */
5626         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
5627         POSTING_READ(DBUF_CTL);
5628
5629         udelay(10);
5630
5631         if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
5632                 DRM_ERROR("DBuf power disable timeout\n");
5633
5634         /* disable DPLL0 */
5635         I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE);
5636         if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1))
5637                 DRM_ERROR("Couldn't disable DPLL0\n");
5638
5639         intel_display_power_put(dev_priv, POWER_DOMAIN_PLLS);
5640 }
5641
5642 void skl_init_cdclk(struct drm_i915_private *dev_priv)
5643 {
5644         u32 val;
5645         unsigned int required_vco;
5646
5647         /* enable PCH reset handshake */
5648         val = I915_READ(HSW_NDE_RSTWRN_OPT);
5649         I915_WRITE(HSW_NDE_RSTWRN_OPT, val | RESET_PCH_HANDSHAKE_ENABLE);
5650
5651         /* enable PG1 and Misc I/O */
5652         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
5653
5654         /* DPLL0 already enabed !? */
5655         if (I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE) {
5656                 DRM_DEBUG_DRIVER("DPLL0 already running\n");
5657                 return;
5658         }
5659
5660         /* enable DPLL0 */
5661         required_vco = skl_cdclk_get_vco(dev_priv->skl_boot_cdclk);
5662         skl_dpll0_enable(dev_priv, required_vco);
5663
5664         /* set CDCLK to the frequency the BIOS chose */
5665         skl_set_cdclk(dev_priv, dev_priv->skl_boot_cdclk);
5666
5667         /* enable DBUF power */
5668         I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
5669         POSTING_READ(DBUF_CTL);
5670
5671         udelay(10);
5672
5673         if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
5674                 DRM_ERROR("DBuf power enable timeout\n");
5675 }
5676
5677 /* returns HPLL frequency in kHz */
5678 static int valleyview_get_vco(struct drm_i915_private *dev_priv)
5679 {
5680         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
5681
5682         /* Obtain SKU information */
5683         mutex_lock(&dev_priv->sb_lock);
5684         hpll_freq = vlv_cck_read(dev_priv, CCK_FUSE_REG) &
5685                 CCK_FUSE_HPLL_FREQ_MASK;
5686         mutex_unlock(&dev_priv->sb_lock);
5687
5688         return vco_freq[hpll_freq] * 1000;
5689 }
5690
5691 /* Adjust CDclk dividers to allow high res or save power if possible */
5692 static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
5693 {
5694         struct drm_i915_private *dev_priv = dev->dev_private;
5695         u32 val, cmd;
5696
5697         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
5698                                         != dev_priv->cdclk_freq);
5699
5700         if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
5701                 cmd = 2;
5702         else if (cdclk == 266667)
5703                 cmd = 1;
5704         else
5705                 cmd = 0;
5706
5707         mutex_lock(&dev_priv->rps.hw_lock);
5708         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5709         val &= ~DSPFREQGUAR_MASK;
5710         val |= (cmd << DSPFREQGUAR_SHIFT);
5711         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
5712         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
5713                       DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
5714                      50)) {
5715                 DRM_ERROR("timed out waiting for CDclk change\n");
5716         }
5717         mutex_unlock(&dev_priv->rps.hw_lock);
5718
5719         mutex_lock(&dev_priv->sb_lock);
5720
5721         if (cdclk == 400000) {
5722                 u32 divider;
5723
5724                 divider = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
5725
5726                 /* adjust cdclk divider */
5727                 val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
5728                 val &= ~DISPLAY_FREQUENCY_VALUES;
5729                 val |= divider;
5730                 vlv_cck_write(dev_priv, CCK_DISPLAY_CLOCK_CONTROL, val);
5731
5732                 if (wait_for((vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL) &
5733                               DISPLAY_FREQUENCY_STATUS) == (divider << DISPLAY_FREQUENCY_STATUS_SHIFT),
5734                              50))
5735                         DRM_ERROR("timed out waiting for CDclk change\n");
5736         }
5737
5738         /* adjust self-refresh exit latency value */
5739         val = vlv_bunit_read(dev_priv, BUNIT_REG_BISOC);
5740         val &= ~0x7f;
5741
5742         /*
5743          * For high bandwidth configs, we set a higher latency in the bunit
5744          * so that the core display fetch happens in time to avoid underruns.
5745          */
5746         if (cdclk == 400000)
5747                 val |= 4500 / 250; /* 4.5 usec */
5748         else
5749                 val |= 3000 / 250; /* 3.0 usec */
5750         vlv_bunit_write(dev_priv, BUNIT_REG_BISOC, val);
5751
5752         mutex_unlock(&dev_priv->sb_lock);
5753
5754         intel_update_cdclk(dev);
5755 }
5756
5757 static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
5758 {
5759         struct drm_i915_private *dev_priv = dev->dev_private;
5760         u32 val, cmd;
5761
5762         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
5763                                                 != dev_priv->cdclk_freq);
5764
5765         switch (cdclk) {
5766         case 333333:
5767         case 320000:
5768         case 266667:
5769         case 200000:
5770                 break;
5771         default:
5772                 MISSING_CASE(cdclk);
5773                 return;
5774         }
5775
5776         /*
5777          * Specs are full of misinformation, but testing on actual
5778          * hardware has shown that we just need to write the desired
5779          * CCK divider into the Punit register.
5780          */
5781         cmd = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
5782
5783         mutex_lock(&dev_priv->rps.hw_lock);
5784         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
5785         val &= ~DSPFREQGUAR_MASK_CHV;
5786         val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
5787         vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
5788         if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
5789                       DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
5790                      50)) {
5791                 DRM_ERROR("timed out waiting for CDclk change\n");
5792         }
5793         mutex_unlock(&dev_priv->rps.hw_lock);
5794
5795         intel_update_cdclk(dev);
5796 }
5797
5798 static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv,
5799                                  int max_pixclk)
5800 {
5801         int freq_320 = (dev_priv->hpll_freq <<  1) % 320000 != 0 ? 333333 : 320000;
5802         int limit = IS_CHERRYVIEW(dev_priv) ? 95 : 90;
5803
5804         /*
5805          * Really only a few cases to deal with, as only 4 CDclks are supported:
5806          *   200MHz
5807          *   267MHz
5808          *   320/333MHz (depends on HPLL freq)
5809          *   400MHz (VLV only)
5810          * So we check to see whether we're above 90% (VLV) or 95% (CHV)
5811          * of the lower bin and adjust if needed.
5812          *
5813          * We seem to get an unstable or solid color picture at 200MHz.
5814          * Not sure what's wrong. For now use 200MHz only when all pipes
5815          * are off.
5816          */
5817         if (!IS_CHERRYVIEW(dev_priv) &&
5818             max_pixclk > freq_320*limit/100)
5819                 return 400000;
5820         else if (max_pixclk > 266667*limit/100)
5821                 return freq_320;
5822         else if (max_pixclk > 0)
5823                 return 266667;
5824         else
5825                 return 200000;
5826 }
5827
5828 static int broxton_calc_cdclk(struct drm_i915_private *dev_priv,
5829                               int max_pixclk)
5830 {
5831         /*
5832          * FIXME:
5833          * - remove the guardband, it's not needed on BXT
5834          * - set 19.2MHz bypass frequency if there are no active pipes
5835          */
5836         if (max_pixclk > 576000*9/10)
5837                 return 624000;
5838         else if (max_pixclk > 384000*9/10)
5839                 return 576000;
5840         else if (max_pixclk > 288000*9/10)
5841                 return 384000;
5842         else if (max_pixclk > 144000*9/10)
5843                 return 288000;
5844         else
5845                 return 144000;
5846 }
5847
5848 /* Compute the max pixel clock for new configuration. Uses atomic state if
5849  * that's non-NULL, look at current state otherwise. */
5850 static int intel_mode_max_pixclk(struct drm_device *dev,
5851                                  struct drm_atomic_state *state)
5852 {
5853         struct intel_crtc *intel_crtc;
5854         struct intel_crtc_state *crtc_state;
5855         int max_pixclk = 0;
5856
5857         for_each_intel_crtc(dev, intel_crtc) {
5858                 crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
5859                 if (IS_ERR(crtc_state))
5860                         return PTR_ERR(crtc_state);
5861
5862                 if (!crtc_state->base.enable)
5863                         continue;
5864
5865                 max_pixclk = max(max_pixclk,
5866                                  crtc_state->base.adjusted_mode.crtc_clock);
5867         }
5868
5869         return max_pixclk;
5870 }
5871
5872 static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state)
5873 {
5874         struct drm_device *dev = state->dev;
5875         struct drm_i915_private *dev_priv = dev->dev_private;
5876         int max_pixclk = intel_mode_max_pixclk(dev, state);
5877
5878         if (max_pixclk < 0)
5879                 return max_pixclk;
5880
5881         to_intel_atomic_state(state)->cdclk =
5882                 valleyview_calc_cdclk(dev_priv, max_pixclk);
5883
5884         return 0;
5885 }
5886
5887 static int broxton_modeset_calc_cdclk(struct drm_atomic_state *state)
5888 {
5889         struct drm_device *dev = state->dev;
5890         struct drm_i915_private *dev_priv = dev->dev_private;
5891         int max_pixclk = intel_mode_max_pixclk(dev, state);
5892
5893         if (max_pixclk < 0)
5894                 return max_pixclk;
5895
5896         to_intel_atomic_state(state)->cdclk =
5897                 broxton_calc_cdclk(dev_priv, max_pixclk);
5898
5899         return 0;
5900 }
5901
5902 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
5903 {
5904         unsigned int credits, default_credits;
5905
5906         if (IS_CHERRYVIEW(dev_priv))
5907                 default_credits = PFI_CREDIT(12);
5908         else
5909                 default_credits = PFI_CREDIT(8);
5910
5911         if (DIV_ROUND_CLOSEST(dev_priv->cdclk_freq, 1000) >= dev_priv->rps.cz_freq) {
5912                 /* CHV suggested value is 31 or 63 */
5913                 if (IS_CHERRYVIEW(dev_priv))
5914                         credits = PFI_CREDIT_63;
5915                 else
5916                         credits = PFI_CREDIT(15);
5917         } else {
5918                 credits = default_credits;
5919         }
5920
5921         /*
5922          * WA - write default credits before re-programming
5923          * FIXME: should we also set the resend bit here?
5924          */
5925         I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE |
5926                    default_credits);
5927
5928         I915_WRITE(GCI_CONTROL, VGA_FAST_MODE_DISABLE |
5929                    credits | PFI_CREDIT_RESEND);
5930
5931         /*
5932          * FIXME is this guaranteed to clear
5933          * immediately or should we poll for it?
5934          */
5935         WARN_ON(I915_READ(GCI_CONTROL) & PFI_CREDIT_RESEND);
5936 }
5937
5938 static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state)
5939 {
5940         struct drm_device *dev = old_state->dev;
5941         unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
5942         struct drm_i915_private *dev_priv = dev->dev_private;
5943
5944         /*
5945          * FIXME: We can end up here with all power domains off, yet
5946          * with a CDCLK frequency other than the minimum. To account
5947          * for this take the PIPE-A power domain, which covers the HW
5948          * blocks needed for the following programming. This can be
5949          * removed once it's guaranteed that we get here either with
5950          * the minimum CDCLK set, or the required power domains
5951          * enabled.
5952          */
5953         intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
5954
5955         if (IS_CHERRYVIEW(dev))
5956                 cherryview_set_cdclk(dev, req_cdclk);
5957         else
5958                 valleyview_set_cdclk(dev, req_cdclk);
5959
5960         vlv_program_pfi_credits(dev_priv);
5961
5962         intel_display_power_put(dev_priv, POWER_DOMAIN_PIPE_A);
5963 }
5964
5965 static void valleyview_crtc_enable(struct drm_crtc *crtc)
5966 {
5967         struct drm_device *dev = crtc->dev;
5968         struct drm_i915_private *dev_priv = to_i915(dev);
5969         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
5970         struct intel_encoder *encoder;
5971         int pipe = intel_crtc->pipe;
5972         bool is_dsi;
5973
5974         if (WARN_ON(intel_crtc->active))
5975                 return;
5976
5977         is_dsi = intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI);
5978
5979         if (!is_dsi) {
5980                 if (IS_CHERRYVIEW(dev))
5981                         chv_prepare_pll(intel_crtc, intel_crtc->config);
5982                 else
5983                         vlv_prepare_pll(intel_crtc, intel_crtc->config);
5984         }
5985
5986         if (intel_crtc->config->has_dp_encoder)
5987                 intel_dp_set_m_n(intel_crtc, M1_N1);
5988
5989         intel_set_pipe_timings(intel_crtc);
5990
5991         if (IS_CHERRYVIEW(dev) && pipe == PIPE_B) {
5992                 struct drm_i915_private *dev_priv = dev->dev_private;
5993
5994                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
5995                 I915_WRITE(CHV_CANVAS(pipe), 0);
5996         }
5997
5998         i9xx_set_pipeconf(intel_crtc);
5999
6000         intel_crtc->active = true;
6001
6002         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6003
6004         for_each_encoder_on_crtc(dev, crtc, encoder)
6005                 if (encoder->pre_pll_enable)
6006                         encoder->pre_pll_enable(encoder);
6007
6008         if (!is_dsi) {
6009                 if (IS_CHERRYVIEW(dev))
6010                         chv_enable_pll(intel_crtc, intel_crtc->config);
6011                 else
6012                         vlv_enable_pll(intel_crtc, intel_crtc->config);
6013         }
6014
6015         for_each_encoder_on_crtc(dev, crtc, encoder)
6016                 if (encoder->pre_enable)
6017                         encoder->pre_enable(encoder);
6018
6019         i9xx_pfit_enable(intel_crtc);
6020
6021         intel_crtc_load_lut(crtc);
6022
6023         intel_update_watermarks(crtc);
6024         intel_enable_pipe(intel_crtc);
6025
6026         assert_vblank_disabled(crtc);
6027         drm_crtc_vblank_on(crtc);
6028
6029         for_each_encoder_on_crtc(dev, crtc, encoder)
6030                 encoder->enable(encoder);
6031 }
6032
6033 static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
6034 {
6035         struct drm_device *dev = crtc->base.dev;
6036         struct drm_i915_private *dev_priv = dev->dev_private;
6037
6038         I915_WRITE(FP0(crtc->pipe), crtc->config->dpll_hw_state.fp0);
6039         I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1);
6040 }
6041
6042 static void i9xx_crtc_enable(struct drm_crtc *crtc)
6043 {
6044         struct drm_device *dev = crtc->dev;
6045         struct drm_i915_private *dev_priv = to_i915(dev);
6046         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6047         struct intel_encoder *encoder;
6048         int pipe = intel_crtc->pipe;
6049
6050         if (WARN_ON(intel_crtc->active))
6051                 return;
6052
6053         i9xx_set_pll_dividers(intel_crtc);
6054
6055         if (intel_crtc->config->has_dp_encoder)
6056                 intel_dp_set_m_n(intel_crtc, M1_N1);
6057
6058         intel_set_pipe_timings(intel_crtc);
6059
6060         i9xx_set_pipeconf(intel_crtc);
6061
6062         intel_crtc->active = true;
6063
6064         if (!IS_GEN2(dev))
6065                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
6066
6067         for_each_encoder_on_crtc(dev, crtc, encoder)
6068                 if (encoder->pre_enable)
6069                         encoder->pre_enable(encoder);
6070
6071         i9xx_enable_pll(intel_crtc);
6072
6073         i9xx_pfit_enable(intel_crtc);
6074
6075         intel_crtc_load_lut(crtc);
6076
6077         intel_update_watermarks(crtc);
6078         intel_enable_pipe(intel_crtc);
6079
6080         assert_vblank_disabled(crtc);
6081         drm_crtc_vblank_on(crtc);
6082
6083         for_each_encoder_on_crtc(dev, crtc, encoder)
6084                 encoder->enable(encoder);
6085 }
6086
6087 static void i9xx_pfit_disable(struct intel_crtc *crtc)
6088 {
6089         struct drm_device *dev = crtc->base.dev;
6090         struct drm_i915_private *dev_priv = dev->dev_private;
6091
6092         if (!crtc->config->gmch_pfit.control)
6093                 return;
6094
6095         assert_pipe_disabled(dev_priv, crtc->pipe);
6096
6097         DRM_DEBUG_DRIVER("disabling pfit, current: 0x%08x\n",
6098                          I915_READ(PFIT_CONTROL));
6099         I915_WRITE(PFIT_CONTROL, 0);
6100 }
6101
6102 static void i9xx_crtc_disable(struct drm_crtc *crtc)
6103 {
6104         struct drm_device *dev = crtc->dev;
6105         struct drm_i915_private *dev_priv = dev->dev_private;
6106         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6107         struct intel_encoder *encoder;
6108         int pipe = intel_crtc->pipe;
6109
6110         /*
6111          * On gen2 planes are double buffered but the pipe isn't, so we must
6112          * wait for planes to fully turn off before disabling the pipe.
6113          * We also need to wait on all gmch platforms because of the
6114          * self-refresh mode constraint explained above.
6115          */
6116         intel_wait_for_vblank(dev, pipe);
6117
6118         for_each_encoder_on_crtc(dev, crtc, encoder)
6119                 encoder->disable(encoder);
6120
6121         drm_crtc_vblank_off(crtc);
6122         assert_vblank_disabled(crtc);
6123
6124         intel_disable_pipe(intel_crtc);
6125
6126         i9xx_pfit_disable(intel_crtc);
6127
6128         for_each_encoder_on_crtc(dev, crtc, encoder)
6129                 if (encoder->post_disable)
6130                         encoder->post_disable(encoder);
6131
6132         if (!intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DSI)) {
6133                 if (IS_CHERRYVIEW(dev))
6134                         chv_disable_pll(dev_priv, pipe);
6135                 else if (IS_VALLEYVIEW(dev))
6136                         vlv_disable_pll(dev_priv, pipe);
6137                 else
6138                         i9xx_disable_pll(intel_crtc);
6139         }
6140
6141         if (!IS_GEN2(dev))
6142                 intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
6143 }
6144
6145 static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
6146 {
6147         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6148         struct drm_i915_private *dev_priv = to_i915(crtc->dev);
6149         enum intel_display_power_domain domain;
6150         unsigned long domains;
6151
6152         if (!intel_crtc->active)
6153                 return;
6154
6155         if (to_intel_plane_state(crtc->primary->state)->visible) {
6156                 intel_crtc_wait_for_pending_flips(crtc);
6157                 intel_pre_disable_primary(crtc);
6158         }
6159
6160         intel_crtc_disable_planes(crtc, crtc->state->plane_mask);
6161         dev_priv->display.crtc_disable(crtc);
6162
6163         domains = intel_crtc->enabled_power_domains;
6164         for_each_power_domain(domain, domains)
6165                 intel_display_power_put(dev_priv, domain);
6166         intel_crtc->enabled_power_domains = 0;
6167 }
6168
6169 /*
6170  * turn all crtc's off, but do not adjust state
6171  * This has to be paired with a call to intel_modeset_setup_hw_state.
6172  */
6173 void intel_display_suspend(struct drm_device *dev)
6174 {
6175         struct drm_crtc *crtc;
6176
6177         for_each_crtc(dev, crtc)
6178                 intel_crtc_disable_noatomic(crtc);
6179 }
6180
6181 /* Master function to enable/disable CRTC and corresponding power wells */
6182 int intel_crtc_control(struct drm_crtc *crtc, bool enable)
6183 {
6184         struct drm_device *dev = crtc->dev;
6185         struct drm_mode_config *config = &dev->mode_config;
6186         struct drm_modeset_acquire_ctx *ctx = config->acquire_ctx;
6187         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
6188         struct intel_crtc_state *pipe_config;
6189         struct drm_atomic_state *state;
6190         int ret;
6191
6192         if (enable == intel_crtc->active)
6193                 return 0;
6194
6195         if (enable && !crtc->state->enable)
6196                 return 0;
6197
6198         /* this function should be called with drm_modeset_lock_all for now */
6199         if (WARN_ON(!ctx))
6200                 return -EIO;
6201         lockdep_assert_held(&ctx->ww_ctx);
6202
6203         state = drm_atomic_state_alloc(dev);
6204         if (WARN_ON(!state))
6205                 return -ENOMEM;
6206
6207         state->acquire_ctx = ctx;
6208         state->allow_modeset = true;
6209
6210         pipe_config = intel_atomic_get_crtc_state(state, intel_crtc);
6211         if (IS_ERR(pipe_config)) {
6212                 ret = PTR_ERR(pipe_config);
6213                 goto err;
6214         }
6215         pipe_config->base.active = enable;
6216
6217         ret = intel_set_mode(state);
6218         if (!ret)
6219                 return ret;
6220
6221 err:
6222         DRM_ERROR("Updating crtc active failed with %i\n", ret);
6223         drm_atomic_state_free(state);
6224         return ret;
6225 }
6226
6227 /**
6228  * Sets the power management mode of the pipe and plane.
6229  */
6230 void intel_crtc_update_dpms(struct drm_crtc *crtc)
6231 {
6232         struct drm_device *dev = crtc->dev;
6233         struct intel_encoder *intel_encoder;
6234         bool enable = false;
6235
6236         for_each_encoder_on_crtc(dev, crtc, intel_encoder)
6237                 enable |= intel_encoder->connectors_active;
6238
6239         intel_crtc_control(crtc, enable);
6240 }
6241
6242 void intel_encoder_destroy(struct drm_encoder *encoder)
6243 {
6244         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
6245
6246         drm_encoder_cleanup(encoder);
6247         kfree(intel_encoder);
6248 }
6249
6250 /* Simple dpms helper for encoders with just one connector, no cloning and only
6251  * one kind of off state. It clamps all !ON modes to fully OFF and changes the
6252  * state of the entire output pipe. */
6253 static void intel_encoder_dpms(struct intel_encoder *encoder, int mode)
6254 {
6255         if (mode == DRM_MODE_DPMS_ON) {
6256                 encoder->connectors_active = true;
6257
6258                 intel_crtc_update_dpms(encoder->base.crtc);
6259         } else {
6260                 encoder->connectors_active = false;
6261
6262                 intel_crtc_update_dpms(encoder->base.crtc);
6263         }
6264 }
6265
6266 /* Cross check the actual hw state with our own modeset state tracking (and it's
6267  * internal consistency). */
6268 static void intel_connector_check_state(struct intel_connector *connector)
6269 {
6270         if (connector->get_hw_state(connector)) {
6271                 struct intel_encoder *encoder = connector->encoder;
6272                 struct drm_crtc *crtc;
6273                 bool encoder_enabled;
6274                 enum pipe pipe;
6275
6276                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
6277                               connector->base.base.id,
6278                               connector->base.name);
6279
6280                 /* there is no real hw state for MST connectors */
6281                 if (connector->mst_port)
6282                         return;
6283
6284                 I915_STATE_WARN(connector->base.dpms == DRM_MODE_DPMS_OFF,
6285                      "wrong connector dpms state\n");
6286                 I915_STATE_WARN(connector->base.encoder != &encoder->base,
6287                      "active connector not linked to encoder\n");
6288
6289                 if (encoder) {
6290                         I915_STATE_WARN(!encoder->connectors_active,
6291                              "encoder->connectors_active not set\n");
6292
6293                         encoder_enabled = encoder->get_hw_state(encoder, &pipe);
6294                         I915_STATE_WARN(!encoder_enabled, "encoder not enabled\n");
6295                         if (I915_STATE_WARN_ON(!encoder->base.crtc))
6296                                 return;
6297
6298                         crtc = encoder->base.crtc;
6299
6300                         I915_STATE_WARN(!crtc->state->enable,
6301                                         "crtc not enabled\n");
6302                         I915_STATE_WARN(!to_intel_crtc(crtc)->active, "crtc not active\n");
6303                         I915_STATE_WARN(pipe != to_intel_crtc(crtc)->pipe,
6304                              "encoder active on the wrong pipe\n");
6305                 }
6306         }
6307 }
6308
6309 int intel_connector_init(struct intel_connector *connector)
6310 {
6311         struct drm_connector_state *connector_state;
6312
6313         connector_state = kzalloc(sizeof *connector_state, GFP_KERNEL);
6314         if (!connector_state)
6315                 return -ENOMEM;
6316
6317         connector->base.state = connector_state;
6318         return 0;
6319 }
6320
6321 struct intel_connector *intel_connector_alloc(void)
6322 {
6323         struct intel_connector *connector;
6324
6325         connector = kzalloc(sizeof *connector, GFP_KERNEL);
6326         if (!connector)
6327                 return NULL;
6328
6329         if (intel_connector_init(connector) < 0) {
6330                 kfree(connector);
6331                 return NULL;
6332         }
6333
6334         return connector;
6335 }
6336
6337 /* Even simpler default implementation, if there's really no special case to
6338  * consider. */
6339 void intel_connector_dpms(struct drm_connector *connector, int mode)
6340 {
6341         /* All the simple cases only support two dpms states. */
6342         if (mode != DRM_MODE_DPMS_ON)
6343                 mode = DRM_MODE_DPMS_OFF;
6344
6345         if (mode == connector->dpms)
6346                 return;
6347
6348         connector->dpms = mode;
6349
6350         /* Only need to change hw state when actually enabled */
6351         if (connector->encoder)
6352                 intel_encoder_dpms(to_intel_encoder(connector->encoder), mode);
6353
6354         intel_modeset_check_state(connector->dev);
6355 }
6356
6357 /* Simple connector->get_hw_state implementation for encoders that support only
6358  * one connector and no cloning and hence the encoder state determines the state
6359  * of the connector. */
6360 bool intel_connector_get_hw_state(struct intel_connector *connector)
6361 {
6362         enum pipe pipe = 0;
6363         struct intel_encoder *encoder = connector->encoder;
6364
6365         return encoder->get_hw_state(encoder, &pipe);
6366 }
6367
6368 static int pipe_required_fdi_lanes(struct intel_crtc_state *crtc_state)
6369 {
6370         if (crtc_state->base.enable && crtc_state->has_pch_encoder)
6371                 return crtc_state->fdi_lanes;
6372
6373         return 0;
6374 }
6375
6376 static int ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
6377                                      struct intel_crtc_state *pipe_config)
6378 {
6379         struct drm_atomic_state *state = pipe_config->base.state;
6380         struct intel_crtc *other_crtc;
6381         struct intel_crtc_state *other_crtc_state;
6382
6383         DRM_DEBUG_KMS("checking fdi config on pipe %c, lanes %i\n",
6384                       pipe_name(pipe), pipe_config->fdi_lanes);
6385         if (pipe_config->fdi_lanes > 4) {
6386                 DRM_DEBUG_KMS("invalid fdi lane config on pipe %c: %i lanes\n",
6387                               pipe_name(pipe), pipe_config->fdi_lanes);
6388                 return -EINVAL;
6389         }
6390
6391         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
6392                 if (pipe_config->fdi_lanes > 2) {
6393                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
6394                                       pipe_config->fdi_lanes);
6395                         return -EINVAL;
6396                 } else {
6397                         return 0;
6398                 }
6399         }
6400
6401         if (INTEL_INFO(dev)->num_pipes == 2)
6402                 return 0;
6403
6404         /* Ivybridge 3 pipe is really complicated */
6405         switch (pipe) {
6406         case PIPE_A:
6407                 return 0;
6408         case PIPE_B:
6409                 if (pipe_config->fdi_lanes <= 2)
6410                         return 0;
6411
6412                 other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_C));
6413                 other_crtc_state =
6414                         intel_atomic_get_crtc_state(state, other_crtc);
6415                 if (IS_ERR(other_crtc_state))
6416                         return PTR_ERR(other_crtc_state);
6417
6418                 if (pipe_required_fdi_lanes(other_crtc_state) > 0) {
6419                         DRM_DEBUG_KMS("invalid shared fdi lane config on pipe %c: %i lanes\n",
6420                                       pipe_name(pipe), pipe_config->fdi_lanes);
6421                         return -EINVAL;
6422                 }
6423                 return 0;
6424         case PIPE_C:
6425                 if (pipe_config->fdi_lanes > 2) {
6426                         DRM_DEBUG_KMS("only 2 lanes on pipe %c: required %i lanes\n",
6427                                       pipe_name(pipe), pipe_config->fdi_lanes);
6428                         return -EINVAL;
6429                 }
6430
6431                 other_crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, PIPE_B));
6432                 other_crtc_state =
6433                         intel_atomic_get_crtc_state(state, other_crtc);
6434                 if (IS_ERR(other_crtc_state))
6435                         return PTR_ERR(other_crtc_state);
6436
6437                 if (pipe_required_fdi_lanes(other_crtc_state) > 2) {
6438                         DRM_DEBUG_KMS("fdi link B uses too many lanes to enable link C\n");
6439                         return -EINVAL;
6440                 }
6441                 return 0;
6442         default:
6443                 BUG();
6444         }
6445 }
6446
6447 #define RETRY 1
6448 static int ironlake_fdi_compute_config(struct intel_crtc *intel_crtc,
6449                                        struct intel_crtc_state *pipe_config)
6450 {
6451         struct drm_device *dev = intel_crtc->base.dev;
6452         struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6453         int lane, link_bw, fdi_dotclock, ret;
6454         bool needs_recompute = false;
6455
6456 retry:
6457         /* FDI is a binary signal running at ~2.7GHz, encoding
6458          * each output octet as 10 bits. The actual frequency
6459          * is stored as a divider into a 100MHz clock, and the
6460          * mode pixel clock is stored in units of 1KHz.
6461          * Hence the bw of each lane in terms of the mode signal
6462          * is:
6463          */
6464         link_bw = intel_fdi_link_freq(dev) * MHz(100)/KHz(1)/10;
6465
6466         fdi_dotclock = adjusted_mode->crtc_clock;
6467
6468         lane = ironlake_get_lanes_required(fdi_dotclock, link_bw,
6469                                            pipe_config->pipe_bpp);
6470
6471         pipe_config->fdi_lanes = lane;
6472
6473         intel_link_compute_m_n(pipe_config->pipe_bpp, lane, fdi_dotclock,
6474                                link_bw, &pipe_config->fdi_m_n);
6475
6476         ret = ironlake_check_fdi_lanes(intel_crtc->base.dev,
6477                                        intel_crtc->pipe, pipe_config);
6478         if (ret == -EINVAL && pipe_config->pipe_bpp > 6*3) {
6479                 pipe_config->pipe_bpp -= 2*3;
6480                 DRM_DEBUG_KMS("fdi link bw constraint, reducing pipe bpp to %i\n",
6481                               pipe_config->pipe_bpp);
6482                 needs_recompute = true;
6483                 pipe_config->bw_constrained = true;
6484
6485                 goto retry;
6486         }
6487
6488         if (needs_recompute)
6489                 return RETRY;
6490
6491         return ret;
6492 }
6493
6494 static bool pipe_config_supports_ips(struct drm_i915_private *dev_priv,
6495                                      struct intel_crtc_state *pipe_config)
6496 {
6497         if (pipe_config->pipe_bpp > 24)
6498                 return false;
6499
6500         /* HSW can handle pixel rate up to cdclk? */
6501         if (IS_HASWELL(dev_priv->dev))
6502                 return true;
6503
6504         /*
6505          * We compare against max which means we must take
6506          * the increased cdclk requirement into account when
6507          * calculating the new cdclk.
6508          *
6509          * Should measure whether using a lower cdclk w/o IPS
6510          */
6511         return ilk_pipe_pixel_rate(pipe_config) <=
6512                 dev_priv->max_cdclk_freq * 95 / 100;
6513 }
6514
6515 static void hsw_compute_ips_config(struct intel_crtc *crtc,
6516                                    struct intel_crtc_state *pipe_config)
6517 {
6518         struct drm_device *dev = crtc->base.dev;
6519         struct drm_i915_private *dev_priv = dev->dev_private;
6520
6521         pipe_config->ips_enabled = i915.enable_ips &&
6522                 hsw_crtc_supports_ips(crtc) &&
6523                 pipe_config_supports_ips(dev_priv, pipe_config);
6524 }
6525
6526 static int intel_crtc_compute_config(struct intel_crtc *crtc,
6527                                      struct intel_crtc_state *pipe_config)
6528 {
6529         struct drm_device *dev = crtc->base.dev;
6530         struct drm_i915_private *dev_priv = dev->dev_private;
6531         struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
6532
6533         /* FIXME should check pixel clock limits on all platforms */
6534         if (INTEL_INFO(dev)->gen < 4) {
6535                 int clock_limit = dev_priv->max_cdclk_freq;
6536
6537                 /*
6538                  * Enable pixel doubling when the dot clock
6539                  * is > 90% of the (display) core speed.
6540                  *
6541                  * GDG double wide on either pipe,
6542                  * otherwise pipe A only.
6543                  */
6544                 if ((crtc->pipe == PIPE_A || IS_I915G(dev)) &&
6545                     adjusted_mode->crtc_clock > clock_limit * 9 / 10) {
6546                         clock_limit *= 2;
6547                         pipe_config->double_wide = true;
6548                 }
6549
6550                 if (adjusted_mode->crtc_clock > clock_limit * 9 / 10)
6551                         return -EINVAL;
6552         }
6553
6554         /*
6555          * Pipe horizontal size must be even in:
6556          * - DVO ganged mode
6557          * - LVDS dual channel mode
6558          * - Double wide pipe
6559          */
6560         if ((intel_pipe_will_have_type(pipe_config, INTEL_OUTPUT_LVDS) &&
6561              intel_is_dual_link_lvds(dev)) || pipe_config->double_wide)
6562                 pipe_config->pipe_src_w &= ~1;
6563
6564         /* Cantiga+ cannot handle modes with a hsync front porch of 0.
6565          * WaPruneModeWithIncorrectHsyncOffset:ctg,elk,ilk,snb,ivb,vlv,hsw.
6566          */
6567         if ((INTEL_INFO(dev)->gen > 4 || IS_G4X(dev)) &&
6568                 adjusted_mode->hsync_start == adjusted_mode->hdisplay)
6569                 return -EINVAL;
6570
6571         if (HAS_IPS(dev))
6572                 hsw_compute_ips_config(crtc, pipe_config);
6573
6574         if (pipe_config->has_pch_encoder)
6575                 return ironlake_fdi_compute_config(crtc, pipe_config);
6576
6577         return 0;
6578 }
6579
6580 static int skylake_get_display_clock_speed(struct drm_device *dev)
6581 {
6582         struct drm_i915_private *dev_priv = to_i915(dev);
6583         uint32_t lcpll1 = I915_READ(LCPLL1_CTL);
6584         uint32_t cdctl = I915_READ(CDCLK_CTL);
6585         uint32_t linkrate;
6586
6587         if (!(lcpll1 & LCPLL_PLL_ENABLE))
6588                 return 24000; /* 24MHz is the cd freq with NSSC ref */
6589
6590         if ((cdctl & CDCLK_FREQ_SEL_MASK) == CDCLK_FREQ_540)
6591                 return 540000;
6592
6593         linkrate = (I915_READ(DPLL_CTRL1) &
6594                     DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) >> 1;
6595
6596         if (linkrate == DPLL_CTRL1_LINK_RATE_2160 ||
6597             linkrate == DPLL_CTRL1_LINK_RATE_1080) {
6598                 /* vco 8640 */
6599                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
6600                 case CDCLK_FREQ_450_432:
6601                         return 432000;
6602                 case CDCLK_FREQ_337_308:
6603                         return 308570;
6604                 case CDCLK_FREQ_675_617:
6605                         return 617140;
6606                 default:
6607                         WARN(1, "Unknown cd freq selection\n");
6608                 }
6609         } else {
6610                 /* vco 8100 */
6611                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
6612                 case CDCLK_FREQ_450_432:
6613                         return 450000;
6614                 case CDCLK_FREQ_337_308:
6615                         return 337500;
6616                 case CDCLK_FREQ_675_617:
6617                         return 675000;
6618                 default:
6619                         WARN(1, "Unknown cd freq selection\n");
6620                 }
6621         }
6622
6623         /* error case, do as if DPLL0 isn't enabled */
6624         return 24000;
6625 }
6626
6627 static int broadwell_get_display_clock_speed(struct drm_device *dev)
6628 {
6629         struct drm_i915_private *dev_priv = dev->dev_private;
6630         uint32_t lcpll = I915_READ(LCPLL_CTL);
6631         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
6632
6633         if (lcpll & LCPLL_CD_SOURCE_FCLK)
6634                 return 800000;
6635         else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
6636                 return 450000;
6637         else if (freq == LCPLL_CLK_FREQ_450)
6638                 return 450000;
6639         else if (freq == LCPLL_CLK_FREQ_54O_BDW)
6640                 return 540000;
6641         else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
6642                 return 337500;
6643         else
6644                 return 675000;
6645 }
6646
6647 static int haswell_get_display_clock_speed(struct drm_device *dev)
6648 {
6649         struct drm_i915_private *dev_priv = dev->dev_private;
6650         uint32_t lcpll = I915_READ(LCPLL_CTL);
6651         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
6652
6653         if (lcpll & LCPLL_CD_SOURCE_FCLK)
6654                 return 800000;
6655         else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
6656                 return 450000;
6657         else if (freq == LCPLL_CLK_FREQ_450)
6658                 return 450000;
6659         else if (IS_HSW_ULT(dev))
6660                 return 337500;
6661         else
6662                 return 540000;
6663 }
6664
6665 static int valleyview_get_display_clock_speed(struct drm_device *dev)
6666 {
6667         struct drm_i915_private *dev_priv = dev->dev_private;
6668         u32 val;
6669         int divider;
6670
6671         if (dev_priv->hpll_freq == 0)
6672                 dev_priv->hpll_freq = valleyview_get_vco(dev_priv);
6673
6674         mutex_lock(&dev_priv->sb_lock);
6675         val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
6676         mutex_unlock(&dev_priv->sb_lock);
6677
6678         divider = val & DISPLAY_FREQUENCY_VALUES;
6679
6680         WARN((val & DISPLAY_FREQUENCY_STATUS) !=
6681              (divider << DISPLAY_FREQUENCY_STATUS_SHIFT),
6682              "cdclk change in progress\n");
6683
6684         return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, divider + 1);
6685 }
6686
6687 static int ilk_get_display_clock_speed(struct drm_device *dev)
6688 {
6689         return 450000;
6690 }
6691
6692 static int i945_get_display_clock_speed(struct drm_device *dev)
6693 {
6694         return 400000;
6695 }
6696
6697 static int i915_get_display_clock_speed(struct drm_device *dev)
6698 {
6699         return 333333;
6700 }
6701
6702 static int i9xx_misc_get_display_clock_speed(struct drm_device *dev)
6703 {
6704         return 200000;
6705 }
6706
6707 static int pnv_get_display_clock_speed(struct drm_device *dev)
6708 {
6709         u16 gcfgc = 0;
6710
6711         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
6712
6713         switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
6714         case GC_DISPLAY_CLOCK_267_MHZ_PNV:
6715                 return 266667;
6716         case GC_DISPLAY_CLOCK_333_MHZ_PNV:
6717                 return 333333;
6718         case GC_DISPLAY_CLOCK_444_MHZ_PNV:
6719                 return 444444;
6720         case GC_DISPLAY_CLOCK_200_MHZ_PNV:
6721                 return 200000;
6722         default:
6723                 DRM_ERROR("Unknown pnv display core clock 0x%04x\n", gcfgc);
6724         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
6725                 return 133333;
6726         case GC_DISPLAY_CLOCK_167_MHZ_PNV:
6727                 return 166667;
6728         }
6729 }
6730
6731 static int i915gm_get_display_clock_speed(struct drm_device *dev)
6732 {
6733         u16 gcfgc = 0;
6734
6735         pci_read_config_word(dev->pdev, GCFGC, &gcfgc);
6736
6737         if (gcfgc & GC_LOW_FREQUENCY_ENABLE)
6738                 return 133333;
6739         else {
6740                 switch (gcfgc & GC_DISPLAY_CLOCK_MASK) {
6741                 case GC_DISPLAY_CLOCK_333_MHZ:
6742                         return 333333;
6743                 default:
6744                 case GC_DISPLAY_CLOCK_190_200_MHZ:
6745                         return 190000;
6746                 }
6747         }
6748 }
6749
6750 static int i865_get_display_clock_speed(struct drm_device *dev)
6751 {
6752         return 266667;
6753 }
6754
6755 static int i85x_get_display_clock_speed(struct drm_device *dev)
6756 {
6757         u16 hpllcc = 0;
6758
6759         /*
6760          * 852GM/852GMV only supports 133 MHz and the HPLLCC
6761          * encoding is different :(
6762          * FIXME is this the right way to detect 852GM/852GMV?
6763          */
6764         if (dev->pdev->revision == 0x1)
6765                 return 133333;
6766
6767         pci_bus_read_config_word(dev->pdev->bus,
6768                                  PCI_DEVFN(0, 3), HPLLCC, &hpllcc);
6769
6770         /* Assume that the hardware is in the high speed state.  This
6771          * should be the default.
6772          */
6773         switch (hpllcc & GC_CLOCK_CONTROL_MASK) {
6774         case GC_CLOCK_133_200:
6775         case GC_CLOCK_133_200_2:
6776         case GC_CLOCK_100_200:
6777                 return 200000;
6778         case GC_CLOCK_166_250:
6779                 return 250000;
6780         case GC_CLOCK_100_133:
6781                 return 133333;
6782         case GC_CLOCK_133_266:
6783         case GC_CLOCK_133_266_2:
6784         case GC_CLOCK_166_266:
6785                 return 266667;
6786         }
6787
6788         /* Shouldn't happen */
6789         return 0;
6790 }
6791
6792 static int i830_get_display_clock_speed(struct drm_device *dev)
6793 {
6794         return 133333;
6795 }
6796
6797 static unsigned int intel_hpll_vco(struct drm_device *dev)
6798 {
6799         struct drm_i915_private *dev_priv = dev->dev_private;
6800         static const unsigned int blb_vco[8] = {
6801                 [0] = 3200000,
6802                 [1] = 4000000,
6803                 [2] = 5333333,
6804                 [3] = 4800000,
6805                 [4] = 6400000,
6806         };
6807         static const unsigned int pnv_vco[8] = {
6808                 [0] = 3200000,
6809                 [1] = 4000000,
6810                 [2] = 5333333,
6811                 [3] = 4800000,
6812                 [4] = 2666667,
6813         };
6814         static const unsigned int cl_vco[8] = {
6815                 [0] = 3200000,
6816                 [1] = 4000000,
6817                 [2] = 5333333,
6818                 [3] = 6400000,
6819                 [4] = 3333333,
6820                 [5] = 3566667,
6821                 [6] = 4266667,
6822         };
6823         static const unsigned int elk_vco[8] = {
6824                 [0] = 3200000,
6825                 [1] = 4000000,
6826                 [2] = 5333333,
6827                 [3] = 4800000,
6828         };
6829         static const unsigned int ctg_vco[8] = {
6830                 [0] = 3200000,
6831                 [1] = 4000000,
6832                 [2] = 5333333,
6833                 [3] = 6400000,
6834                 [4] = 2666667,
6835                 [5] = 4266667,
6836         };
6837         const unsigned int *vco_table;
6838         unsigned int vco;
6839         uint8_t tmp = 0;
6840
6841         /* FIXME other chipsets? */
6842         if (IS_GM45(dev))
6843                 vco_table = ctg_vco;
6844         else if (IS_G4X(dev))
6845                 vco_table = elk_vco;
6846         else if (IS_CRESTLINE(dev))
6847                 vco_table = cl_vco;
6848         else if (IS_PINEVIEW(dev))
6849                 vco_table = pnv_vco;
6850         else if (IS_G33(dev))
6851                 vco_table = blb_vco;
6852         else
6853                 return 0;
6854
6855         tmp = I915_READ(IS_MOBILE(dev) ? HPLLVCO_MOBILE : HPLLVCO);
6856
6857         vco = vco_table[tmp & 0x7];
6858         if (vco == 0)
6859                 DRM_ERROR("Bad HPLL VCO (HPLLVCO=0x%02x)\n", tmp);
6860         else
6861                 DRM_DEBUG_KMS("HPLL VCO %u kHz\n", vco);
6862
6863         return vco;
6864 }
6865
6866 static int gm45_get_display_clock_speed(struct drm_device *dev)
6867 {
6868         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
6869         uint16_t tmp = 0;
6870
6871         pci_read_config_word(dev->pdev, GCFGC, &tmp);
6872
6873         cdclk_sel = (tmp >> 12) & 0x1;
6874
6875         switch (vco) {
6876         case 2666667:
6877         case 4000000:
6878         case 5333333:
6879                 return cdclk_sel ? 333333 : 222222;
6880         case 3200000:
6881                 return cdclk_sel ? 320000 : 228571;
6882         default:
6883                 DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u, CFGC=0x%04x\n", vco, tmp);
6884                 return 222222;
6885         }
6886 }
6887
6888 static int i965gm_get_display_clock_speed(struct drm_device *dev)
6889 {
6890         static const uint8_t div_3200[] = { 16, 10,  8 };
6891         static const uint8_t div_4000[] = { 20, 12, 10 };
6892         static const uint8_t div_5333[] = { 24, 16, 14 };
6893         const uint8_t *div_table;
6894         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
6895         uint16_t tmp = 0;
6896
6897         pci_read_config_word(dev->pdev, GCFGC, &tmp);
6898
6899         cdclk_sel = ((tmp >> 8) & 0x1f) - 1;
6900
6901         if (cdclk_sel >= ARRAY_SIZE(div_3200))
6902                 goto fail;
6903
6904         switch (vco) {
6905         case 3200000:
6906                 div_table = div_3200;
6907                 break;
6908         case 4000000:
6909                 div_table = div_4000;
6910                 break;
6911         case 5333333:
6912                 div_table = div_5333;
6913                 break;
6914         default:
6915                 goto fail;
6916         }
6917
6918         return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]);
6919
6920 fail:
6921         DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%04x\n", vco, tmp);
6922         return 200000;
6923 }
6924
6925 static int g33_get_display_clock_speed(struct drm_device *dev)
6926 {
6927         static const uint8_t div_3200[] = { 12, 10,  8,  7, 5, 16 };
6928         static const uint8_t div_4000[] = { 14, 12, 10,  8, 6, 20 };
6929         static const uint8_t div_4800[] = { 20, 14, 12, 10, 8, 24 };
6930         static const uint8_t div_5333[] = { 20, 16, 12, 12, 8, 28 };
6931         const uint8_t *div_table;
6932         unsigned int cdclk_sel, vco = intel_hpll_vco(dev);
6933         uint16_t tmp = 0;
6934
6935         pci_read_config_word(dev->pdev, GCFGC, &tmp);
6936
6937         cdclk_sel = (tmp >> 4) & 0x7;
6938
6939         if (cdclk_sel >= ARRAY_SIZE(div_3200))
6940                 goto fail;
6941
6942         switch (vco) {
6943         case 3200000:
6944                 div_table = div_3200;
6945                 break;
6946         case 4000000:
6947                 div_table = div_4000;
6948                 break;
6949         case 4800000:
6950                 div_table = div_4800;
6951                 break;
6952         case 5333333:
6953                 div_table = div_5333;
6954                 break;
6955         default:
6956                 goto fail;
6957         }
6958
6959         return DIV_ROUND_CLOSEST(vco, div_table[cdclk_sel]);
6960
6961 fail:
6962         DRM_ERROR("Unable to determine CDCLK. HPLL VCO=%u kHz, CFGC=0x%08x\n", vco, tmp);
6963         return 190476;
6964 }
6965
6966 static void
6967 intel_reduce_m_n_ratio(uint32_t *num, uint32_t *den)
6968 {
6969         while (*num > DATA_LINK_M_N_MASK ||
6970                *den > DATA_LINK_M_N_MASK) {
6971                 *num >>= 1;
6972                 *den >>= 1;
6973         }
6974 }
6975
6976 static void compute_m_n(unsigned int m, unsigned int n,
6977                         uint32_t *ret_m, uint32_t *ret_n)
6978 {
6979         *ret_n = min_t(unsigned int, roundup_pow_of_two(n), DATA_LINK_N_MAX);
6980         *ret_m = div_u64((uint64_t) m * *ret_n, n);
6981         intel_reduce_m_n_ratio(ret_m, ret_n);
6982 }
6983
6984 void
6985 intel_link_compute_m_n(int bits_per_pixel, int nlanes,
6986                        int pixel_clock, int link_clock,
6987                        struct intel_link_m_n *m_n)
6988 {
6989         m_n->tu = 64;
6990
6991         compute_m_n(bits_per_pixel * pixel_clock,
6992                     link_clock * nlanes * 8,
6993                     &m_n->gmch_m, &m_n->gmch_n);
6994
6995         compute_m_n(pixel_clock, link_clock,
6996                     &m_n->link_m, &m_n->link_n);
6997 }
6998
6999 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
7000 {
7001         if (i915.panel_use_ssc >= 0)
7002                 return i915.panel_use_ssc != 0;
7003         return dev_priv->vbt.lvds_use_ssc
7004                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
7005 }
7006
7007 static int i9xx_get_refclk(const struct intel_crtc_state *crtc_state,
7008                            int num_connectors)
7009 {
7010         struct drm_device *dev = crtc_state->base.crtc->dev;
7011         struct drm_i915_private *dev_priv = dev->dev_private;
7012         int refclk;
7013
7014         WARN_ON(!crtc_state->base.state);
7015
7016         if (IS_VALLEYVIEW(dev) || IS_BROXTON(dev)) {
7017                 refclk = 100000;
7018         } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7019             intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
7020                 refclk = dev_priv->vbt.lvds_ssc_freq;
7021                 DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
7022         } else if (!IS_GEN2(dev)) {
7023                 refclk = 96000;
7024         } else {
7025                 refclk = 48000;
7026         }
7027
7028         return refclk;
7029 }
7030
7031 static uint32_t pnv_dpll_compute_fp(struct dpll *dpll)
7032 {
7033         return (1 << dpll->n) << 16 | dpll->m2;
7034 }
7035
7036 static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
7037 {
7038         return dpll->n << 16 | dpll->m1 << 8 | dpll->m2;
7039 }
7040
7041 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
7042                                      struct intel_crtc_state *crtc_state,
7043                                      intel_clock_t *reduced_clock)
7044 {
7045         struct drm_device *dev = crtc->base.dev;
7046         u32 fp, fp2 = 0;
7047
7048         if (IS_PINEVIEW(dev)) {
7049                 fp = pnv_dpll_compute_fp(&crtc_state->dpll);
7050                 if (reduced_clock)
7051                         fp2 = pnv_dpll_compute_fp(reduced_clock);
7052         } else {
7053                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
7054                 if (reduced_clock)
7055                         fp2 = i9xx_dpll_compute_fp(reduced_clock);
7056         }
7057
7058         crtc_state->dpll_hw_state.fp0 = fp;
7059
7060         crtc->lowfreq_avail = false;
7061         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7062             reduced_clock) {
7063                 crtc_state->dpll_hw_state.fp1 = fp2;
7064                 crtc->lowfreq_avail = true;
7065         } else {
7066                 crtc_state->dpll_hw_state.fp1 = fp;
7067         }
7068 }
7069
7070 static void vlv_pllb_recal_opamp(struct drm_i915_private *dev_priv, enum pipe
7071                 pipe)
7072 {
7073         u32 reg_val;
7074
7075         /*
7076          * PLLB opamp always calibrates to max value of 0x3f, force enable it
7077          * and set it to a reasonable value instead.
7078          */
7079         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7080         reg_val &= 0xffffff00;
7081         reg_val |= 0x00000030;
7082         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7083
7084         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7085         reg_val &= 0x8cffffff;
7086         reg_val = 0x8c000000;
7087         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7088
7089         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW9(1));
7090         reg_val &= 0xffffff00;
7091         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9(1), reg_val);
7092
7093         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_REF_DW13);
7094         reg_val &= 0x00ffffff;
7095         reg_val |= 0xb0000000;
7096         vlv_dpio_write(dev_priv, pipe, VLV_REF_DW13, reg_val);
7097 }
7098
7099 static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
7100                                          struct intel_link_m_n *m_n)
7101 {
7102         struct drm_device *dev = crtc->base.dev;
7103         struct drm_i915_private *dev_priv = dev->dev_private;
7104         int pipe = crtc->pipe;
7105
7106         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7107         I915_WRITE(PCH_TRANS_DATA_N1(pipe), m_n->gmch_n);
7108         I915_WRITE(PCH_TRANS_LINK_M1(pipe), m_n->link_m);
7109         I915_WRITE(PCH_TRANS_LINK_N1(pipe), m_n->link_n);
7110 }
7111
7112 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
7113                                          struct intel_link_m_n *m_n,
7114                                          struct intel_link_m_n *m2_n2)
7115 {
7116         struct drm_device *dev = crtc->base.dev;
7117         struct drm_i915_private *dev_priv = dev->dev_private;
7118         int pipe = crtc->pipe;
7119         enum transcoder transcoder = crtc->config->cpu_transcoder;
7120
7121         if (INTEL_INFO(dev)->gen >= 5) {
7122                 I915_WRITE(PIPE_DATA_M1(transcoder), TU_SIZE(m_n->tu) | m_n->gmch_m);
7123                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
7124                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
7125                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
7126                 /* M2_N2 registers to be set only for gen < 8 (M2_N2 available
7127                  * for gen < 8) and if DRRS is supported (to make sure the
7128                  * registers are not unnecessarily accessed).
7129                  */
7130                 if (m2_n2 && (IS_CHERRYVIEW(dev) || INTEL_INFO(dev)->gen < 8) &&
7131                         crtc->config->has_drrs) {
7132                         I915_WRITE(PIPE_DATA_M2(transcoder),
7133                                         TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
7134                         I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
7135                         I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
7136                         I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
7137                 }
7138         } else {
7139                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
7140                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
7141                 I915_WRITE(PIPE_LINK_M_G4X(pipe), m_n->link_m);
7142                 I915_WRITE(PIPE_LINK_N_G4X(pipe), m_n->link_n);
7143         }
7144 }
7145
7146 void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n)
7147 {
7148         struct intel_link_m_n *dp_m_n, *dp_m2_n2 = NULL;
7149
7150         if (m_n == M1_N1) {
7151                 dp_m_n = &crtc->config->dp_m_n;
7152                 dp_m2_n2 = &crtc->config->dp_m2_n2;
7153         } else if (m_n == M2_N2) {
7154
7155                 /*
7156                  * M2_N2 registers are not supported. Hence m2_n2 divider value
7157                  * needs to be programmed into M1_N1.
7158                  */
7159                 dp_m_n = &crtc->config->dp_m2_n2;
7160         } else {
7161                 DRM_ERROR("Unsupported divider value\n");
7162                 return;
7163         }
7164
7165         if (crtc->config->has_pch_encoder)
7166                 intel_pch_transcoder_set_m_n(crtc, &crtc->config->dp_m_n);
7167         else
7168                 intel_cpu_transcoder_set_m_n(crtc, dp_m_n, dp_m2_n2);
7169 }
7170
7171 static void vlv_compute_dpll(struct intel_crtc *crtc,
7172                              struct intel_crtc_state *pipe_config)
7173 {
7174         u32 dpll, dpll_md;
7175
7176         /*
7177          * Enable DPIO clock input. We should never disable the reference
7178          * clock for pipe B, since VGA hotplug / manual detection depends
7179          * on it.
7180          */
7181         dpll = DPLL_EXT_BUFFER_ENABLE_VLV | DPLL_REFA_CLK_ENABLE_VLV |
7182                 DPLL_VGA_MODE_DIS | DPLL_INTEGRATED_CLOCK_VLV;
7183         /* We should never disable this, set it here for state tracking */
7184         if (crtc->pipe == PIPE_B)
7185                 dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7186         dpll |= DPLL_VCO_ENABLE;
7187         pipe_config->dpll_hw_state.dpll = dpll;
7188
7189         dpll_md = (pipe_config->pixel_multiplier - 1)
7190                 << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7191         pipe_config->dpll_hw_state.dpll_md = dpll_md;
7192 }
7193
7194 static void vlv_prepare_pll(struct intel_crtc *crtc,
7195                             const struct intel_crtc_state *pipe_config)
7196 {
7197         struct drm_device *dev = crtc->base.dev;
7198         struct drm_i915_private *dev_priv = dev->dev_private;
7199         int pipe = crtc->pipe;
7200         u32 mdiv;
7201         u32 bestn, bestm1, bestm2, bestp1, bestp2;
7202         u32 coreclk, reg_val;
7203
7204         mutex_lock(&dev_priv->sb_lock);
7205
7206         bestn = pipe_config->dpll.n;
7207         bestm1 = pipe_config->dpll.m1;
7208         bestm2 = pipe_config->dpll.m2;
7209         bestp1 = pipe_config->dpll.p1;
7210         bestp2 = pipe_config->dpll.p2;
7211
7212         /* See eDP HDMI DPIO driver vbios notes doc */
7213
7214         /* PLL B needs special handling */
7215         if (pipe == PIPE_B)
7216                 vlv_pllb_recal_opamp(dev_priv, pipe);
7217
7218         /* Set up Tx target for periodic Rcomp update */
7219         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW9_BCAST, 0x0100000f);
7220
7221         /* Disable target IRef on PLL */
7222         reg_val = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW8(pipe));
7223         reg_val &= 0x00ffffff;
7224         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW8(pipe), reg_val);
7225
7226         /* Disable fast lock */
7227         vlv_dpio_write(dev_priv, pipe, VLV_CMN_DW0, 0x610);
7228
7229         /* Set idtafcrecal before PLL is enabled */
7230         mdiv = ((bestm1 << DPIO_M1DIV_SHIFT) | (bestm2 & DPIO_M2DIV_MASK));
7231         mdiv |= ((bestp1 << DPIO_P1_SHIFT) | (bestp2 << DPIO_P2_SHIFT));
7232         mdiv |= ((bestn << DPIO_N_SHIFT));
7233         mdiv |= (1 << DPIO_K_SHIFT);
7234
7235         /*
7236          * Post divider depends on pixel clock rate, DAC vs digital (and LVDS,
7237          * but we don't support that).
7238          * Note: don't use the DAC post divider as it seems unstable.
7239          */
7240         mdiv |= (DPIO_POST_DIV_HDMIDP << DPIO_POST_DIV_SHIFT);
7241         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7242
7243         mdiv |= DPIO_ENABLE_CALIBRATION;
7244         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW3(pipe), mdiv);
7245
7246         /* Set HBR and RBR LPF coefficients */
7247         if (pipe_config->port_clock == 162000 ||
7248             intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG) ||
7249             intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI))
7250                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7251                                  0x009f0003);
7252         else
7253                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
7254                                  0x00d0000f);
7255
7256         if (pipe_config->has_dp_encoder) {
7257                 /* Use SSC source */
7258                 if (pipe == PIPE_A)
7259                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7260                                          0x0df40000);
7261                 else
7262                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7263                                          0x0df70000);
7264         } else { /* HDMI or VGA */
7265                 /* Use bend source */
7266                 if (pipe == PIPE_A)
7267                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7268                                          0x0df70000);
7269                 else
7270                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
7271                                          0x0df40000);
7272         }
7273
7274         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
7275         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
7276         if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
7277             intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
7278                 coreclk |= 0x01000000;
7279         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
7280
7281         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW11(pipe), 0x87871000);
7282         mutex_unlock(&dev_priv->sb_lock);
7283 }
7284
7285 static void chv_compute_dpll(struct intel_crtc *crtc,
7286                              struct intel_crtc_state *pipe_config)
7287 {
7288         pipe_config->dpll_hw_state.dpll = DPLL_SSC_REF_CLOCK_CHV |
7289                 DPLL_REFA_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS |
7290                 DPLL_VCO_ENABLE;
7291         if (crtc->pipe != PIPE_A)
7292                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
7293
7294         pipe_config->dpll_hw_state.dpll_md =
7295                 (pipe_config->pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7296 }
7297
7298 static void chv_prepare_pll(struct intel_crtc *crtc,
7299                             const struct intel_crtc_state *pipe_config)
7300 {
7301         struct drm_device *dev = crtc->base.dev;
7302         struct drm_i915_private *dev_priv = dev->dev_private;
7303         int pipe = crtc->pipe;
7304         int dpll_reg = DPLL(crtc->pipe);
7305         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7306         u32 loopfilter, tribuf_calcntr;
7307         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
7308         u32 dpio_val;
7309         int vco;
7310
7311         bestn = pipe_config->dpll.n;
7312         bestm2_frac = pipe_config->dpll.m2 & 0x3fffff;
7313         bestm1 = pipe_config->dpll.m1;
7314         bestm2 = pipe_config->dpll.m2 >> 22;
7315         bestp1 = pipe_config->dpll.p1;
7316         bestp2 = pipe_config->dpll.p2;
7317         vco = pipe_config->dpll.vco;
7318         dpio_val = 0;
7319         loopfilter = 0;
7320
7321         /*
7322          * Enable Refclk and SSC
7323          */
7324         I915_WRITE(dpll_reg,
7325                    pipe_config->dpll_hw_state.dpll & ~DPLL_VCO_ENABLE);
7326
7327         mutex_lock(&dev_priv->sb_lock);
7328
7329         /* p1 and p2 divider */
7330         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW13(port),
7331                         5 << DPIO_CHV_S1_DIV_SHIFT |
7332                         bestp1 << DPIO_CHV_P1_DIV_SHIFT |
7333                         bestp2 << DPIO_CHV_P2_DIV_SHIFT |
7334                         1 << DPIO_CHV_K_DIV_SHIFT);
7335
7336         /* Feedback post-divider - m2 */
7337         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW0(port), bestm2);
7338
7339         /* Feedback refclk divider - n and m1 */
7340         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW1(port),
7341                         DPIO_CHV_M1_DIV_BY_2 |
7342                         1 << DPIO_CHV_N_DIV_SHIFT);
7343
7344         /* M2 fraction division */
7345         if (bestm2_frac)
7346                 vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW2(port), bestm2_frac);
7347
7348         /* M2 fraction division enable */
7349         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW3(port));
7350         dpio_val &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
7351         dpio_val |= (2 << DPIO_CHV_FEEDFWD_GAIN_SHIFT);
7352         if (bestm2_frac)
7353                 dpio_val |= DPIO_CHV_FRAC_DIV_EN;
7354         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW3(port), dpio_val);
7355
7356         /* Program digital lock detect threshold */
7357         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW9(port));
7358         dpio_val &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
7359                                         DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
7360         dpio_val |= (0x5 << DPIO_CHV_INT_LOCK_THRESHOLD_SHIFT);
7361         if (!bestm2_frac)
7362                 dpio_val |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
7363         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW9(port), dpio_val);
7364
7365         /* Loop filter */
7366         if (vco == 5400000) {
7367                 loopfilter |= (0x3 << DPIO_CHV_PROP_COEFF_SHIFT);
7368                 loopfilter |= (0x8 << DPIO_CHV_INT_COEFF_SHIFT);
7369                 loopfilter |= (0x1 << DPIO_CHV_GAIN_CTRL_SHIFT);
7370                 tribuf_calcntr = 0x9;
7371         } else if (vco <= 6200000) {
7372                 loopfilter |= (0x5 << DPIO_CHV_PROP_COEFF_SHIFT);
7373                 loopfilter |= (0xB << DPIO_CHV_INT_COEFF_SHIFT);
7374                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7375                 tribuf_calcntr = 0x9;
7376         } else if (vco <= 6480000) {
7377                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7378                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7379                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7380                 tribuf_calcntr = 0x8;
7381         } else {
7382                 /* Not supported. Apply the same limits as in the max case */
7383                 loopfilter |= (0x4 << DPIO_CHV_PROP_COEFF_SHIFT);
7384                 loopfilter |= (0x9 << DPIO_CHV_INT_COEFF_SHIFT);
7385                 loopfilter |= (0x3 << DPIO_CHV_GAIN_CTRL_SHIFT);
7386                 tribuf_calcntr = 0;
7387         }
7388         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW6(port), loopfilter);
7389
7390         dpio_val = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW8(port));
7391         dpio_val &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
7392         dpio_val |= (tribuf_calcntr << DPIO_CHV_TDC_TARGET_CNT_SHIFT);
7393         vlv_dpio_write(dev_priv, pipe, CHV_PLL_DW8(port), dpio_val);
7394
7395         /* AFC Recal */
7396         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port),
7397                         vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW14(port)) |
7398                         DPIO_AFC_RECAL);
7399
7400         mutex_unlock(&dev_priv->sb_lock);
7401 }
7402
7403 /**
7404  * vlv_force_pll_on - forcibly enable just the PLL
7405  * @dev_priv: i915 private structure
7406  * @pipe: pipe PLL to enable
7407  * @dpll: PLL configuration
7408  *
7409  * Enable the PLL for @pipe using the supplied @dpll config. To be used
7410  * in cases where we need the PLL enabled even when @pipe is not going to
7411  * be enabled.
7412  */
7413 void vlv_force_pll_on(struct drm_device *dev, enum pipe pipe,
7414                       const struct dpll *dpll)
7415 {
7416         struct intel_crtc *crtc =
7417                 to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
7418         struct intel_crtc_state pipe_config = {
7419                 .base.crtc = &crtc->base,
7420                 .pixel_multiplier = 1,
7421                 .dpll = *dpll,
7422         };
7423
7424         if (IS_CHERRYVIEW(dev)) {
7425                 chv_compute_dpll(crtc, &pipe_config);
7426                 chv_prepare_pll(crtc, &pipe_config);
7427                 chv_enable_pll(crtc, &pipe_config);
7428         } else {
7429                 vlv_compute_dpll(crtc, &pipe_config);
7430                 vlv_prepare_pll(crtc, &pipe_config);
7431                 vlv_enable_pll(crtc, &pipe_config);
7432         }
7433 }
7434
7435 /**
7436  * vlv_force_pll_off - forcibly disable just the PLL
7437  * @dev_priv: i915 private structure
7438  * @pipe: pipe PLL to disable
7439  *
7440  * Disable the PLL for @pipe. To be used in cases where we need
7441  * the PLL enabled even when @pipe is not going to be enabled.
7442  */
7443 void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe)
7444 {
7445         if (IS_CHERRYVIEW(dev))
7446                 chv_disable_pll(to_i915(dev), pipe);
7447         else
7448                 vlv_disable_pll(to_i915(dev), pipe);
7449 }
7450
7451 static void i9xx_compute_dpll(struct intel_crtc *crtc,
7452                               struct intel_crtc_state *crtc_state,
7453                               intel_clock_t *reduced_clock,
7454                               int num_connectors)
7455 {
7456         struct drm_device *dev = crtc->base.dev;
7457         struct drm_i915_private *dev_priv = dev->dev_private;
7458         u32 dpll;
7459         bool is_sdvo;
7460         struct dpll *clock = &crtc_state->dpll;
7461
7462         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7463
7464         is_sdvo = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO) ||
7465                 intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI);
7466
7467         dpll = DPLL_VGA_MODE_DIS;
7468
7469         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
7470                 dpll |= DPLLB_MODE_LVDS;
7471         else
7472                 dpll |= DPLLB_MODE_DAC_SERIAL;
7473
7474         if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
7475                 dpll |= (crtc_state->pixel_multiplier - 1)
7476                         << SDVO_MULTIPLIER_SHIFT_HIRES;
7477         }
7478
7479         if (is_sdvo)
7480                 dpll |= DPLL_SDVO_HIGH_SPEED;
7481
7482         if (crtc_state->has_dp_encoder)
7483                 dpll |= DPLL_SDVO_HIGH_SPEED;
7484
7485         /* compute bitmask from p1 value */
7486         if (IS_PINEVIEW(dev))
7487                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
7488         else {
7489                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7490                 if (IS_G4X(dev) && reduced_clock)
7491                         dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
7492         }
7493         switch (clock->p2) {
7494         case 5:
7495                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
7496                 break;
7497         case 7:
7498                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
7499                 break;
7500         case 10:
7501                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
7502                 break;
7503         case 14:
7504                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
7505                 break;
7506         }
7507         if (INTEL_INFO(dev)->gen >= 4)
7508                 dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
7509
7510         if (crtc_state->sdvo_tv_clock)
7511                 dpll |= PLL_REF_INPUT_TVCLKINBC;
7512         else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7513                  intel_panel_use_ssc(dev_priv) && num_connectors < 2)
7514                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7515         else
7516                 dpll |= PLL_REF_INPUT_DREFCLK;
7517
7518         dpll |= DPLL_VCO_ENABLE;
7519         crtc_state->dpll_hw_state.dpll = dpll;
7520
7521         if (INTEL_INFO(dev)->gen >= 4) {
7522                 u32 dpll_md = (crtc_state->pixel_multiplier - 1)
7523                         << DPLL_MD_UDI_MULTIPLIER_SHIFT;
7524                 crtc_state->dpll_hw_state.dpll_md = dpll_md;
7525         }
7526 }
7527
7528 static void i8xx_compute_dpll(struct intel_crtc *crtc,
7529                               struct intel_crtc_state *crtc_state,
7530                               intel_clock_t *reduced_clock,
7531                               int num_connectors)
7532 {
7533         struct drm_device *dev = crtc->base.dev;
7534         struct drm_i915_private *dev_priv = dev->dev_private;
7535         u32 dpll;
7536         struct dpll *clock = &crtc_state->dpll;
7537
7538         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
7539
7540         dpll = DPLL_VGA_MODE_DIS;
7541
7542         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
7543                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7544         } else {
7545                 if (clock->p1 == 2)
7546                         dpll |= PLL_P1_DIVIDE_BY_TWO;
7547                 else
7548                         dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
7549                 if (clock->p2 == 4)
7550                         dpll |= PLL_P2_DIVIDE_BY_4;
7551         }
7552
7553         if (!IS_I830(dev) && intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO))
7554                 dpll |= DPLL_DVO_2X_MODE;
7555
7556         if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
7557                  intel_panel_use_ssc(dev_priv) && num_connectors < 2)
7558                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
7559         else
7560                 dpll |= PLL_REF_INPUT_DREFCLK;
7561
7562         dpll |= DPLL_VCO_ENABLE;
7563         crtc_state->dpll_hw_state.dpll = dpll;
7564 }
7565
7566 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
7567 {
7568         struct drm_device *dev = intel_crtc->base.dev;
7569         struct drm_i915_private *dev_priv = dev->dev_private;
7570         enum pipe pipe = intel_crtc->pipe;
7571         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
7572         struct drm_display_mode *adjusted_mode =
7573                 &intel_crtc->config->base.adjusted_mode;
7574         uint32_t crtc_vtotal, crtc_vblank_end;
7575         int vsyncshift = 0;
7576
7577         /* We need to be careful not to changed the adjusted mode, for otherwise
7578          * the hw state checker will get angry at the mismatch. */
7579         crtc_vtotal = adjusted_mode->crtc_vtotal;
7580         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
7581
7582         if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
7583                 /* the chip adds 2 halflines automatically */
7584                 crtc_vtotal -= 1;
7585                 crtc_vblank_end -= 1;
7586
7587                 if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
7588                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
7589                 else
7590                         vsyncshift = adjusted_mode->crtc_hsync_start -
7591                                 adjusted_mode->crtc_htotal / 2;
7592                 if (vsyncshift < 0)
7593                         vsyncshift += adjusted_mode->crtc_htotal;
7594         }
7595
7596         if (INTEL_INFO(dev)->gen > 3)
7597                 I915_WRITE(VSYNCSHIFT(cpu_transcoder), vsyncshift);
7598
7599         I915_WRITE(HTOTAL(cpu_transcoder),
7600                    (adjusted_mode->crtc_hdisplay - 1) |
7601                    ((adjusted_mode->crtc_htotal - 1) << 16));
7602         I915_WRITE(HBLANK(cpu_transcoder),
7603                    (adjusted_mode->crtc_hblank_start - 1) |
7604                    ((adjusted_mode->crtc_hblank_end - 1) << 16));
7605         I915_WRITE(HSYNC(cpu_transcoder),
7606                    (adjusted_mode->crtc_hsync_start - 1) |
7607                    ((adjusted_mode->crtc_hsync_end - 1) << 16));
7608
7609         I915_WRITE(VTOTAL(cpu_transcoder),
7610                    (adjusted_mode->crtc_vdisplay - 1) |
7611                    ((crtc_vtotal - 1) << 16));
7612         I915_WRITE(VBLANK(cpu_transcoder),
7613                    (adjusted_mode->crtc_vblank_start - 1) |
7614                    ((crtc_vblank_end - 1) << 16));
7615         I915_WRITE(VSYNC(cpu_transcoder),
7616                    (adjusted_mode->crtc_vsync_start - 1) |
7617                    ((adjusted_mode->crtc_vsync_end - 1) << 16));
7618
7619         /* Workaround: when the EDP input selection is B, the VTOTAL_B must be
7620          * programmed with the VTOTAL_EDP value. Same for VTOTAL_C. This is
7621          * documented on the DDI_FUNC_CTL register description, EDP Input Select
7622          * bits. */
7623         if (IS_HASWELL(dev) && cpu_transcoder == TRANSCODER_EDP &&
7624             (pipe == PIPE_B || pipe == PIPE_C))
7625                 I915_WRITE(VTOTAL(pipe), I915_READ(VTOTAL(cpu_transcoder)));
7626
7627         /* pipesrc controls the size that is scaled from, which should
7628          * always be the user's requested size.
7629          */
7630         I915_WRITE(PIPESRC(pipe),
7631                    ((intel_crtc->config->pipe_src_w - 1) << 16) |
7632                    (intel_crtc->config->pipe_src_h - 1));
7633 }
7634
7635 static void intel_get_pipe_timings(struct intel_crtc *crtc,
7636                                    struct intel_crtc_state *pipe_config)
7637 {
7638         struct drm_device *dev = crtc->base.dev;
7639         struct drm_i915_private *dev_priv = dev->dev_private;
7640         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
7641         uint32_t tmp;
7642
7643         tmp = I915_READ(HTOTAL(cpu_transcoder));
7644         pipe_config->base.adjusted_mode.crtc_hdisplay = (tmp & 0xffff) + 1;
7645         pipe_config->base.adjusted_mode.crtc_htotal = ((tmp >> 16) & 0xffff) + 1;
7646         tmp = I915_READ(HBLANK(cpu_transcoder));
7647         pipe_config->base.adjusted_mode.crtc_hblank_start = (tmp & 0xffff) + 1;
7648         pipe_config->base.adjusted_mode.crtc_hblank_end = ((tmp >> 16) & 0xffff) + 1;
7649         tmp = I915_READ(HSYNC(cpu_transcoder));
7650         pipe_config->base.adjusted_mode.crtc_hsync_start = (tmp & 0xffff) + 1;
7651         pipe_config->base.adjusted_mode.crtc_hsync_end = ((tmp >> 16) & 0xffff) + 1;
7652
7653         tmp = I915_READ(VTOTAL(cpu_transcoder));
7654         pipe_config->base.adjusted_mode.crtc_vdisplay = (tmp & 0xffff) + 1;
7655         pipe_config->base.adjusted_mode.crtc_vtotal = ((tmp >> 16) & 0xffff) + 1;
7656         tmp = I915_READ(VBLANK(cpu_transcoder));
7657         pipe_config->base.adjusted_mode.crtc_vblank_start = (tmp & 0xffff) + 1;
7658         pipe_config->base.adjusted_mode.crtc_vblank_end = ((tmp >> 16) & 0xffff) + 1;
7659         tmp = I915_READ(VSYNC(cpu_transcoder));
7660         pipe_config->base.adjusted_mode.crtc_vsync_start = (tmp & 0xffff) + 1;
7661         pipe_config->base.adjusted_mode.crtc_vsync_end = ((tmp >> 16) & 0xffff) + 1;
7662
7663         if (I915_READ(PIPECONF(cpu_transcoder)) & PIPECONF_INTERLACE_MASK) {
7664                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_INTERLACE;
7665                 pipe_config->base.adjusted_mode.crtc_vtotal += 1;
7666                 pipe_config->base.adjusted_mode.crtc_vblank_end += 1;
7667         }
7668
7669         tmp = I915_READ(PIPESRC(crtc->pipe));
7670         pipe_config->pipe_src_h = (tmp & 0xffff) + 1;
7671         pipe_config->pipe_src_w = ((tmp >> 16) & 0xffff) + 1;
7672
7673         pipe_config->base.mode.vdisplay = pipe_config->pipe_src_h;
7674         pipe_config->base.mode.hdisplay = pipe_config->pipe_src_w;
7675 }
7676
7677 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
7678                                  struct intel_crtc_state *pipe_config)
7679 {
7680         mode->hdisplay = pipe_config->base.adjusted_mode.crtc_hdisplay;
7681         mode->htotal = pipe_config->base.adjusted_mode.crtc_htotal;
7682         mode->hsync_start = pipe_config->base.adjusted_mode.crtc_hsync_start;
7683         mode->hsync_end = pipe_config->base.adjusted_mode.crtc_hsync_end;
7684
7685         mode->vdisplay = pipe_config->base.adjusted_mode.crtc_vdisplay;
7686         mode->vtotal = pipe_config->base.adjusted_mode.crtc_vtotal;
7687         mode->vsync_start = pipe_config->base.adjusted_mode.crtc_vsync_start;
7688         mode->vsync_end = pipe_config->base.adjusted_mode.crtc_vsync_end;
7689
7690         mode->flags = pipe_config->base.adjusted_mode.flags;
7691
7692         mode->clock = pipe_config->base.adjusted_mode.crtc_clock;
7693         mode->flags |= pipe_config->base.adjusted_mode.flags;
7694 }
7695
7696 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
7697 {
7698         struct drm_device *dev = intel_crtc->base.dev;
7699         struct drm_i915_private *dev_priv = dev->dev_private;
7700         uint32_t pipeconf;
7701
7702         pipeconf = 0;
7703
7704         if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
7705             (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
7706                 pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE;
7707
7708         if (intel_crtc->config->double_wide)
7709                 pipeconf |= PIPECONF_DOUBLE_WIDE;
7710
7711         /* only g4x and later have fancy bpc/dither controls */
7712         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
7713                 /* Bspec claims that we can't use dithering for 30bpp pipes. */
7714                 if (intel_crtc->config->dither && intel_crtc->config->pipe_bpp != 30)
7715                         pipeconf |= PIPECONF_DITHER_EN |
7716                                     PIPECONF_DITHER_TYPE_SP;
7717
7718                 switch (intel_crtc->config->pipe_bpp) {
7719                 case 18:
7720                         pipeconf |= PIPECONF_6BPC;
7721                         break;
7722                 case 24:
7723                         pipeconf |= PIPECONF_8BPC;
7724                         break;
7725                 case 30:
7726                         pipeconf |= PIPECONF_10BPC;
7727                         break;
7728                 default:
7729                         /* Case prevented by intel_choose_pipe_bpp_dither. */
7730                         BUG();
7731                 }
7732         }
7733
7734         if (HAS_PIPE_CXSR(dev)) {
7735                 if (intel_crtc->lowfreq_avail) {
7736                         DRM_DEBUG_KMS("enabling CxSR downclocking\n");
7737                         pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
7738                 } else {
7739                         DRM_DEBUG_KMS("disabling CxSR downclocking\n");
7740                 }
7741         }
7742
7743         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
7744                 if (INTEL_INFO(dev)->gen < 4 ||
7745                     intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
7746                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
7747                 else
7748                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
7749         } else
7750                 pipeconf |= PIPECONF_PROGRESSIVE;
7751
7752         if (IS_VALLEYVIEW(dev) && intel_crtc->config->limited_color_range)
7753                 pipeconf |= PIPECONF_COLOR_RANGE_SELECT;
7754
7755         I915_WRITE(PIPECONF(intel_crtc->pipe), pipeconf);
7756         POSTING_READ(PIPECONF(intel_crtc->pipe));
7757 }
7758
7759 static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
7760                                    struct intel_crtc_state *crtc_state)
7761 {
7762         struct drm_device *dev = crtc->base.dev;
7763         struct drm_i915_private *dev_priv = dev->dev_private;
7764         int refclk, num_connectors = 0;
7765         intel_clock_t clock;
7766         bool ok;
7767         bool is_dsi = false;
7768         struct intel_encoder *encoder;
7769         const intel_limit_t *limit;
7770         struct drm_atomic_state *state = crtc_state->base.state;
7771         struct drm_connector *connector;
7772         struct drm_connector_state *connector_state;
7773         int i;
7774
7775         memset(&crtc_state->dpll_hw_state, 0,
7776                sizeof(crtc_state->dpll_hw_state));
7777
7778         for_each_connector_in_state(state, connector, connector_state, i) {
7779                 if (connector_state->crtc != &crtc->base)
7780                         continue;
7781
7782                 encoder = to_intel_encoder(connector_state->best_encoder);
7783
7784                 switch (encoder->type) {
7785                 case INTEL_OUTPUT_DSI:
7786                         is_dsi = true;
7787                         break;
7788                 default:
7789                         break;
7790                 }
7791
7792                 num_connectors++;
7793         }
7794
7795         if (is_dsi)
7796                 return 0;
7797
7798         if (!crtc_state->clock_set) {
7799                 refclk = i9xx_get_refclk(crtc_state, num_connectors);
7800
7801                 /*
7802                  * Returns a set of divisors for the desired target clock with
7803                  * the given refclk, or FALSE.  The returned values represent
7804                  * the clock equation: reflck * (5 * (m1 + 2) + (m2 + 2)) / (n +
7805                  * 2) / p1 / p2.
7806                  */
7807                 limit = intel_limit(crtc_state, refclk);
7808                 ok = dev_priv->display.find_dpll(limit, crtc_state,
7809                                                  crtc_state->port_clock,
7810                                                  refclk, NULL, &clock);
7811                 if (!ok) {
7812                         DRM_ERROR("Couldn't find PLL settings for mode!\n");
7813                         return -EINVAL;
7814                 }
7815
7816                 /* Compat-code for transition, will disappear. */
7817                 crtc_state->dpll.n = clock.n;
7818                 crtc_state->dpll.m1 = clock.m1;
7819                 crtc_state->dpll.m2 = clock.m2;
7820                 crtc_state->dpll.p1 = clock.p1;
7821                 crtc_state->dpll.p2 = clock.p2;
7822         }
7823
7824         if (IS_GEN2(dev)) {
7825                 i8xx_compute_dpll(crtc, crtc_state, NULL,
7826                                   num_connectors);
7827         } else if (IS_CHERRYVIEW(dev)) {
7828                 chv_compute_dpll(crtc, crtc_state);
7829         } else if (IS_VALLEYVIEW(dev)) {
7830                 vlv_compute_dpll(crtc, crtc_state);
7831         } else {
7832                 i9xx_compute_dpll(crtc, crtc_state, NULL,
7833                                   num_connectors);
7834         }
7835
7836         return 0;
7837 }
7838
7839 static void i9xx_get_pfit_config(struct intel_crtc *crtc,
7840                                  struct intel_crtc_state *pipe_config)
7841 {
7842         struct drm_device *dev = crtc->base.dev;
7843         struct drm_i915_private *dev_priv = dev->dev_private;
7844         uint32_t tmp;
7845
7846         if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev)))
7847                 return;
7848
7849         tmp = I915_READ(PFIT_CONTROL);
7850         if (!(tmp & PFIT_ENABLE))
7851                 return;
7852
7853         /* Check whether the pfit is attached to our pipe. */
7854         if (INTEL_INFO(dev)->gen < 4) {
7855                 if (crtc->pipe != PIPE_B)
7856                         return;
7857         } else {
7858                 if ((tmp & PFIT_PIPE_MASK) != (crtc->pipe << PFIT_PIPE_SHIFT))
7859                         return;
7860         }
7861
7862         pipe_config->gmch_pfit.control = tmp;
7863         pipe_config->gmch_pfit.pgm_ratios = I915_READ(PFIT_PGM_RATIOS);
7864         if (INTEL_INFO(dev)->gen < 5)
7865                 pipe_config->gmch_pfit.lvds_border_bits =
7866                         I915_READ(LVDS) & LVDS_BORDER_ENABLE;
7867 }
7868
7869 static void vlv_crtc_clock_get(struct intel_crtc *crtc,
7870                                struct intel_crtc_state *pipe_config)
7871 {
7872         struct drm_device *dev = crtc->base.dev;
7873         struct drm_i915_private *dev_priv = dev->dev_private;
7874         int pipe = pipe_config->cpu_transcoder;
7875         intel_clock_t clock;
7876         u32 mdiv;
7877         int refclk = 100000;
7878
7879         /* In case of MIPI DPLL will not even be used */
7880         if (!(pipe_config->dpll_hw_state.dpll & DPLL_VCO_ENABLE))
7881                 return;
7882
7883         mutex_lock(&dev_priv->sb_lock);
7884         mdiv = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW3(pipe));
7885         mutex_unlock(&dev_priv->sb_lock);
7886
7887         clock.m1 = (mdiv >> DPIO_M1DIV_SHIFT) & 7;
7888         clock.m2 = mdiv & DPIO_M2DIV_MASK;
7889         clock.n = (mdiv >> DPIO_N_SHIFT) & 0xf;
7890         clock.p1 = (mdiv >> DPIO_P1_SHIFT) & 7;
7891         clock.p2 = (mdiv >> DPIO_P2_SHIFT) & 0x1f;
7892
7893         vlv_clock(refclk, &clock);
7894
7895         /* clock.dot is the fast clock */
7896         pipe_config->port_clock = clock.dot / 5;
7897 }
7898
7899 static void
7900 i9xx_get_initial_plane_config(struct intel_crtc *crtc,
7901                               struct intel_initial_plane_config *plane_config)
7902 {
7903         struct drm_device *dev = crtc->base.dev;
7904         struct drm_i915_private *dev_priv = dev->dev_private;
7905         u32 val, base, offset;
7906         int pipe = crtc->pipe, plane = crtc->plane;
7907         int fourcc, pixel_format;
7908         unsigned int aligned_height;
7909         struct drm_framebuffer *fb;
7910         struct intel_framebuffer *intel_fb;
7911
7912         val = I915_READ(DSPCNTR(plane));
7913         if (!(val & DISPLAY_PLANE_ENABLE))
7914                 return;
7915
7916         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
7917         if (!intel_fb) {
7918                 DRM_DEBUG_KMS("failed to alloc fb\n");
7919                 return;
7920         }
7921
7922         fb = &intel_fb->base;
7923
7924         if (INTEL_INFO(dev)->gen >= 4) {
7925                 if (val & DISPPLANE_TILED) {
7926                         plane_config->tiling = I915_TILING_X;
7927                         fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
7928                 }
7929         }
7930
7931         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
7932         fourcc = i9xx_format_to_fourcc(pixel_format);
7933         fb->pixel_format = fourcc;
7934         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
7935
7936         if (INTEL_INFO(dev)->gen >= 4) {
7937                 if (plane_config->tiling)
7938                         offset = I915_READ(DSPTILEOFF(plane));
7939                 else
7940                         offset = I915_READ(DSPLINOFF(plane));
7941                 base = I915_READ(DSPSURF(plane)) & 0xfffff000;
7942         } else {
7943                 base = I915_READ(DSPADDR(plane));
7944         }
7945         plane_config->base = base;
7946
7947         val = I915_READ(PIPESRC(pipe));
7948         fb->width = ((val >> 16) & 0xfff) + 1;
7949         fb->height = ((val >> 0) & 0xfff) + 1;
7950
7951         val = I915_READ(DSPSTRIDE(pipe));
7952         fb->pitches[0] = val & 0xffffffc0;
7953
7954         aligned_height = intel_fb_align_height(dev, fb->height,
7955                                                fb->pixel_format,
7956                                                fb->modifier[0]);
7957
7958         plane_config->size = fb->pitches[0] * aligned_height;
7959
7960         DRM_DEBUG_KMS("pipe/plane %c/%d with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
7961                       pipe_name(pipe), plane, fb->width, fb->height,
7962                       fb->bits_per_pixel, base, fb->pitches[0],
7963                       plane_config->size);
7964
7965         plane_config->fb = intel_fb;
7966 }
7967
7968 static void chv_crtc_clock_get(struct intel_crtc *crtc,
7969                                struct intel_crtc_state *pipe_config)
7970 {
7971         struct drm_device *dev = crtc->base.dev;
7972         struct drm_i915_private *dev_priv = dev->dev_private;
7973         int pipe = pipe_config->cpu_transcoder;
7974         enum dpio_channel port = vlv_pipe_to_channel(pipe);
7975         intel_clock_t clock;
7976         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2;
7977         int refclk = 100000;
7978
7979         mutex_lock(&dev_priv->sb_lock);
7980         cmn_dw13 = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW13(port));
7981         pll_dw0 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW0(port));
7982         pll_dw1 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW1(port));
7983         pll_dw2 = vlv_dpio_read(dev_priv, pipe, CHV_PLL_DW2(port));
7984         mutex_unlock(&dev_priv->sb_lock);
7985
7986         clock.m1 = (pll_dw1 & 0x7) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
7987         clock.m2 = ((pll_dw0 & 0xff) << 22) | (pll_dw2 & 0x3fffff);
7988         clock.n = (pll_dw1 >> DPIO_CHV_N_DIV_SHIFT) & 0xf;
7989         clock.p1 = (cmn_dw13 >> DPIO_CHV_P1_DIV_SHIFT) & 0x7;
7990         clock.p2 = (cmn_dw13 >> DPIO_CHV_P2_DIV_SHIFT) & 0x1f;
7991
7992         chv_clock(refclk, &clock);
7993
7994         /* clock.dot is the fast clock */
7995         pipe_config->port_clock = clock.dot / 5;
7996 }
7997
7998 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
7999                                  struct intel_crtc_state *pipe_config)
8000 {
8001         struct drm_device *dev = crtc->base.dev;
8002         struct drm_i915_private *dev_priv = dev->dev_private;
8003         uint32_t tmp;
8004
8005         if (!intel_display_power_is_enabled(dev_priv,
8006                                             POWER_DOMAIN_PIPE(crtc->pipe)))
8007                 return false;
8008
8009         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
8010         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
8011
8012         tmp = I915_READ(PIPECONF(crtc->pipe));
8013         if (!(tmp & PIPECONF_ENABLE))
8014                 return false;
8015
8016         if (IS_G4X(dev) || IS_VALLEYVIEW(dev)) {
8017                 switch (tmp & PIPECONF_BPC_MASK) {
8018                 case PIPECONF_6BPC:
8019                         pipe_config->pipe_bpp = 18;
8020                         break;
8021                 case PIPECONF_8BPC:
8022                         pipe_config->pipe_bpp = 24;
8023                         break;
8024                 case PIPECONF_10BPC:
8025                         pipe_config->pipe_bpp = 30;
8026                         break;
8027                 default:
8028                         break;
8029                 }
8030         }
8031
8032         if (IS_VALLEYVIEW(dev) && (tmp & PIPECONF_COLOR_RANGE_SELECT))
8033                 pipe_config->limited_color_range = true;
8034
8035         if (INTEL_INFO(dev)->gen < 4)
8036                 pipe_config->double_wide = tmp & PIPECONF_DOUBLE_WIDE;
8037
8038         intel_get_pipe_timings(crtc, pipe_config);
8039
8040         i9xx_get_pfit_config(crtc, pipe_config);
8041
8042         if (INTEL_INFO(dev)->gen >= 4) {
8043                 tmp = I915_READ(DPLL_MD(crtc->pipe));
8044                 pipe_config->pixel_multiplier =
8045                         ((tmp & DPLL_MD_UDI_MULTIPLIER_MASK)
8046                          >> DPLL_MD_UDI_MULTIPLIER_SHIFT) + 1;
8047                 pipe_config->dpll_hw_state.dpll_md = tmp;
8048         } else if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) {
8049                 tmp = I915_READ(DPLL(crtc->pipe));
8050                 pipe_config->pixel_multiplier =
8051                         ((tmp & SDVO_MULTIPLIER_MASK)
8052                          >> SDVO_MULTIPLIER_SHIFT_HIRES) + 1;
8053         } else {
8054                 /* Note that on i915G/GM the pixel multiplier is in the sdvo
8055                  * port and will be fixed up in the encoder->get_config
8056                  * function. */
8057                 pipe_config->pixel_multiplier = 1;
8058         }
8059         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
8060         if (!IS_VALLEYVIEW(dev)) {
8061                 /*
8062                  * DPLL_DVO_2X_MODE must be enabled for both DPLLs
8063                  * on 830. Filter it out here so that we don't
8064                  * report errors due to that.
8065                  */
8066                 if (IS_I830(dev))
8067                         pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
8068
8069                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
8070                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
8071         } else {
8072                 /* Mask out read-only status bits. */
8073                 pipe_config->dpll_hw_state.dpll &= ~(DPLL_LOCK_VLV |
8074                                                      DPLL_PORTC_READY_MASK |
8075                                                      DPLL_PORTB_READY_MASK);
8076         }
8077
8078         if (IS_CHERRYVIEW(dev))
8079                 chv_crtc_clock_get(crtc, pipe_config);
8080         else if (IS_VALLEYVIEW(dev))
8081                 vlv_crtc_clock_get(crtc, pipe_config);
8082         else
8083                 i9xx_crtc_clock_get(crtc, pipe_config);
8084
8085         return true;
8086 }
8087
8088 static void ironlake_init_pch_refclk(struct drm_device *dev)
8089 {
8090         struct drm_i915_private *dev_priv = dev->dev_private;
8091         struct intel_encoder *encoder;
8092         u32 val, final;
8093         bool has_lvds = false;
8094         bool has_cpu_edp = false;
8095         bool has_panel = false;
8096         bool has_ck505 = false;
8097         bool can_ssc = false;
8098
8099         /* We need to take the global config into account */
8100         for_each_intel_encoder(dev, encoder) {
8101                 switch (encoder->type) {
8102                 case INTEL_OUTPUT_LVDS:
8103                         has_panel = true;
8104                         has_lvds = true;
8105                         break;
8106                 case INTEL_OUTPUT_EDP:
8107                         has_panel = true;
8108                         if (enc_to_dig_port(&encoder->base)->port == PORT_A)
8109                                 has_cpu_edp = true;
8110                         break;
8111                 default:
8112                         break;
8113                 }
8114         }
8115
8116         if (HAS_PCH_IBX(dev)) {
8117                 has_ck505 = dev_priv->vbt.display_clock_mode;
8118                 can_ssc = has_ck505;
8119         } else {
8120                 has_ck505 = false;
8121                 can_ssc = true;
8122         }
8123
8124         DRM_DEBUG_KMS("has_panel %d has_lvds %d has_ck505 %d\n",
8125                       has_panel, has_lvds, has_ck505);
8126
8127         /* Ironlake: try to setup display ref clock before DPLL
8128          * enabling. This is only under driver's control after
8129          * PCH B stepping, previous chipset stepping should be
8130          * ignoring this setting.
8131          */
8132         val = I915_READ(PCH_DREF_CONTROL);
8133
8134         /* As we must carefully and slowly disable/enable each source in turn,
8135          * compute the final state we want first and check if we need to
8136          * make any changes at all.
8137          */
8138         final = val;
8139         final &= ~DREF_NONSPREAD_SOURCE_MASK;
8140         if (has_ck505)
8141                 final |= DREF_NONSPREAD_CK505_ENABLE;
8142         else
8143                 final |= DREF_NONSPREAD_SOURCE_ENABLE;
8144
8145         final &= ~DREF_SSC_SOURCE_MASK;
8146         final &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8147         final &= ~DREF_SSC1_ENABLE;
8148
8149         if (has_panel) {
8150                 final |= DREF_SSC_SOURCE_ENABLE;
8151
8152                 if (intel_panel_use_ssc(dev_priv) && can_ssc)
8153                         final |= DREF_SSC1_ENABLE;
8154
8155                 if (has_cpu_edp) {
8156                         if (intel_panel_use_ssc(dev_priv) && can_ssc)
8157                                 final |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8158                         else
8159                                 final |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8160                 } else
8161                         final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8162         } else {
8163                 final |= DREF_SSC_SOURCE_DISABLE;
8164                 final |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8165         }
8166
8167         if (final == val)
8168                 return;
8169
8170         /* Always enable nonspread source */
8171         val &= ~DREF_NONSPREAD_SOURCE_MASK;
8172
8173         if (has_ck505)
8174                 val |= DREF_NONSPREAD_CK505_ENABLE;
8175         else
8176                 val |= DREF_NONSPREAD_SOURCE_ENABLE;
8177
8178         if (has_panel) {
8179                 val &= ~DREF_SSC_SOURCE_MASK;
8180                 val |= DREF_SSC_SOURCE_ENABLE;
8181
8182                 /* SSC must be turned on before enabling the CPU output  */
8183                 if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8184                         DRM_DEBUG_KMS("Using SSC on panel\n");
8185                         val |= DREF_SSC1_ENABLE;
8186                 } else
8187                         val &= ~DREF_SSC1_ENABLE;
8188
8189                 /* Get SSC going before enabling the outputs */
8190                 I915_WRITE(PCH_DREF_CONTROL, val);
8191                 POSTING_READ(PCH_DREF_CONTROL);
8192                 udelay(200);
8193
8194                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8195
8196                 /* Enable CPU source on CPU attached eDP */
8197                 if (has_cpu_edp) {
8198                         if (intel_panel_use_ssc(dev_priv) && can_ssc) {
8199                                 DRM_DEBUG_KMS("Using SSC on eDP\n");
8200                                 val |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD;
8201                         } else
8202                                 val |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD;
8203                 } else
8204                         val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8205
8206                 I915_WRITE(PCH_DREF_CONTROL, val);
8207                 POSTING_READ(PCH_DREF_CONTROL);
8208                 udelay(200);
8209         } else {
8210                 DRM_DEBUG_KMS("Disabling SSC entirely\n");
8211
8212                 val &= ~DREF_CPU_SOURCE_OUTPUT_MASK;
8213
8214                 /* Turn off CPU output */
8215                 val |= DREF_CPU_SOURCE_OUTPUT_DISABLE;
8216
8217                 I915_WRITE(PCH_DREF_CONTROL, val);
8218                 POSTING_READ(PCH_DREF_CONTROL);
8219                 udelay(200);
8220
8221                 /* Turn off the SSC source */
8222                 val &= ~DREF_SSC_SOURCE_MASK;
8223                 val |= DREF_SSC_SOURCE_DISABLE;
8224
8225                 /* Turn off SSC1 */
8226                 val &= ~DREF_SSC1_ENABLE;
8227
8228                 I915_WRITE(PCH_DREF_CONTROL, val);
8229                 POSTING_READ(PCH_DREF_CONTROL);
8230                 udelay(200);
8231         }
8232
8233         BUG_ON(val != final);
8234 }
8235
8236 static void lpt_reset_fdi_mphy(struct drm_i915_private *dev_priv)
8237 {
8238         uint32_t tmp;
8239
8240         tmp = I915_READ(SOUTH_CHICKEN2);
8241         tmp |= FDI_MPHY_IOSFSB_RESET_CTL;
8242         I915_WRITE(SOUTH_CHICKEN2, tmp);
8243
8244         if (wait_for_atomic_us(I915_READ(SOUTH_CHICKEN2) &
8245                                FDI_MPHY_IOSFSB_RESET_STATUS, 100))
8246                 DRM_ERROR("FDI mPHY reset assert timeout\n");
8247
8248         tmp = I915_READ(SOUTH_CHICKEN2);
8249         tmp &= ~FDI_MPHY_IOSFSB_RESET_CTL;
8250         I915_WRITE(SOUTH_CHICKEN2, tmp);
8251
8252         if (wait_for_atomic_us((I915_READ(SOUTH_CHICKEN2) &
8253                                 FDI_MPHY_IOSFSB_RESET_STATUS) == 0, 100))
8254                 DRM_ERROR("FDI mPHY reset de-assert timeout\n");
8255 }
8256
8257 /* WaMPhyProgramming:hsw */
8258 static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
8259 {
8260         uint32_t tmp;
8261
8262         tmp = intel_sbi_read(dev_priv, 0x8008, SBI_MPHY);
8263         tmp &= ~(0xFF << 24);
8264         tmp |= (0x12 << 24);
8265         intel_sbi_write(dev_priv, 0x8008, tmp, SBI_MPHY);
8266
8267         tmp = intel_sbi_read(dev_priv, 0x2008, SBI_MPHY);
8268         tmp |= (1 << 11);
8269         intel_sbi_write(dev_priv, 0x2008, tmp, SBI_MPHY);
8270
8271         tmp = intel_sbi_read(dev_priv, 0x2108, SBI_MPHY);
8272         tmp |= (1 << 11);
8273         intel_sbi_write(dev_priv, 0x2108, tmp, SBI_MPHY);
8274
8275         tmp = intel_sbi_read(dev_priv, 0x206C, SBI_MPHY);
8276         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8277         intel_sbi_write(dev_priv, 0x206C, tmp, SBI_MPHY);
8278
8279         tmp = intel_sbi_read(dev_priv, 0x216C, SBI_MPHY);
8280         tmp |= (1 << 24) | (1 << 21) | (1 << 18);
8281         intel_sbi_write(dev_priv, 0x216C, tmp, SBI_MPHY);
8282
8283         tmp = intel_sbi_read(dev_priv, 0x2080, SBI_MPHY);
8284         tmp &= ~(7 << 13);
8285         tmp |= (5 << 13);
8286         intel_sbi_write(dev_priv, 0x2080, tmp, SBI_MPHY);
8287
8288         tmp = intel_sbi_read(dev_priv, 0x2180, SBI_MPHY);
8289         tmp &= ~(7 << 13);
8290         tmp |= (5 << 13);
8291         intel_sbi_write(dev_priv, 0x2180, tmp, SBI_MPHY);
8292
8293         tmp = intel_sbi_read(dev_priv, 0x208C, SBI_MPHY);
8294         tmp &= ~0xFF;
8295         tmp |= 0x1C;
8296         intel_sbi_write(dev_priv, 0x208C, tmp, SBI_MPHY);
8297
8298         tmp = intel_sbi_read(dev_priv, 0x218C, SBI_MPHY);
8299         tmp &= ~0xFF;
8300         tmp |= 0x1C;
8301         intel_sbi_write(dev_priv, 0x218C, tmp, SBI_MPHY);
8302
8303         tmp = intel_sbi_read(dev_priv, 0x2098, SBI_MPHY);
8304         tmp &= ~(0xFF << 16);
8305         tmp |= (0x1C << 16);
8306         intel_sbi_write(dev_priv, 0x2098, tmp, SBI_MPHY);
8307
8308         tmp = intel_sbi_read(dev_priv, 0x2198, SBI_MPHY);
8309         tmp &= ~(0xFF << 16);
8310         tmp |= (0x1C << 16);
8311         intel_sbi_write(dev_priv, 0x2198, tmp, SBI_MPHY);
8312
8313         tmp = intel_sbi_read(dev_priv, 0x20C4, SBI_MPHY);
8314         tmp |= (1 << 27);
8315         intel_sbi_write(dev_priv, 0x20C4, tmp, SBI_MPHY);
8316
8317         tmp = intel_sbi_read(dev_priv, 0x21C4, SBI_MPHY);
8318         tmp |= (1 << 27);
8319         intel_sbi_write(dev_priv, 0x21C4, tmp, SBI_MPHY);
8320
8321         tmp = intel_sbi_read(dev_priv, 0x20EC, SBI_MPHY);
8322         tmp &= ~(0xF << 28);
8323         tmp |= (4 << 28);
8324         intel_sbi_write(dev_priv, 0x20EC, tmp, SBI_MPHY);
8325
8326         tmp = intel_sbi_read(dev_priv, 0x21EC, SBI_MPHY);
8327         tmp &= ~(0xF << 28);
8328         tmp |= (4 << 28);
8329         intel_sbi_write(dev_priv, 0x21EC, tmp, SBI_MPHY);
8330 }
8331
8332 /* Implements 3 different sequences from BSpec chapter "Display iCLK
8333  * Programming" based on the parameters passed:
8334  * - Sequence to enable CLKOUT_DP
8335  * - Sequence to enable CLKOUT_DP without spread
8336  * - Sequence to enable CLKOUT_DP for FDI usage and configure PCH FDI I/O
8337  */
8338 static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
8339                                  bool with_fdi)
8340 {
8341         struct drm_i915_private *dev_priv = dev->dev_private;
8342         uint32_t reg, tmp;
8343
8344         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
8345                 with_spread = true;
8346         if (WARN(dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE &&
8347                  with_fdi, "LP PCH doesn't have FDI\n"))
8348                 with_fdi = false;
8349
8350         mutex_lock(&dev_priv->sb_lock);
8351
8352         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8353         tmp &= ~SBI_SSCCTL_DISABLE;
8354         tmp |= SBI_SSCCTL_PATHALT;
8355         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8356
8357         udelay(24);
8358
8359         if (with_spread) {
8360                 tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8361                 tmp &= ~SBI_SSCCTL_PATHALT;
8362                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8363
8364                 if (with_fdi) {
8365                         lpt_reset_fdi_mphy(dev_priv);
8366                         lpt_program_fdi_mphy(dev_priv);
8367                 }
8368         }
8369
8370         reg = (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) ?
8371                SBI_GEN0 : SBI_DBUFF0;
8372         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8373         tmp |= SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8374         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8375
8376         mutex_unlock(&dev_priv->sb_lock);
8377 }
8378
8379 /* Sequence to disable CLKOUT_DP */
8380 static void lpt_disable_clkout_dp(struct drm_device *dev)
8381 {
8382         struct drm_i915_private *dev_priv = dev->dev_private;
8383         uint32_t reg, tmp;
8384
8385         mutex_lock(&dev_priv->sb_lock);
8386
8387         reg = (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) ?
8388                SBI_GEN0 : SBI_DBUFF0;
8389         tmp = intel_sbi_read(dev_priv, reg, SBI_ICLK);
8390         tmp &= ~SBI_GEN0_CFG_BUFFENABLE_DISABLE;
8391         intel_sbi_write(dev_priv, reg, tmp, SBI_ICLK);
8392
8393         tmp = intel_sbi_read(dev_priv, SBI_SSCCTL, SBI_ICLK);
8394         if (!(tmp & SBI_SSCCTL_DISABLE)) {
8395                 if (!(tmp & SBI_SSCCTL_PATHALT)) {
8396                         tmp |= SBI_SSCCTL_PATHALT;
8397                         intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8398                         udelay(32);
8399                 }
8400                 tmp |= SBI_SSCCTL_DISABLE;
8401                 intel_sbi_write(dev_priv, SBI_SSCCTL, tmp, SBI_ICLK);
8402         }
8403
8404         mutex_unlock(&dev_priv->sb_lock);
8405 }
8406
8407 static void lpt_init_pch_refclk(struct drm_device *dev)
8408 {
8409         struct intel_encoder *encoder;
8410         bool has_vga = false;
8411
8412         for_each_intel_encoder(dev, encoder) {
8413                 switch (encoder->type) {
8414                 case INTEL_OUTPUT_ANALOG:
8415                         has_vga = true;
8416                         break;
8417                 default:
8418                         break;
8419                 }
8420         }
8421
8422         if (has_vga)
8423                 lpt_enable_clkout_dp(dev, true, true);
8424         else
8425                 lpt_disable_clkout_dp(dev);
8426 }
8427
8428 /*
8429  * Initialize reference clocks when the driver loads
8430  */
8431 void intel_init_pch_refclk(struct drm_device *dev)
8432 {
8433         if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
8434                 ironlake_init_pch_refclk(dev);
8435         else if (HAS_PCH_LPT(dev))
8436                 lpt_init_pch_refclk(dev);
8437 }
8438
8439 static int ironlake_get_refclk(struct intel_crtc_state *crtc_state)
8440 {
8441         struct drm_device *dev = crtc_state->base.crtc->dev;
8442         struct drm_i915_private *dev_priv = dev->dev_private;
8443         struct drm_atomic_state *state = crtc_state->base.state;
8444         struct drm_connector *connector;
8445         struct drm_connector_state *connector_state;
8446         struct intel_encoder *encoder;
8447         int num_connectors = 0, i;
8448         bool is_lvds = false;
8449
8450         for_each_connector_in_state(state, connector, connector_state, i) {
8451                 if (connector_state->crtc != crtc_state->base.crtc)
8452                         continue;
8453
8454                 encoder = to_intel_encoder(connector_state->best_encoder);
8455
8456                 switch (encoder->type) {
8457                 case INTEL_OUTPUT_LVDS:
8458                         is_lvds = true;
8459                         break;
8460                 default:
8461                         break;
8462                 }
8463                 num_connectors++;
8464         }
8465
8466         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) {
8467                 DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
8468                               dev_priv->vbt.lvds_ssc_freq);
8469                 return dev_priv->vbt.lvds_ssc_freq;
8470         }
8471
8472         return 120000;
8473 }
8474
8475 static void ironlake_set_pipeconf(struct drm_crtc *crtc)
8476 {
8477         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
8478         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8479         int pipe = intel_crtc->pipe;
8480         uint32_t val;
8481
8482         val = 0;
8483
8484         switch (intel_crtc->config->pipe_bpp) {
8485         case 18:
8486                 val |= PIPECONF_6BPC;
8487                 break;
8488         case 24:
8489                 val |= PIPECONF_8BPC;
8490                 break;
8491         case 30:
8492                 val |= PIPECONF_10BPC;
8493                 break;
8494         case 36:
8495                 val |= PIPECONF_12BPC;
8496                 break;
8497         default:
8498                 /* Case prevented by intel_choose_pipe_bpp_dither. */
8499                 BUG();
8500         }
8501
8502         if (intel_crtc->config->dither)
8503                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8504
8505         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8506                 val |= PIPECONF_INTERLACED_ILK;
8507         else
8508                 val |= PIPECONF_PROGRESSIVE;
8509
8510         if (intel_crtc->config->limited_color_range)
8511                 val |= PIPECONF_COLOR_RANGE_SELECT;
8512
8513         I915_WRITE(PIPECONF(pipe), val);
8514         POSTING_READ(PIPECONF(pipe));
8515 }
8516
8517 /*
8518  * Set up the pipe CSC unit.
8519  *
8520  * Currently only full range RGB to limited range RGB conversion
8521  * is supported, but eventually this should handle various
8522  * RGB<->YCbCr scenarios as well.
8523  */
8524 static void intel_set_pipe_csc(struct drm_crtc *crtc)
8525 {
8526         struct drm_device *dev = crtc->dev;
8527         struct drm_i915_private *dev_priv = dev->dev_private;
8528         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8529         int pipe = intel_crtc->pipe;
8530         uint16_t coeff = 0x7800; /* 1.0 */
8531
8532         /*
8533          * TODO: Check what kind of values actually come out of the pipe
8534          * with these coeff/postoff values and adjust to get the best
8535          * accuracy. Perhaps we even need to take the bpc value into
8536          * consideration.
8537          */
8538
8539         if (intel_crtc->config->limited_color_range)
8540                 coeff = ((235 - 16) * (1 << 12) / 255) & 0xff8; /* 0.xxx... */
8541
8542         /*
8543          * GY/GU and RY/RU should be the other way around according
8544          * to BSpec, but reality doesn't agree. Just set them up in
8545          * a way that results in the correct picture.
8546          */
8547         I915_WRITE(PIPE_CSC_COEFF_RY_GY(pipe), coeff << 16);
8548         I915_WRITE(PIPE_CSC_COEFF_BY(pipe), 0);
8549
8550         I915_WRITE(PIPE_CSC_COEFF_RU_GU(pipe), coeff);
8551         I915_WRITE(PIPE_CSC_COEFF_BU(pipe), 0);
8552
8553         I915_WRITE(PIPE_CSC_COEFF_RV_GV(pipe), 0);
8554         I915_WRITE(PIPE_CSC_COEFF_BV(pipe), coeff << 16);
8555
8556         I915_WRITE(PIPE_CSC_PREOFF_HI(pipe), 0);
8557         I915_WRITE(PIPE_CSC_PREOFF_ME(pipe), 0);
8558         I915_WRITE(PIPE_CSC_PREOFF_LO(pipe), 0);
8559
8560         if (INTEL_INFO(dev)->gen > 6) {
8561                 uint16_t postoff = 0;
8562
8563                 if (intel_crtc->config->limited_color_range)
8564                         postoff = (16 * (1 << 12) / 255) & 0x1fff;
8565
8566                 I915_WRITE(PIPE_CSC_POSTOFF_HI(pipe), postoff);
8567                 I915_WRITE(PIPE_CSC_POSTOFF_ME(pipe), postoff);
8568                 I915_WRITE(PIPE_CSC_POSTOFF_LO(pipe), postoff);
8569
8570                 I915_WRITE(PIPE_CSC_MODE(pipe), 0);
8571         } else {
8572                 uint32_t mode = CSC_MODE_YUV_TO_RGB;
8573
8574                 if (intel_crtc->config->limited_color_range)
8575                         mode |= CSC_BLACK_SCREEN_OFFSET;
8576
8577                 I915_WRITE(PIPE_CSC_MODE(pipe), mode);
8578         }
8579 }
8580
8581 static void haswell_set_pipeconf(struct drm_crtc *crtc)
8582 {
8583         struct drm_device *dev = crtc->dev;
8584         struct drm_i915_private *dev_priv = dev->dev_private;
8585         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
8586         enum pipe pipe = intel_crtc->pipe;
8587         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
8588         uint32_t val;
8589
8590         val = 0;
8591
8592         if (IS_HASWELL(dev) && intel_crtc->config->dither)
8593                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
8594
8595         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
8596                 val |= PIPECONF_INTERLACED_ILK;
8597         else
8598                 val |= PIPECONF_PROGRESSIVE;
8599
8600         I915_WRITE(PIPECONF(cpu_transcoder), val);
8601         POSTING_READ(PIPECONF(cpu_transcoder));
8602
8603         I915_WRITE(GAMMA_MODE(intel_crtc->pipe), GAMMA_MODE_MODE_8BIT);
8604         POSTING_READ(GAMMA_MODE(intel_crtc->pipe));
8605
8606         if (IS_BROADWELL(dev) || INTEL_INFO(dev)->gen >= 9) {
8607                 val = 0;
8608
8609                 switch (intel_crtc->config->pipe_bpp) {
8610                 case 18:
8611                         val |= PIPEMISC_DITHER_6_BPC;
8612                         break;
8613                 case 24:
8614                         val |= PIPEMISC_DITHER_8_BPC;
8615                         break;
8616                 case 30:
8617                         val |= PIPEMISC_DITHER_10_BPC;
8618                         break;
8619                 case 36:
8620                         val |= PIPEMISC_DITHER_12_BPC;
8621                         break;
8622                 default:
8623                         /* Case prevented by pipe_config_set_bpp. */
8624                         BUG();
8625                 }
8626
8627                 if (intel_crtc->config->dither)
8628                         val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
8629
8630                 I915_WRITE(PIPEMISC(pipe), val);
8631         }
8632 }
8633
8634 static bool ironlake_compute_clocks(struct drm_crtc *crtc,
8635                                     struct intel_crtc_state *crtc_state,
8636                                     intel_clock_t *clock,
8637                                     bool *has_reduced_clock,
8638                                     intel_clock_t *reduced_clock)
8639 {
8640         struct drm_device *dev = crtc->dev;
8641         struct drm_i915_private *dev_priv = dev->dev_private;
8642         int refclk;
8643         const intel_limit_t *limit;
8644         bool ret;
8645
8646         refclk = ironlake_get_refclk(crtc_state);
8647
8648         /*
8649          * Returns a set of divisors for the desired target clock with the given
8650          * refclk, or FALSE.  The returned values represent the clock equation:
8651          * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
8652          */
8653         limit = intel_limit(crtc_state, refclk);
8654         ret = dev_priv->display.find_dpll(limit, crtc_state,
8655                                           crtc_state->port_clock,
8656                                           refclk, NULL, clock);
8657         if (!ret)
8658                 return false;
8659
8660         return true;
8661 }
8662
8663 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
8664 {
8665         /*
8666          * Account for spread spectrum to avoid
8667          * oversubscribing the link. Max center spread
8668          * is 2.5%; use 5% for safety's sake.
8669          */
8670         u32 bps = target_clock * bpp * 21 / 20;
8671         return DIV_ROUND_UP(bps, link_bw * 8);
8672 }
8673
8674 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
8675 {
8676         return i9xx_dpll_compute_m(dpll) < factor * dpll->n;
8677 }
8678
8679 static uint32_t ironlake_compute_dpll(struct intel_crtc *intel_crtc,
8680                                       struct intel_crtc_state *crtc_state,
8681                                       u32 *fp,
8682                                       intel_clock_t *reduced_clock, u32 *fp2)
8683 {
8684         struct drm_crtc *crtc = &intel_crtc->base;
8685         struct drm_device *dev = crtc->dev;
8686         struct drm_i915_private *dev_priv = dev->dev_private;
8687         struct drm_atomic_state *state = crtc_state->base.state;
8688         struct drm_connector *connector;
8689         struct drm_connector_state *connector_state;
8690         struct intel_encoder *encoder;
8691         uint32_t dpll;
8692         int factor, num_connectors = 0, i;
8693         bool is_lvds = false, is_sdvo = false;
8694
8695         for_each_connector_in_state(state, connector, connector_state, i) {
8696                 if (connector_state->crtc != crtc_state->base.crtc)
8697                         continue;
8698
8699                 encoder = to_intel_encoder(connector_state->best_encoder);
8700
8701                 switch (encoder->type) {
8702                 case INTEL_OUTPUT_LVDS:
8703                         is_lvds = true;
8704                         break;
8705                 case INTEL_OUTPUT_SDVO:
8706                 case INTEL_OUTPUT_HDMI:
8707                         is_sdvo = true;
8708                         break;
8709                 default:
8710                         break;
8711                 }
8712
8713                 num_connectors++;
8714         }
8715
8716         /* Enable autotuning of the PLL clock (if permissible) */
8717         factor = 21;
8718         if (is_lvds) {
8719                 if ((intel_panel_use_ssc(dev_priv) &&
8720                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
8721                     (HAS_PCH_IBX(dev) && intel_is_dual_link_lvds(dev)))
8722                         factor = 25;
8723         } else if (crtc_state->sdvo_tv_clock)
8724                 factor = 20;
8725
8726         if (ironlake_needs_fb_cb_tune(&crtc_state->dpll, factor))
8727                 *fp |= FP_CB_TUNE;
8728
8729         if (fp2 && (reduced_clock->m < factor * reduced_clock->n))
8730                 *fp2 |= FP_CB_TUNE;
8731
8732         dpll = 0;
8733
8734         if (is_lvds)
8735                 dpll |= DPLLB_MODE_LVDS;
8736         else
8737                 dpll |= DPLLB_MODE_DAC_SERIAL;
8738
8739         dpll |= (crtc_state->pixel_multiplier - 1)
8740                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
8741
8742         if (is_sdvo)
8743                 dpll |= DPLL_SDVO_HIGH_SPEED;
8744         if (crtc_state->has_dp_encoder)
8745                 dpll |= DPLL_SDVO_HIGH_SPEED;
8746
8747         /* compute bitmask from p1 value */
8748         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
8749         /* also FPA1 */
8750         dpll |= (1 << (crtc_state->dpll.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
8751
8752         switch (crtc_state->dpll.p2) {
8753         case 5:
8754                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
8755                 break;
8756         case 7:
8757                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
8758                 break;
8759         case 10:
8760                 dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
8761                 break;
8762         case 14:
8763                 dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
8764                 break;
8765         }
8766
8767         if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
8768                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
8769         else
8770                 dpll |= PLL_REF_INPUT_DREFCLK;
8771
8772         return dpll | DPLL_VCO_ENABLE;
8773 }
8774
8775 static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
8776                                        struct intel_crtc_state *crtc_state)
8777 {
8778         struct drm_device *dev = crtc->base.dev;
8779         intel_clock_t clock, reduced_clock;
8780         u32 dpll = 0, fp = 0, fp2 = 0;
8781         bool ok, has_reduced_clock = false;
8782         bool is_lvds = false;
8783         struct intel_shared_dpll *pll;
8784
8785         memset(&crtc_state->dpll_hw_state, 0,
8786                sizeof(crtc_state->dpll_hw_state));
8787
8788         is_lvds = intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS);
8789
8790         WARN(!(HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)),
8791              "Unexpected PCH type %d\n", INTEL_PCH_TYPE(dev));
8792
8793         ok = ironlake_compute_clocks(&crtc->base, crtc_state, &clock,
8794                                      &has_reduced_clock, &reduced_clock);
8795         if (!ok && !crtc_state->clock_set) {
8796                 DRM_ERROR("Couldn't find PLL settings for mode!\n");
8797                 return -EINVAL;
8798         }
8799         /* Compat-code for transition, will disappear. */
8800         if (!crtc_state->clock_set) {
8801                 crtc_state->dpll.n = clock.n;
8802                 crtc_state->dpll.m1 = clock.m1;
8803                 crtc_state->dpll.m2 = clock.m2;
8804                 crtc_state->dpll.p1 = clock.p1;
8805                 crtc_state->dpll.p2 = clock.p2;
8806         }
8807
8808         /* CPU eDP is the only output that doesn't need a PCH PLL of its own. */
8809         if (crtc_state->has_pch_encoder) {
8810                 fp = i9xx_dpll_compute_fp(&crtc_state->dpll);
8811                 if (has_reduced_clock)
8812                         fp2 = i9xx_dpll_compute_fp(&reduced_clock);
8813
8814                 dpll = ironlake_compute_dpll(crtc, crtc_state,
8815                                              &fp, &reduced_clock,
8816                                              has_reduced_clock ? &fp2 : NULL);
8817
8818                 crtc_state->dpll_hw_state.dpll = dpll;
8819                 crtc_state->dpll_hw_state.fp0 = fp;
8820                 if (has_reduced_clock)
8821                         crtc_state->dpll_hw_state.fp1 = fp2;
8822                 else
8823                         crtc_state->dpll_hw_state.fp1 = fp;
8824
8825                 pll = intel_get_shared_dpll(crtc, crtc_state);
8826                 if (pll == NULL) {
8827                         DRM_DEBUG_DRIVER("failed to find PLL for pipe %c\n",
8828                                          pipe_name(crtc->pipe));
8829                         return -EINVAL;
8830                 }
8831         }
8832
8833         if (is_lvds && has_reduced_clock)
8834                 crtc->lowfreq_avail = true;
8835         else
8836                 crtc->lowfreq_avail = false;
8837
8838         return 0;
8839 }
8840
8841 static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
8842                                          struct intel_link_m_n *m_n)
8843 {
8844         struct drm_device *dev = crtc->base.dev;
8845         struct drm_i915_private *dev_priv = dev->dev_private;
8846         enum pipe pipe = crtc->pipe;
8847
8848         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
8849         m_n->link_n = I915_READ(PCH_TRANS_LINK_N1(pipe));
8850         m_n->gmch_m = I915_READ(PCH_TRANS_DATA_M1(pipe))
8851                 & ~TU_SIZE_MASK;
8852         m_n->gmch_n = I915_READ(PCH_TRANS_DATA_N1(pipe));
8853         m_n->tu = ((I915_READ(PCH_TRANS_DATA_M1(pipe))
8854                     & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8855 }
8856
8857 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
8858                                          enum transcoder transcoder,
8859                                          struct intel_link_m_n *m_n,
8860                                          struct intel_link_m_n *m2_n2)
8861 {
8862         struct drm_device *dev = crtc->base.dev;
8863         struct drm_i915_private *dev_priv = dev->dev_private;
8864         enum pipe pipe = crtc->pipe;
8865
8866         if (INTEL_INFO(dev)->gen >= 5) {
8867                 m_n->link_m = I915_READ(PIPE_LINK_M1(transcoder));
8868                 m_n->link_n = I915_READ(PIPE_LINK_N1(transcoder));
8869                 m_n->gmch_m = I915_READ(PIPE_DATA_M1(transcoder))
8870                         & ~TU_SIZE_MASK;
8871                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
8872                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
8873                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8874                 /* Read M2_N2 registers only for gen < 8 (M2_N2 available for
8875                  * gen < 8) and if DRRS is supported (to make sure the
8876                  * registers are not unnecessarily read).
8877                  */
8878                 if (m2_n2 && INTEL_INFO(dev)->gen < 8 &&
8879                         crtc->config->has_drrs) {
8880                         m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
8881                         m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
8882                         m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
8883                                         & ~TU_SIZE_MASK;
8884                         m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
8885                         m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
8886                                         & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8887                 }
8888         } else {
8889                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
8890                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
8891                 m_n->gmch_m = I915_READ(PIPE_DATA_M_G4X(pipe))
8892                         & ~TU_SIZE_MASK;
8893                 m_n->gmch_n = I915_READ(PIPE_DATA_N_G4X(pipe));
8894                 m_n->tu = ((I915_READ(PIPE_DATA_M_G4X(pipe))
8895                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
8896         }
8897 }
8898
8899 void intel_dp_get_m_n(struct intel_crtc *crtc,
8900                       struct intel_crtc_state *pipe_config)
8901 {
8902         if (pipe_config->has_pch_encoder)
8903                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
8904         else
8905                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8906                                              &pipe_config->dp_m_n,
8907                                              &pipe_config->dp_m2_n2);
8908 }
8909
8910 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
8911                                         struct intel_crtc_state *pipe_config)
8912 {
8913         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
8914                                      &pipe_config->fdi_m_n, NULL);
8915 }
8916
8917 static void skylake_get_pfit_config(struct intel_crtc *crtc,
8918                                     struct intel_crtc_state *pipe_config)
8919 {
8920         struct drm_device *dev = crtc->base.dev;
8921         struct drm_i915_private *dev_priv = dev->dev_private;
8922         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
8923         uint32_t ps_ctrl = 0;
8924         int id = -1;
8925         int i;
8926
8927         /* find scaler attached to this pipe */
8928         for (i = 0; i < crtc->num_scalers; i++) {
8929                 ps_ctrl = I915_READ(SKL_PS_CTRL(crtc->pipe, i));
8930                 if (ps_ctrl & PS_SCALER_EN && !(ps_ctrl & PS_PLANE_SEL_MASK)) {
8931                         id = i;
8932                         pipe_config->pch_pfit.enabled = true;
8933                         pipe_config->pch_pfit.pos = I915_READ(SKL_PS_WIN_POS(crtc->pipe, i));
8934                         pipe_config->pch_pfit.size = I915_READ(SKL_PS_WIN_SZ(crtc->pipe, i));
8935                         break;
8936                 }
8937         }
8938
8939         scaler_state->scaler_id = id;
8940         if (id >= 0) {
8941                 scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
8942         } else {
8943                 scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
8944         }
8945 }
8946
8947 static void
8948 skylake_get_initial_plane_config(struct intel_crtc *crtc,
8949                                  struct intel_initial_plane_config *plane_config)
8950 {
8951         struct drm_device *dev = crtc->base.dev;
8952         struct drm_i915_private *dev_priv = dev->dev_private;
8953         u32 val, base, offset, stride_mult, tiling;
8954         int pipe = crtc->pipe;
8955         int fourcc, pixel_format;
8956         unsigned int aligned_height;
8957         struct drm_framebuffer *fb;
8958         struct intel_framebuffer *intel_fb;
8959
8960         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
8961         if (!intel_fb) {
8962                 DRM_DEBUG_KMS("failed to alloc fb\n");
8963                 return;
8964         }
8965
8966         fb = &intel_fb->base;
8967
8968         val = I915_READ(PLANE_CTL(pipe, 0));
8969         if (!(val & PLANE_CTL_ENABLE))
8970                 goto error;
8971
8972         pixel_format = val & PLANE_CTL_FORMAT_MASK;
8973         fourcc = skl_format_to_fourcc(pixel_format,
8974                                       val & PLANE_CTL_ORDER_RGBX,
8975                                       val & PLANE_CTL_ALPHA_MASK);
8976         fb->pixel_format = fourcc;
8977         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
8978
8979         tiling = val & PLANE_CTL_TILED_MASK;
8980         switch (tiling) {
8981         case PLANE_CTL_TILED_LINEAR:
8982                 fb->modifier[0] = DRM_FORMAT_MOD_NONE;
8983                 break;
8984         case PLANE_CTL_TILED_X:
8985                 plane_config->tiling = I915_TILING_X;
8986                 fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
8987                 break;
8988         case PLANE_CTL_TILED_Y:
8989                 fb->modifier[0] = I915_FORMAT_MOD_Y_TILED;
8990                 break;
8991         case PLANE_CTL_TILED_YF:
8992                 fb->modifier[0] = I915_FORMAT_MOD_Yf_TILED;
8993                 break;
8994         default:
8995                 MISSING_CASE(tiling);
8996                 goto error;
8997         }
8998
8999         base = I915_READ(PLANE_SURF(pipe, 0)) & 0xfffff000;
9000         plane_config->base = base;
9001
9002         offset = I915_READ(PLANE_OFFSET(pipe, 0));
9003
9004         val = I915_READ(PLANE_SIZE(pipe, 0));
9005         fb->height = ((val >> 16) & 0xfff) + 1;
9006         fb->width = ((val >> 0) & 0x1fff) + 1;
9007
9008         val = I915_READ(PLANE_STRIDE(pipe, 0));
9009         stride_mult = intel_fb_stride_alignment(dev, fb->modifier[0],
9010                                                 fb->pixel_format);
9011         fb->pitches[0] = (val & 0x3ff) * stride_mult;
9012
9013         aligned_height = intel_fb_align_height(dev, fb->height,
9014                                                fb->pixel_format,
9015                                                fb->modifier[0]);
9016
9017         plane_config->size = fb->pitches[0] * aligned_height;
9018
9019         DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9020                       pipe_name(pipe), fb->width, fb->height,
9021                       fb->bits_per_pixel, base, fb->pitches[0],
9022                       plane_config->size);
9023
9024         plane_config->fb = intel_fb;
9025         return;
9026
9027 error:
9028         kfree(fb);
9029 }
9030
9031 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
9032                                      struct intel_crtc_state *pipe_config)
9033 {
9034         struct drm_device *dev = crtc->base.dev;
9035         struct drm_i915_private *dev_priv = dev->dev_private;
9036         uint32_t tmp;
9037
9038         tmp = I915_READ(PF_CTL(crtc->pipe));
9039
9040         if (tmp & PF_ENABLE) {
9041                 pipe_config->pch_pfit.enabled = true;
9042                 pipe_config->pch_pfit.pos = I915_READ(PF_WIN_POS(crtc->pipe));
9043                 pipe_config->pch_pfit.size = I915_READ(PF_WIN_SZ(crtc->pipe));
9044
9045                 /* We currently do not free assignements of panel fitters on
9046                  * ivb/hsw (since we don't use the higher upscaling modes which
9047                  * differentiates them) so just WARN about this case for now. */
9048                 if (IS_GEN7(dev)) {
9049                         WARN_ON((tmp & PF_PIPE_SEL_MASK_IVB) !=
9050                                 PF_PIPE_SEL_IVB(crtc->pipe));
9051                 }
9052         }
9053 }
9054
9055 static void
9056 ironlake_get_initial_plane_config(struct intel_crtc *crtc,
9057                                   struct intel_initial_plane_config *plane_config)
9058 {
9059         struct drm_device *dev = crtc->base.dev;
9060         struct drm_i915_private *dev_priv = dev->dev_private;
9061         u32 val, base, offset;
9062         int pipe = crtc->pipe;
9063         int fourcc, pixel_format;
9064         unsigned int aligned_height;
9065         struct drm_framebuffer *fb;
9066         struct intel_framebuffer *intel_fb;
9067
9068         val = I915_READ(DSPCNTR(pipe));
9069         if (!(val & DISPLAY_PLANE_ENABLE))
9070                 return;
9071
9072         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
9073         if (!intel_fb) {
9074                 DRM_DEBUG_KMS("failed to alloc fb\n");
9075                 return;
9076         }
9077
9078         fb = &intel_fb->base;
9079
9080         if (INTEL_INFO(dev)->gen >= 4) {
9081                 if (val & DISPPLANE_TILED) {
9082                         plane_config->tiling = I915_TILING_X;
9083                         fb->modifier[0] = I915_FORMAT_MOD_X_TILED;
9084                 }
9085         }
9086
9087         pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
9088         fourcc = i9xx_format_to_fourcc(pixel_format);
9089         fb->pixel_format = fourcc;
9090         fb->bits_per_pixel = drm_format_plane_cpp(fourcc, 0) * 8;
9091
9092         base = I915_READ(DSPSURF(pipe)) & 0xfffff000;
9093         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
9094                 offset = I915_READ(DSPOFFSET(pipe));
9095         } else {
9096                 if (plane_config->tiling)
9097                         offset = I915_READ(DSPTILEOFF(pipe));
9098                 else
9099                         offset = I915_READ(DSPLINOFF(pipe));
9100         }
9101         plane_config->base = base;
9102
9103         val = I915_READ(PIPESRC(pipe));
9104         fb->width = ((val >> 16) & 0xfff) + 1;
9105         fb->height = ((val >> 0) & 0xfff) + 1;
9106
9107         val = I915_READ(DSPSTRIDE(pipe));
9108         fb->pitches[0] = val & 0xffffffc0;
9109
9110         aligned_height = intel_fb_align_height(dev, fb->height,
9111                                                fb->pixel_format,
9112                                                fb->modifier[0]);
9113
9114         plane_config->size = fb->pitches[0] * aligned_height;
9115
9116         DRM_DEBUG_KMS("pipe %c with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
9117                       pipe_name(pipe), fb->width, fb->height,
9118                       fb->bits_per_pixel, base, fb->pitches[0],
9119                       plane_config->size);
9120
9121         plane_config->fb = intel_fb;
9122 }
9123
9124 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
9125                                      struct intel_crtc_state *pipe_config)
9126 {
9127         struct drm_device *dev = crtc->base.dev;
9128         struct drm_i915_private *dev_priv = dev->dev_private;
9129         uint32_t tmp;
9130
9131         if (!intel_display_power_is_enabled(dev_priv,
9132                                             POWER_DOMAIN_PIPE(crtc->pipe)))
9133                 return false;
9134
9135         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9136         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
9137
9138         tmp = I915_READ(PIPECONF(crtc->pipe));
9139         if (!(tmp & PIPECONF_ENABLE))
9140                 return false;
9141
9142         switch (tmp & PIPECONF_BPC_MASK) {
9143         case PIPECONF_6BPC:
9144                 pipe_config->pipe_bpp = 18;
9145                 break;
9146         case PIPECONF_8BPC:
9147                 pipe_config->pipe_bpp = 24;
9148                 break;
9149         case PIPECONF_10BPC:
9150                 pipe_config->pipe_bpp = 30;
9151                 break;
9152         case PIPECONF_12BPC:
9153                 pipe_config->pipe_bpp = 36;
9154                 break;
9155         default:
9156                 break;
9157         }
9158
9159         if (tmp & PIPECONF_COLOR_RANGE_SELECT)
9160                 pipe_config->limited_color_range = true;
9161
9162         if (I915_READ(PCH_TRANSCONF(crtc->pipe)) & TRANS_ENABLE) {
9163                 struct intel_shared_dpll *pll;
9164
9165                 pipe_config->has_pch_encoder = true;
9166
9167                 tmp = I915_READ(FDI_RX_CTL(crtc->pipe));
9168                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9169                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9170
9171                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9172
9173                 if (HAS_PCH_IBX(dev_priv->dev)) {
9174                         pipe_config->shared_dpll =
9175                                 (enum intel_dpll_id) crtc->pipe;
9176                 } else {
9177                         tmp = I915_READ(PCH_DPLL_SEL);
9178                         if (tmp & TRANS_DPLLB_SEL(crtc->pipe))
9179                                 pipe_config->shared_dpll = DPLL_ID_PCH_PLL_B;
9180                         else
9181                                 pipe_config->shared_dpll = DPLL_ID_PCH_PLL_A;
9182                 }
9183
9184                 pll = &dev_priv->shared_dplls[pipe_config->shared_dpll];
9185
9186                 WARN_ON(!pll->get_hw_state(dev_priv, pll,
9187                                            &pipe_config->dpll_hw_state));
9188
9189                 tmp = pipe_config->dpll_hw_state.dpll;
9190                 pipe_config->pixel_multiplier =
9191                         ((tmp & PLL_REF_SDVO_HDMI_MULTIPLIER_MASK)
9192                          >> PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT) + 1;
9193
9194                 ironlake_pch_clock_get(crtc, pipe_config);
9195         } else {
9196                 pipe_config->pixel_multiplier = 1;
9197         }
9198
9199         intel_get_pipe_timings(crtc, pipe_config);
9200
9201         ironlake_get_pfit_config(crtc, pipe_config);
9202
9203         return true;
9204 }
9205
9206 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
9207 {
9208         struct drm_device *dev = dev_priv->dev;
9209         struct intel_crtc *crtc;
9210
9211         for_each_intel_crtc(dev, crtc)
9212                 I915_STATE_WARN(crtc->active, "CRTC for pipe %c enabled\n",
9213                      pipe_name(crtc->pipe));
9214
9215         I915_STATE_WARN(I915_READ(HSW_PWR_WELL_DRIVER), "Power well on\n");
9216         I915_STATE_WARN(I915_READ(SPLL_CTL) & SPLL_PLL_ENABLE, "SPLL enabled\n");
9217         I915_STATE_WARN(I915_READ(WRPLL_CTL1) & WRPLL_PLL_ENABLE, "WRPLL1 enabled\n");
9218         I915_STATE_WARN(I915_READ(WRPLL_CTL2) & WRPLL_PLL_ENABLE, "WRPLL2 enabled\n");
9219         I915_STATE_WARN(I915_READ(PCH_PP_STATUS) & PP_ON, "Panel power on\n");
9220         I915_STATE_WARN(I915_READ(BLC_PWM_CPU_CTL2) & BLM_PWM_ENABLE,
9221              "CPU PWM1 enabled\n");
9222         if (IS_HASWELL(dev))
9223                 I915_STATE_WARN(I915_READ(HSW_BLC_PWM2_CTL) & BLM_PWM_ENABLE,
9224                      "CPU PWM2 enabled\n");
9225         I915_STATE_WARN(I915_READ(BLC_PWM_PCH_CTL1) & BLM_PCH_PWM_ENABLE,
9226              "PCH PWM1 enabled\n");
9227         I915_STATE_WARN(I915_READ(UTIL_PIN_CTL) & UTIL_PIN_ENABLE,
9228              "Utility pin enabled\n");
9229         I915_STATE_WARN(I915_READ(PCH_GTC_CTL) & PCH_GTC_ENABLE, "PCH GTC enabled\n");
9230
9231         /*
9232          * In theory we can still leave IRQs enabled, as long as only the HPD
9233          * interrupts remain enabled. We used to check for that, but since it's
9234          * gen-specific and since we only disable LCPLL after we fully disable
9235          * the interrupts, the check below should be enough.
9236          */
9237         I915_STATE_WARN(intel_irqs_enabled(dev_priv), "IRQs enabled\n");
9238 }
9239
9240 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
9241 {
9242         struct drm_device *dev = dev_priv->dev;
9243
9244         if (IS_HASWELL(dev))
9245                 return I915_READ(D_COMP_HSW);
9246         else
9247                 return I915_READ(D_COMP_BDW);
9248 }
9249
9250 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
9251 {
9252         struct drm_device *dev = dev_priv->dev;
9253
9254         if (IS_HASWELL(dev)) {
9255                 mutex_lock(&dev_priv->rps.hw_lock);
9256                 if (sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_D_COMP,
9257                                             val))
9258                         DRM_ERROR("Failed to write to D_COMP\n");
9259                 mutex_unlock(&dev_priv->rps.hw_lock);
9260         } else {
9261                 I915_WRITE(D_COMP_BDW, val);
9262                 POSTING_READ(D_COMP_BDW);
9263         }
9264 }
9265
9266 /*
9267  * This function implements pieces of two sequences from BSpec:
9268  * - Sequence for display software to disable LCPLL
9269  * - Sequence for display software to allow package C8+
9270  * The steps implemented here are just the steps that actually touch the LCPLL
9271  * register. Callers should take care of disabling all the display engine
9272  * functions, doing the mode unset, fixing interrupts, etc.
9273  */
9274 static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
9275                               bool switch_to_fclk, bool allow_power_down)
9276 {
9277         uint32_t val;
9278
9279         assert_can_disable_lcpll(dev_priv);
9280
9281         val = I915_READ(LCPLL_CTL);
9282
9283         if (switch_to_fclk) {
9284                 val |= LCPLL_CD_SOURCE_FCLK;
9285                 I915_WRITE(LCPLL_CTL, val);
9286
9287                 if (wait_for_atomic_us(I915_READ(LCPLL_CTL) &
9288                                        LCPLL_CD_SOURCE_FCLK_DONE, 1))
9289                         DRM_ERROR("Switching to FCLK failed\n");
9290
9291                 val = I915_READ(LCPLL_CTL);
9292         }
9293
9294         val |= LCPLL_PLL_DISABLE;
9295         I915_WRITE(LCPLL_CTL, val);
9296         POSTING_READ(LCPLL_CTL);
9297
9298         if (wait_for((I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK) == 0, 1))
9299                 DRM_ERROR("LCPLL still locked\n");
9300
9301         val = hsw_read_dcomp(dev_priv);
9302         val |= D_COMP_COMP_DISABLE;
9303         hsw_write_dcomp(dev_priv, val);
9304         ndelay(100);
9305
9306         if (wait_for((hsw_read_dcomp(dev_priv) & D_COMP_RCOMP_IN_PROGRESS) == 0,
9307                      1))
9308                 DRM_ERROR("D_COMP RCOMP still in progress\n");
9309
9310         if (allow_power_down) {
9311                 val = I915_READ(LCPLL_CTL);
9312                 val |= LCPLL_POWER_DOWN_ALLOW;
9313                 I915_WRITE(LCPLL_CTL, val);
9314                 POSTING_READ(LCPLL_CTL);
9315         }
9316 }
9317
9318 /*
9319  * Fully restores LCPLL, disallowing power down and switching back to LCPLL
9320  * source.
9321  */
9322 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
9323 {
9324         uint32_t val;
9325
9326         val = I915_READ(LCPLL_CTL);
9327
9328         if ((val & (LCPLL_PLL_LOCK | LCPLL_PLL_DISABLE | LCPLL_CD_SOURCE_FCLK |
9329                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
9330                 return;
9331
9332         /*
9333          * Make sure we're not on PC8 state before disabling PC8, otherwise
9334          * we'll hang the machine. To prevent PC8 state, just enable force_wake.
9335          */
9336         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
9337
9338         if (val & LCPLL_POWER_DOWN_ALLOW) {
9339                 val &= ~LCPLL_POWER_DOWN_ALLOW;
9340                 I915_WRITE(LCPLL_CTL, val);
9341                 POSTING_READ(LCPLL_CTL);
9342         }
9343
9344         val = hsw_read_dcomp(dev_priv);
9345         val |= D_COMP_COMP_FORCE;
9346         val &= ~D_COMP_COMP_DISABLE;
9347         hsw_write_dcomp(dev_priv, val);
9348
9349         val = I915_READ(LCPLL_CTL);
9350         val &= ~LCPLL_PLL_DISABLE;
9351         I915_WRITE(LCPLL_CTL, val);
9352
9353         if (wait_for(I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK, 5))
9354                 DRM_ERROR("LCPLL not locked yet\n");
9355
9356         if (val & LCPLL_CD_SOURCE_FCLK) {
9357                 val = I915_READ(LCPLL_CTL);
9358                 val &= ~LCPLL_CD_SOURCE_FCLK;
9359                 I915_WRITE(LCPLL_CTL, val);
9360
9361                 if (wait_for_atomic_us((I915_READ(LCPLL_CTL) &
9362                                         LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9363                         DRM_ERROR("Switching back to LCPLL failed\n");
9364         }
9365
9366         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
9367         intel_update_cdclk(dev_priv->dev);
9368 }
9369
9370 /*
9371  * Package states C8 and deeper are really deep PC states that can only be
9372  * reached when all the devices on the system allow it, so even if the graphics
9373  * device allows PC8+, it doesn't mean the system will actually get to these
9374  * states. Our driver only allows PC8+ when going into runtime PM.
9375  *
9376  * The requirements for PC8+ are that all the outputs are disabled, the power
9377  * well is disabled and most interrupts are disabled, and these are also
9378  * requirements for runtime PM. When these conditions are met, we manually do
9379  * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
9380  * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
9381  * hang the machine.
9382  *
9383  * When we really reach PC8 or deeper states (not just when we allow it) we lose
9384  * the state of some registers, so when we come back from PC8+ we need to
9385  * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
9386  * need to take care of the registers kept by RC6. Notice that this happens even
9387  * if we don't put the device in PCI D3 state (which is what currently happens
9388  * because of the runtime PM support).
9389  *
9390  * For more, read "Display Sequences for Package C8" on the hardware
9391  * documentation.
9392  */
9393 void hsw_enable_pc8(struct drm_i915_private *dev_priv)
9394 {
9395         struct drm_device *dev = dev_priv->dev;
9396         uint32_t val;
9397
9398         DRM_DEBUG_KMS("Enabling package C8+\n");
9399
9400         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
9401                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9402                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
9403                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9404         }
9405
9406         lpt_disable_clkout_dp(dev);
9407         hsw_disable_lcpll(dev_priv, true, true);
9408 }
9409
9410 void hsw_disable_pc8(struct drm_i915_private *dev_priv)
9411 {
9412         struct drm_device *dev = dev_priv->dev;
9413         uint32_t val;
9414
9415         DRM_DEBUG_KMS("Disabling package C8+\n");
9416
9417         hsw_restore_lcpll(dev_priv);
9418         lpt_init_pch_refclk(dev);
9419
9420         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
9421                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
9422                 val |= PCH_LP_PARTITION_LEVEL_DISABLE;
9423                 I915_WRITE(SOUTH_DSPCLK_GATE_D, val);
9424         }
9425
9426         intel_prepare_ddi(dev);
9427 }
9428
9429 static void broxton_modeset_commit_cdclk(struct drm_atomic_state *old_state)
9430 {
9431         struct drm_device *dev = old_state->dev;
9432         unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
9433
9434         broxton_set_cdclk(dev, req_cdclk);
9435 }
9436
9437 /* compute the max rate for new configuration */
9438 static int ilk_max_pixel_rate(struct drm_atomic_state *state)
9439 {
9440         struct intel_crtc *intel_crtc;
9441         struct intel_crtc_state *crtc_state;
9442         int max_pixel_rate = 0;
9443
9444         for_each_intel_crtc(state->dev, intel_crtc) {
9445                 int pixel_rate;
9446
9447                 crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
9448                 if (IS_ERR(crtc_state))
9449                         return PTR_ERR(crtc_state);
9450
9451                 if (!crtc_state->base.enable)
9452                         continue;
9453
9454                 pixel_rate = ilk_pipe_pixel_rate(crtc_state);
9455
9456                 /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
9457                 if (IS_BROADWELL(state->dev) && crtc_state->ips_enabled)
9458                         pixel_rate = DIV_ROUND_UP(pixel_rate * 100, 95);
9459
9460                 max_pixel_rate = max(max_pixel_rate, pixel_rate);
9461         }
9462
9463         return max_pixel_rate;
9464 }
9465
9466 static void broadwell_set_cdclk(struct drm_device *dev, int cdclk)
9467 {
9468         struct drm_i915_private *dev_priv = dev->dev_private;
9469         uint32_t val, data;
9470         int ret;
9471
9472         if (WARN((I915_READ(LCPLL_CTL) &
9473                   (LCPLL_PLL_DISABLE | LCPLL_PLL_LOCK |
9474                    LCPLL_CD_CLOCK_DISABLE | LCPLL_ROOT_CD_CLOCK_DISABLE |
9475                    LCPLL_CD2X_CLOCK_DISABLE | LCPLL_POWER_DOWN_ALLOW |
9476                    LCPLL_CD_SOURCE_FCLK)) != LCPLL_PLL_LOCK,
9477                  "trying to change cdclk frequency with cdclk not enabled\n"))
9478                 return;
9479
9480         mutex_lock(&dev_priv->rps.hw_lock);
9481         ret = sandybridge_pcode_write(dev_priv,
9482                                       BDW_PCODE_DISPLAY_FREQ_CHANGE_REQ, 0x0);
9483         mutex_unlock(&dev_priv->rps.hw_lock);
9484         if (ret) {
9485                 DRM_ERROR("failed to inform pcode about cdclk change\n");
9486                 return;
9487         }
9488
9489         val = I915_READ(LCPLL_CTL);
9490         val |= LCPLL_CD_SOURCE_FCLK;
9491         I915_WRITE(LCPLL_CTL, val);
9492
9493         if (wait_for_atomic_us(I915_READ(LCPLL_CTL) &
9494                                LCPLL_CD_SOURCE_FCLK_DONE, 1))
9495                 DRM_ERROR("Switching to FCLK failed\n");
9496
9497         val = I915_READ(LCPLL_CTL);
9498         val &= ~LCPLL_CLK_FREQ_MASK;
9499
9500         switch (cdclk) {
9501         case 450000:
9502                 val |= LCPLL_CLK_FREQ_450;
9503                 data = 0;
9504                 break;
9505         case 540000:
9506                 val |= LCPLL_CLK_FREQ_54O_BDW;
9507                 data = 1;
9508                 break;
9509         case 337500:
9510                 val |= LCPLL_CLK_FREQ_337_5_BDW;
9511                 data = 2;
9512                 break;
9513         case 675000:
9514                 val |= LCPLL_CLK_FREQ_675_BDW;
9515                 data = 3;
9516                 break;
9517         default:
9518                 WARN(1, "invalid cdclk frequency\n");
9519                 return;
9520         }
9521
9522         I915_WRITE(LCPLL_CTL, val);
9523
9524         val = I915_READ(LCPLL_CTL);
9525         val &= ~LCPLL_CD_SOURCE_FCLK;
9526         I915_WRITE(LCPLL_CTL, val);
9527
9528         if (wait_for_atomic_us((I915_READ(LCPLL_CTL) &
9529                                 LCPLL_CD_SOURCE_FCLK_DONE) == 0, 1))
9530                 DRM_ERROR("Switching back to LCPLL failed\n");
9531
9532         mutex_lock(&dev_priv->rps.hw_lock);
9533         sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, data);
9534         mutex_unlock(&dev_priv->rps.hw_lock);
9535
9536         intel_update_cdclk(dev);
9537
9538         WARN(cdclk != dev_priv->cdclk_freq,
9539              "cdclk requested %d kHz but got %d kHz\n",
9540              cdclk, dev_priv->cdclk_freq);
9541 }
9542
9543 static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
9544 {
9545         struct drm_i915_private *dev_priv = to_i915(state->dev);
9546         int max_pixclk = ilk_max_pixel_rate(state);
9547         int cdclk;
9548
9549         /*
9550          * FIXME should also account for plane ratio
9551          * once 64bpp pixel formats are supported.
9552          */
9553         if (max_pixclk > 540000)
9554                 cdclk = 675000;
9555         else if (max_pixclk > 450000)
9556                 cdclk = 540000;
9557         else if (max_pixclk > 337500)
9558                 cdclk = 450000;
9559         else
9560                 cdclk = 337500;
9561
9562         /*
9563          * FIXME move the cdclk caclulation to
9564          * compute_config() so we can fail gracegully.
9565          */
9566         if (cdclk > dev_priv->max_cdclk_freq) {
9567                 DRM_ERROR("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
9568                           cdclk, dev_priv->max_cdclk_freq);
9569                 cdclk = dev_priv->max_cdclk_freq;
9570         }
9571
9572         to_intel_atomic_state(state)->cdclk = cdclk;
9573
9574         return 0;
9575 }
9576
9577 static void broadwell_modeset_commit_cdclk(struct drm_atomic_state *old_state)
9578 {
9579         struct drm_device *dev = old_state->dev;
9580         unsigned int req_cdclk = to_intel_atomic_state(old_state)->cdclk;
9581
9582         broadwell_set_cdclk(dev, req_cdclk);
9583 }
9584
9585 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
9586                                       struct intel_crtc_state *crtc_state)
9587 {
9588         if (!intel_ddi_pll_select(crtc, crtc_state))
9589                 return -EINVAL;
9590
9591         crtc->lowfreq_avail = false;
9592
9593         return 0;
9594 }
9595
9596 static void bxt_get_ddi_pll(struct drm_i915_private *dev_priv,
9597                                 enum port port,
9598                                 struct intel_crtc_state *pipe_config)
9599 {
9600         switch (port) {
9601         case PORT_A:
9602                 pipe_config->ddi_pll_sel = SKL_DPLL0;
9603                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL1;
9604                 break;
9605         case PORT_B:
9606                 pipe_config->ddi_pll_sel = SKL_DPLL1;
9607                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL2;
9608                 break;
9609         case PORT_C:
9610                 pipe_config->ddi_pll_sel = SKL_DPLL2;
9611                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL3;
9612                 break;
9613         default:
9614                 DRM_ERROR("Incorrect port type\n");
9615         }
9616 }
9617
9618 static void skylake_get_ddi_pll(struct drm_i915_private *dev_priv,
9619                                 enum port port,
9620                                 struct intel_crtc_state *pipe_config)
9621 {
9622         u32 temp, dpll_ctl1;
9623
9624         temp = I915_READ(DPLL_CTRL2) & DPLL_CTRL2_DDI_CLK_SEL_MASK(port);
9625         pipe_config->ddi_pll_sel = temp >> (port * 3 + 1);
9626
9627         switch (pipe_config->ddi_pll_sel) {
9628         case SKL_DPLL0:
9629                 /*
9630                  * On SKL the eDP DPLL (DPLL0 as we don't use SSC) is not part
9631                  * of the shared DPLL framework and thus needs to be read out
9632                  * separately
9633                  */
9634                 dpll_ctl1 = I915_READ(DPLL_CTRL1);
9635                 pipe_config->dpll_hw_state.ctrl1 = dpll_ctl1 & 0x3f;
9636                 break;
9637         case SKL_DPLL1:
9638                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL1;
9639                 break;
9640         case SKL_DPLL2:
9641                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL2;
9642                 break;
9643         case SKL_DPLL3:
9644                 pipe_config->shared_dpll = DPLL_ID_SKL_DPLL3;
9645                 break;
9646         }
9647 }
9648
9649 static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
9650                                 enum port port,
9651                                 struct intel_crtc_state *pipe_config)
9652 {
9653         pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
9654
9655         switch (pipe_config->ddi_pll_sel) {
9656         case PORT_CLK_SEL_WRPLL1:
9657                 pipe_config->shared_dpll = DPLL_ID_WRPLL1;
9658                 break;
9659         case PORT_CLK_SEL_WRPLL2:
9660                 pipe_config->shared_dpll = DPLL_ID_WRPLL2;
9661                 break;
9662         }
9663 }
9664
9665 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
9666                                        struct intel_crtc_state *pipe_config)
9667 {
9668         struct drm_device *dev = crtc->base.dev;
9669         struct drm_i915_private *dev_priv = dev->dev_private;
9670         struct intel_shared_dpll *pll;
9671         enum port port;
9672         uint32_t tmp;
9673
9674         tmp = I915_READ(TRANS_DDI_FUNC_CTL(pipe_config->cpu_transcoder));
9675
9676         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
9677
9678         if (IS_SKYLAKE(dev))
9679                 skylake_get_ddi_pll(dev_priv, port, pipe_config);
9680         else if (IS_BROXTON(dev))
9681                 bxt_get_ddi_pll(dev_priv, port, pipe_config);
9682         else
9683                 haswell_get_ddi_pll(dev_priv, port, pipe_config);
9684
9685         if (pipe_config->shared_dpll >= 0) {
9686                 pll = &dev_priv->shared_dplls[pipe_config->shared_dpll];
9687
9688                 WARN_ON(!pll->get_hw_state(dev_priv, pll,
9689                                            &pipe_config->dpll_hw_state));
9690         }
9691
9692         /*
9693          * Haswell has only FDI/PCH transcoder A. It is which is connected to
9694          * DDI E. So just check whether this pipe is wired to DDI E and whether
9695          * the PCH transcoder is on.
9696          */
9697         if (INTEL_INFO(dev)->gen < 9 &&
9698             (port == PORT_E) && I915_READ(LPT_TRANSCONF) & TRANS_ENABLE) {
9699                 pipe_config->has_pch_encoder = true;
9700
9701                 tmp = I915_READ(FDI_RX_CTL(PIPE_A));
9702                 pipe_config->fdi_lanes = ((FDI_DP_PORT_WIDTH_MASK & tmp) >>
9703                                           FDI_DP_PORT_WIDTH_SHIFT) + 1;
9704
9705                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
9706         }
9707 }
9708
9709 static bool haswell_get_pipe_config(struct intel_crtc *crtc,
9710                                     struct intel_crtc_state *pipe_config)
9711 {
9712         struct drm_device *dev = crtc->base.dev;
9713         struct drm_i915_private *dev_priv = dev->dev_private;
9714         enum intel_display_power_domain pfit_domain;
9715         uint32_t tmp;
9716
9717         if (!intel_display_power_is_enabled(dev_priv,
9718                                          POWER_DOMAIN_PIPE(crtc->pipe)))
9719                 return false;
9720
9721         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
9722         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
9723
9724         tmp = I915_READ(TRANS_DDI_FUNC_CTL(TRANSCODER_EDP));
9725         if (tmp & TRANS_DDI_FUNC_ENABLE) {
9726                 enum pipe trans_edp_pipe;
9727                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
9728                 default:
9729                         WARN(1, "unknown pipe linked to edp transcoder\n");
9730                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
9731                 case TRANS_DDI_EDP_INPUT_A_ON:
9732                         trans_edp_pipe = PIPE_A;
9733                         break;
9734                 case TRANS_DDI_EDP_INPUT_B_ONOFF:
9735                         trans_edp_pipe = PIPE_B;
9736                         break;
9737                 case TRANS_DDI_EDP_INPUT_C_ONOFF:
9738                         trans_edp_pipe = PIPE_C;
9739                         break;
9740                 }
9741
9742                 if (trans_edp_pipe == crtc->pipe)
9743                         pipe_config->cpu_transcoder = TRANSCODER_EDP;
9744         }
9745
9746         if (!intel_display_power_is_enabled(dev_priv,
9747                         POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder)))
9748                 return false;
9749
9750         tmp = I915_READ(PIPECONF(pipe_config->cpu_transcoder));
9751         if (!(tmp & PIPECONF_ENABLE))
9752                 return false;
9753
9754         haswell_get_ddi_port_state(crtc, pipe_config);
9755
9756         intel_get_pipe_timings(crtc, pipe_config);
9757
9758         if (INTEL_INFO(dev)->gen >= 9) {
9759                 skl_init_scalers(dev, crtc, pipe_config);
9760         }
9761
9762         pfit_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
9763
9764         if (INTEL_INFO(dev)->gen >= 9) {
9765                 pipe_config->scaler_state.scaler_id = -1;
9766                 pipe_config->scaler_state.scaler_users &= ~(1 << SKL_CRTC_INDEX);
9767         }
9768
9769         if (intel_display_power_is_enabled(dev_priv, pfit_domain)) {
9770                 if (INTEL_INFO(dev)->gen == 9)
9771                         skylake_get_pfit_config(crtc, pipe_config);
9772                 else if (INTEL_INFO(dev)->gen < 9)
9773                         ironlake_get_pfit_config(crtc, pipe_config);
9774                 else
9775                         MISSING_CASE(INTEL_INFO(dev)->gen);
9776         }
9777
9778         if (IS_HASWELL(dev))
9779                 pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
9780                         (I915_READ(IPS_CTL) & IPS_ENABLE);
9781
9782         if (pipe_config->cpu_transcoder != TRANSCODER_EDP) {
9783                 pipe_config->pixel_multiplier =
9784                         I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
9785         } else {
9786                 pipe_config->pixel_multiplier = 1;
9787         }
9788
9789         return true;
9790 }
9791
9792 static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
9793 {
9794         struct drm_device *dev = crtc->dev;
9795         struct drm_i915_private *dev_priv = dev->dev_private;
9796         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
9797         uint32_t cntl = 0, size = 0;
9798
9799         if (base) {
9800                 unsigned int width = intel_crtc->base.cursor->state->crtc_w;
9801                 unsigned int height = intel_crtc->base.cursor->state->crtc_h;
9802                 unsigned int stride = roundup_pow_of_two(width) * 4;
9803
9804                 switch (stride) {
9805                 default:
9806                         WARN_ONCE(1, "Invalid cursor width/stride, width=%u, stride=%u\n",
9807                                   width, stride);
9808                         stride = 256;
9809                         /* fallthrough */
9810                 case 256:
9811                 case 512:
9812                 case 1024:
9813                 case 2048:
9814                         break;
9815                 }
9816
9817                 cntl |= CURSOR_ENABLE |
9818                         CURSOR_GAMMA_ENABLE |
9819                         CURSOR_FORMAT_ARGB |
9820                         CURSOR_STRIDE(stride);
9821
9822                 size = (height << 12) | width;
9823         }
9824
9825         if (intel_crtc->cursor_cntl != 0 &&
9826             (intel_crtc->cursor_base != base ||
9827              intel_crtc->cursor_size != size ||
9828              intel_crtc->cursor_cntl != cntl)) {
9829                 /* On these chipsets we can only modify the base/size/stride
9830                  * whilst the cursor is disabled.
9831                  */
9832                 I915_WRITE(_CURACNTR, 0);
9833                 POSTING_READ(_CURACNTR);
9834                 intel_crtc->cursor_cntl = 0;
9835         }
9836
9837         if (intel_crtc->cursor_base != base) {
9838                 I915_WRITE(_CURABASE, base);
9839                 intel_crtc->cursor_base = base;
9840         }
9841
9842         if (intel_crtc->cursor_size != size) {
9843                 I915_WRITE(CURSIZE, size);
9844                 intel_crtc->cursor_size = size;
9845         }
9846
9847         if (intel_crtc->cursor_cntl != cntl) {
9848                 I915_WRITE(_CURACNTR, cntl);
9849                 POSTING_READ(_CURACNTR);
9850                 intel_crtc->cursor_cntl = cntl;
9851         }
9852 }
9853
9854 static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
9855 {
9856         struct drm_device *dev = crtc->dev;
9857         struct drm_i915_private *dev_priv = dev->dev_private;
9858         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
9859         int pipe = intel_crtc->pipe;
9860         uint32_t cntl;
9861
9862         cntl = 0;
9863         if (base) {
9864                 cntl = MCURSOR_GAMMA_ENABLE;
9865                 switch (intel_crtc->base.cursor->state->crtc_w) {
9866                         case 64:
9867                                 cntl |= CURSOR_MODE_64_ARGB_AX;
9868                                 break;
9869                         case 128:
9870                                 cntl |= CURSOR_MODE_128_ARGB_AX;
9871                                 break;
9872                         case 256:
9873                                 cntl |= CURSOR_MODE_256_ARGB_AX;
9874                                 break;
9875                         default:
9876                                 MISSING_CASE(intel_crtc->base.cursor->state->crtc_w);
9877                                 return;
9878                 }
9879                 cntl |= pipe << 28; /* Connect to correct pipe */
9880
9881                 if (IS_HASWELL(dev) || IS_BROADWELL(dev))
9882                         cntl |= CURSOR_PIPE_CSC_ENABLE;
9883         }
9884
9885         if (crtc->cursor->state->rotation == BIT(DRM_ROTATE_180))
9886                 cntl |= CURSOR_ROTATE_180;
9887
9888         if (intel_crtc->cursor_cntl != cntl) {
9889                 I915_WRITE(CURCNTR(pipe), cntl);
9890                 POSTING_READ(CURCNTR(pipe));
9891                 intel_crtc->cursor_cntl = cntl;
9892         }
9893
9894         /* and commit changes on next vblank */
9895         I915_WRITE(CURBASE(pipe), base);
9896         POSTING_READ(CURBASE(pipe));
9897
9898         intel_crtc->cursor_base = base;
9899 }
9900
9901 /* If no-part of the cursor is visible on the framebuffer, then the GPU may hang... */
9902 static void intel_crtc_update_cursor(struct drm_crtc *crtc,
9903                                      bool on)
9904 {
9905         struct drm_device *dev = crtc->dev;
9906         struct drm_i915_private *dev_priv = dev->dev_private;
9907         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
9908         int pipe = intel_crtc->pipe;
9909         int x = crtc->cursor_x;
9910         int y = crtc->cursor_y;
9911         u32 base = 0, pos = 0;
9912
9913         if (on)
9914                 base = intel_crtc->cursor_addr;
9915
9916         if (x >= intel_crtc->config->pipe_src_w)
9917                 base = 0;
9918
9919         if (y >= intel_crtc->config->pipe_src_h)
9920                 base = 0;
9921
9922         if (x < 0) {
9923                 if (x + intel_crtc->base.cursor->state->crtc_w <= 0)
9924                         base = 0;
9925
9926                 pos |= CURSOR_POS_SIGN << CURSOR_X_SHIFT;
9927                 x = -x;
9928         }
9929         pos |= x << CURSOR_X_SHIFT;
9930
9931         if (y < 0) {
9932                 if (y + intel_crtc->base.cursor->state->crtc_h <= 0)
9933                         base = 0;
9934
9935                 pos |= CURSOR_POS_SIGN << CURSOR_Y_SHIFT;
9936                 y = -y;
9937         }
9938         pos |= y << CURSOR_Y_SHIFT;
9939
9940         if (base == 0 && intel_crtc->cursor_base == 0)
9941                 return;
9942
9943         I915_WRITE(CURPOS(pipe), pos);
9944
9945         /* ILK+ do this automagically */
9946         if (HAS_GMCH_DISPLAY(dev) &&
9947             crtc->cursor->state->rotation == BIT(DRM_ROTATE_180)) {
9948                 base += (intel_crtc->base.cursor->state->crtc_h *
9949                         intel_crtc->base.cursor->state->crtc_w - 1) * 4;
9950         }
9951
9952         if (IS_845G(dev) || IS_I865G(dev))
9953                 i845_update_cursor(crtc, base);
9954         else
9955                 i9xx_update_cursor(crtc, base);
9956 }
9957
9958 static bool cursor_size_ok(struct drm_device *dev,
9959                            uint32_t width, uint32_t height)
9960 {
9961         if (width == 0 || height == 0)
9962                 return false;
9963
9964         /*
9965          * 845g/865g are special in that they are only limited by
9966          * the width of their cursors, the height is arbitrary up to
9967          * the precision of the register. Everything else requires
9968          * square cursors, limited to a few power-of-two sizes.
9969          */
9970         if (IS_845G(dev) || IS_I865G(dev)) {
9971                 if ((width & 63) != 0)
9972                         return false;
9973
9974                 if (width > (IS_845G(dev) ? 64 : 512))
9975                         return false;
9976
9977                 if (height > 1023)
9978                         return false;
9979         } else {
9980                 switch (width | height) {
9981                 case 256:
9982                 case 128:
9983                         if (IS_GEN2(dev))
9984                                 return false;
9985                 case 64:
9986                         break;
9987                 default:
9988                         return false;
9989                 }
9990         }
9991
9992         return true;
9993 }
9994
9995 static void intel_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
9996                                  u16 *blue, uint32_t start, uint32_t size)
9997 {
9998         int end = (start + size > 256) ? 256 : start + size, i;
9999         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10000
10001         for (i = start; i < end; i++) {
10002                 intel_crtc->lut_r[i] = red[i] >> 8;
10003                 intel_crtc->lut_g[i] = green[i] >> 8;
10004                 intel_crtc->lut_b[i] = blue[i] >> 8;
10005         }
10006
10007         intel_crtc_load_lut(crtc);
10008 }
10009
10010 /* VESA 640x480x72Hz mode to set on the pipe */
10011 static struct drm_display_mode load_detect_mode = {
10012         DRM_MODE("640x480", DRM_MODE_TYPE_DEFAULT, 31500, 640, 664,
10013                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
10014 };
10015
10016 struct drm_framebuffer *
10017 __intel_framebuffer_create(struct drm_device *dev,
10018                            struct drm_mode_fb_cmd2 *mode_cmd,
10019                            struct drm_i915_gem_object *obj)
10020 {
10021         struct intel_framebuffer *intel_fb;
10022         int ret;
10023
10024         intel_fb = kzalloc(sizeof(*intel_fb), GFP_KERNEL);
10025         if (!intel_fb) {
10026                 drm_gem_object_unreference(&obj->base);
10027                 return ERR_PTR(-ENOMEM);
10028         }
10029
10030         ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
10031         if (ret)
10032                 goto err;
10033
10034         return &intel_fb->base;
10035 err:
10036         drm_gem_object_unreference(&obj->base);
10037         kfree(intel_fb);
10038
10039         return ERR_PTR(ret);
10040 }
10041
10042 static struct drm_framebuffer *
10043 intel_framebuffer_create(struct drm_device *dev,
10044                          struct drm_mode_fb_cmd2 *mode_cmd,
10045                          struct drm_i915_gem_object *obj)
10046 {
10047         struct drm_framebuffer *fb;
10048         int ret;
10049
10050         ret = i915_mutex_lock_interruptible(dev);
10051         if (ret)
10052                 return ERR_PTR(ret);
10053         fb = __intel_framebuffer_create(dev, mode_cmd, obj);
10054         mutex_unlock(&dev->struct_mutex);
10055
10056         return fb;
10057 }
10058
10059 static u32
10060 intel_framebuffer_pitch_for_width(int width, int bpp)
10061 {
10062         u32 pitch = DIV_ROUND_UP(width * bpp, 8);
10063         return ALIGN(pitch, 64);
10064 }
10065
10066 static u32
10067 intel_framebuffer_size_for_mode(struct drm_display_mode *mode, int bpp)
10068 {
10069         u32 pitch = intel_framebuffer_pitch_for_width(mode->hdisplay, bpp);
10070         return PAGE_ALIGN(pitch * mode->vdisplay);
10071 }
10072
10073 static struct drm_framebuffer *
10074 intel_framebuffer_create_for_mode(struct drm_device *dev,
10075                                   struct drm_display_mode *mode,
10076                                   int depth, int bpp)
10077 {
10078         struct drm_i915_gem_object *obj;
10079         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
10080
10081         obj = i915_gem_alloc_object(dev,
10082                                     intel_framebuffer_size_for_mode(mode, bpp));
10083         if (obj == NULL)
10084                 return ERR_PTR(-ENOMEM);
10085
10086         mode_cmd.width = mode->hdisplay;
10087         mode_cmd.height = mode->vdisplay;
10088         mode_cmd.pitches[0] = intel_framebuffer_pitch_for_width(mode_cmd.width,
10089                                                                 bpp);
10090         mode_cmd.pixel_format = drm_mode_legacy_fb_format(bpp, depth);
10091
10092         return intel_framebuffer_create(dev, &mode_cmd, obj);
10093 }
10094
10095 static struct drm_framebuffer *
10096 mode_fits_in_fbdev(struct drm_device *dev,
10097                    struct drm_display_mode *mode)
10098 {
10099 #ifdef CONFIG_DRM_I915_FBDEV
10100         struct drm_i915_private *dev_priv = dev->dev_private;
10101         struct drm_i915_gem_object *obj;
10102         struct drm_framebuffer *fb;
10103
10104         if (!dev_priv->fbdev)
10105                 return NULL;
10106
10107         if (!dev_priv->fbdev->fb)
10108                 return NULL;
10109
10110         obj = dev_priv->fbdev->fb->obj;
10111         BUG_ON(!obj);
10112
10113         fb = &dev_priv->fbdev->fb->base;
10114         if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay,
10115                                                                fb->bits_per_pixel))
10116                 return NULL;
10117
10118         if (obj->base.size < mode->vdisplay * fb->pitches[0])
10119                 return NULL;
10120
10121         return fb;
10122 #else
10123         return NULL;
10124 #endif
10125 }
10126
10127 static int intel_modeset_setup_plane_state(struct drm_atomic_state *state,
10128                                            struct drm_crtc *crtc,
10129                                            struct drm_display_mode *mode,
10130                                            struct drm_framebuffer *fb,
10131                                            int x, int y)
10132 {
10133         struct drm_plane_state *plane_state;
10134         int hdisplay, vdisplay;
10135         int ret;
10136
10137         plane_state = drm_atomic_get_plane_state(state, crtc->primary);
10138         if (IS_ERR(plane_state))
10139                 return PTR_ERR(plane_state);
10140
10141         if (mode)
10142                 drm_crtc_get_hv_timing(mode, &hdisplay, &vdisplay);
10143         else
10144                 hdisplay = vdisplay = 0;
10145
10146         ret = drm_atomic_set_crtc_for_plane(plane_state, fb ? crtc : NULL);
10147         if (ret)
10148                 return ret;
10149         drm_atomic_set_fb_for_plane(plane_state, fb);
10150         plane_state->crtc_x = 0;
10151         plane_state->crtc_y = 0;
10152         plane_state->crtc_w = hdisplay;
10153         plane_state->crtc_h = vdisplay;
10154         plane_state->src_x = x << 16;
10155         plane_state->src_y = y << 16;
10156         plane_state->src_w = hdisplay << 16;
10157         plane_state->src_h = vdisplay << 16;
10158
10159         return 0;
10160 }
10161
10162 bool intel_get_load_detect_pipe(struct drm_connector *connector,
10163                                 struct drm_display_mode *mode,
10164                                 struct intel_load_detect_pipe *old,
10165                                 struct drm_modeset_acquire_ctx *ctx)
10166 {
10167         struct intel_crtc *intel_crtc;
10168         struct intel_encoder *intel_encoder =
10169                 intel_attached_encoder(connector);
10170         struct drm_crtc *possible_crtc;
10171         struct drm_encoder *encoder = &intel_encoder->base;
10172         struct drm_crtc *crtc = NULL;
10173         struct drm_device *dev = encoder->dev;
10174         struct drm_framebuffer *fb;
10175         struct drm_mode_config *config = &dev->mode_config;
10176         struct drm_atomic_state *state = NULL;
10177         struct drm_connector_state *connector_state;
10178         struct intel_crtc_state *crtc_state;
10179         int ret, i = -1;
10180
10181         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10182                       connector->base.id, connector->name,
10183                       encoder->base.id, encoder->name);
10184
10185 retry:
10186         ret = drm_modeset_lock(&config->connection_mutex, ctx);
10187         if (ret)
10188                 goto fail_unlock;
10189
10190         /*
10191          * Algorithm gets a little messy:
10192          *
10193          *   - if the connector already has an assigned crtc, use it (but make
10194          *     sure it's on first)
10195          *
10196          *   - try to find the first unused crtc that can drive this connector,
10197          *     and use that if we find one
10198          */
10199
10200         /* See if we already have a CRTC for this connector */
10201         if (encoder->crtc) {
10202                 crtc = encoder->crtc;
10203
10204                 ret = drm_modeset_lock(&crtc->mutex, ctx);
10205                 if (ret)
10206                         goto fail_unlock;
10207                 ret = drm_modeset_lock(&crtc->primary->mutex, ctx);
10208                 if (ret)
10209                         goto fail_unlock;
10210
10211                 old->dpms_mode = connector->dpms;
10212                 old->load_detect_temp = false;
10213
10214                 /* Make sure the crtc and connector are running */
10215                 if (connector->dpms != DRM_MODE_DPMS_ON)
10216                         connector->funcs->dpms(connector, DRM_MODE_DPMS_ON);
10217
10218                 return true;
10219         }
10220
10221         /* Find an unused one (if possible) */
10222         for_each_crtc(dev, possible_crtc) {
10223                 i++;
10224                 if (!(encoder->possible_crtcs & (1 << i)))
10225                         continue;
10226                 if (possible_crtc->state->enable)
10227                         continue;
10228                 /* This can occur when applying the pipe A quirk on resume. */
10229                 if (to_intel_crtc(possible_crtc)->new_enabled)
10230                         continue;
10231
10232                 crtc = possible_crtc;
10233                 break;
10234         }
10235
10236         /*
10237          * If we didn't find an unused CRTC, don't use any.
10238          */
10239         if (!crtc) {
10240                 DRM_DEBUG_KMS("no pipe available for load-detect\n");
10241                 goto fail_unlock;
10242         }
10243
10244         ret = drm_modeset_lock(&crtc->mutex, ctx);
10245         if (ret)
10246                 goto fail_unlock;
10247         ret = drm_modeset_lock(&crtc->primary->mutex, ctx);
10248         if (ret)
10249                 goto fail_unlock;
10250         intel_encoder->new_crtc = to_intel_crtc(crtc);
10251         to_intel_connector(connector)->new_encoder = intel_encoder;
10252
10253         intel_crtc = to_intel_crtc(crtc);
10254         intel_crtc->new_enabled = true;
10255         old->dpms_mode = connector->dpms;
10256         old->load_detect_temp = true;
10257         old->release_fb = NULL;
10258
10259         state = drm_atomic_state_alloc(dev);
10260         if (!state)
10261                 return false;
10262
10263         state->acquire_ctx = ctx;
10264
10265         connector_state = drm_atomic_get_connector_state(state, connector);
10266         if (IS_ERR(connector_state)) {
10267                 ret = PTR_ERR(connector_state);
10268                 goto fail;
10269         }
10270
10271         connector_state->crtc = crtc;
10272         connector_state->best_encoder = &intel_encoder->base;
10273
10274         crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10275         if (IS_ERR(crtc_state)) {
10276                 ret = PTR_ERR(crtc_state);
10277                 goto fail;
10278         }
10279
10280         crtc_state->base.active = crtc_state->base.enable = true;
10281
10282         if (!mode)
10283                 mode = &load_detect_mode;
10284
10285         /* We need a framebuffer large enough to accommodate all accesses
10286          * that the plane may generate whilst we perform load detection.
10287          * We can not rely on the fbcon either being present (we get called
10288          * during its initialisation to detect all boot displays, or it may
10289          * not even exist) or that it is large enough to satisfy the
10290          * requested mode.
10291          */
10292         fb = mode_fits_in_fbdev(dev, mode);
10293         if (fb == NULL) {
10294                 DRM_DEBUG_KMS("creating tmp fb for load-detection\n");
10295                 fb = intel_framebuffer_create_for_mode(dev, mode, 24, 32);
10296                 old->release_fb = fb;
10297         } else
10298                 DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
10299         if (IS_ERR(fb)) {
10300                 DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
10301                 goto fail;
10302         }
10303
10304         ret = intel_modeset_setup_plane_state(state, crtc, mode, fb, 0, 0);
10305         if (ret)
10306                 goto fail;
10307
10308         drm_mode_copy(&crtc_state->base.mode, mode);
10309
10310         if (intel_set_mode(state)) {
10311                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
10312                 if (old->release_fb)
10313                         old->release_fb->funcs->destroy(old->release_fb);
10314                 goto fail;
10315         }
10316         crtc->primary->crtc = crtc;
10317
10318         /* let the connector get through one full cycle before testing */
10319         intel_wait_for_vblank(dev, intel_crtc->pipe);
10320         return true;
10321
10322  fail:
10323         intel_crtc->new_enabled = crtc->state->enable;
10324 fail_unlock:
10325         drm_atomic_state_free(state);
10326         state = NULL;
10327
10328         if (ret == -EDEADLK) {
10329                 drm_modeset_backoff(ctx);
10330                 goto retry;
10331         }
10332
10333         return false;
10334 }
10335
10336 void intel_release_load_detect_pipe(struct drm_connector *connector,
10337                                     struct intel_load_detect_pipe *old,
10338                                     struct drm_modeset_acquire_ctx *ctx)
10339 {
10340         struct drm_device *dev = connector->dev;
10341         struct intel_encoder *intel_encoder =
10342                 intel_attached_encoder(connector);
10343         struct drm_encoder *encoder = &intel_encoder->base;
10344         struct drm_crtc *crtc = encoder->crtc;
10345         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10346         struct drm_atomic_state *state;
10347         struct drm_connector_state *connector_state;
10348         struct intel_crtc_state *crtc_state;
10349         int ret;
10350
10351         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
10352                       connector->base.id, connector->name,
10353                       encoder->base.id, encoder->name);
10354
10355         if (old->load_detect_temp) {
10356                 state = drm_atomic_state_alloc(dev);
10357                 if (!state)
10358                         goto fail;
10359
10360                 state->acquire_ctx = ctx;
10361
10362                 connector_state = drm_atomic_get_connector_state(state, connector);
10363                 if (IS_ERR(connector_state))
10364                         goto fail;
10365
10366                 crtc_state = intel_atomic_get_crtc_state(state, intel_crtc);
10367                 if (IS_ERR(crtc_state))
10368                         goto fail;
10369
10370                 to_intel_connector(connector)->new_encoder = NULL;
10371                 intel_encoder->new_crtc = NULL;
10372                 intel_crtc->new_enabled = false;
10373
10374                 connector_state->best_encoder = NULL;
10375                 connector_state->crtc = NULL;
10376
10377                 crtc_state->base.enable = crtc_state->base.active = false;
10378
10379                 ret = intel_modeset_setup_plane_state(state, crtc, NULL, NULL,
10380                                                       0, 0);
10381                 if (ret)
10382                         goto fail;
10383
10384                 ret = intel_set_mode(state);
10385                 if (ret)
10386                         goto fail;
10387
10388                 if (old->release_fb) {
10389                         drm_framebuffer_unregister_private(old->release_fb);
10390                         drm_framebuffer_unreference(old->release_fb);
10391                 }
10392
10393                 return;
10394         }
10395
10396         /* Switch crtc and encoder back off if necessary */
10397         if (old->dpms_mode != DRM_MODE_DPMS_ON)
10398                 connector->funcs->dpms(connector, old->dpms_mode);
10399
10400         return;
10401 fail:
10402         DRM_DEBUG_KMS("Couldn't release load detect pipe.\n");
10403         drm_atomic_state_free(state);
10404 }
10405
10406 static int i9xx_pll_refclk(struct drm_device *dev,
10407                            const struct intel_crtc_state *pipe_config)
10408 {
10409         struct drm_i915_private *dev_priv = dev->dev_private;
10410         u32 dpll = pipe_config->dpll_hw_state.dpll;
10411
10412         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
10413                 return dev_priv->vbt.lvds_ssc_freq;
10414         else if (HAS_PCH_SPLIT(dev))
10415                 return 120000;
10416         else if (!IS_GEN2(dev))
10417                 return 96000;
10418         else
10419                 return 48000;
10420 }
10421
10422 /* Returns the clock of the currently programmed mode of the given pipe. */
10423 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
10424                                 struct intel_crtc_state *pipe_config)
10425 {
10426         struct drm_device *dev = crtc->base.dev;
10427         struct drm_i915_private *dev_priv = dev->dev_private;
10428         int pipe = pipe_config->cpu_transcoder;
10429         u32 dpll = pipe_config->dpll_hw_state.dpll;
10430         u32 fp;
10431         intel_clock_t clock;
10432         int refclk = i9xx_pll_refclk(dev, pipe_config);
10433
10434         if ((dpll & DISPLAY_RATE_SELECT_FPA1) == 0)
10435                 fp = pipe_config->dpll_hw_state.fp0;
10436         else
10437                 fp = pipe_config->dpll_hw_state.fp1;
10438
10439         clock.m1 = (fp & FP_M1_DIV_MASK) >> FP_M1_DIV_SHIFT;
10440         if (IS_PINEVIEW(dev)) {
10441                 clock.n = ffs((fp & FP_N_PINEVIEW_DIV_MASK) >> FP_N_DIV_SHIFT) - 1;
10442                 clock.m2 = (fp & FP_M2_PINEVIEW_DIV_MASK) >> FP_M2_DIV_SHIFT;
10443         } else {
10444                 clock.n = (fp & FP_N_DIV_MASK) >> FP_N_DIV_SHIFT;
10445                 clock.m2 = (fp & FP_M2_DIV_MASK) >> FP_M2_DIV_SHIFT;
10446         }
10447
10448         if (!IS_GEN2(dev)) {
10449                 if (IS_PINEVIEW(dev))
10450                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW) >>
10451                                 DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW);
10452                 else
10453                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK) >>
10454                                DPLL_FPA01_P1_POST_DIV_SHIFT);
10455
10456                 switch (dpll & DPLL_MODE_MASK) {
10457                 case DPLLB_MODE_DAC_SERIAL:
10458                         clock.p2 = dpll & DPLL_DAC_SERIAL_P2_CLOCK_DIV_5 ?
10459                                 5 : 10;
10460                         break;
10461                 case DPLLB_MODE_LVDS:
10462                         clock.p2 = dpll & DPLLB_LVDS_P2_CLOCK_DIV_7 ?
10463                                 7 : 14;
10464                         break;
10465                 default:
10466                         DRM_DEBUG_KMS("Unknown DPLL mode %08x in programmed "
10467                                   "mode\n", (int)(dpll & DPLL_MODE_MASK));
10468                         return;
10469                 }
10470
10471                 if (IS_PINEVIEW(dev))
10472                         pineview_clock(refclk, &clock);
10473                 else
10474                         i9xx_clock(refclk, &clock);
10475         } else {
10476                 u32 lvds = IS_I830(dev) ? 0 : I915_READ(LVDS);
10477                 bool is_lvds = (pipe == 1) && (lvds & LVDS_PORT_EN);
10478
10479                 if (is_lvds) {
10480                         clock.p1 = ffs((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830_LVDS) >>
10481                                        DPLL_FPA01_P1_POST_DIV_SHIFT);
10482
10483                         if (lvds & LVDS_CLKB_POWER_UP)
10484                                 clock.p2 = 7;
10485                         else
10486                                 clock.p2 = 14;
10487                 } else {
10488                         if (dpll & PLL_P1_DIVIDE_BY_TWO)
10489                                 clock.p1 = 2;
10490                         else {
10491                                 clock.p1 = ((dpll & DPLL_FPA01_P1_POST_DIV_MASK_I830) >>
10492                                             DPLL_FPA01_P1_POST_DIV_SHIFT) + 2;
10493                         }
10494                         if (dpll & PLL_P2_DIVIDE_BY_4)
10495                                 clock.p2 = 4;
10496                         else
10497                                 clock.p2 = 2;
10498                 }
10499
10500                 i9xx_clock(refclk, &clock);
10501         }
10502
10503         /*
10504          * This value includes pixel_multiplier. We will use
10505          * port_clock to compute adjusted_mode.crtc_clock in the
10506          * encoder's get_config() function.
10507          */
10508         pipe_config->port_clock = clock.dot;
10509 }
10510
10511 int intel_dotclock_calculate(int link_freq,
10512                              const struct intel_link_m_n *m_n)
10513 {
10514         /*
10515          * The calculation for the data clock is:
10516          * pixel_clock = ((m/n)*(link_clock * nr_lanes))/bpp
10517          * But we want to avoid losing precison if possible, so:
10518          * pixel_clock = ((m * link_clock * nr_lanes)/(n*bpp))
10519          *
10520          * and the link clock is simpler:
10521          * link_clock = (m * link_clock) / n
10522          */
10523
10524         if (!m_n->link_n)
10525                 return 0;
10526
10527         return div_u64((u64)m_n->link_m * link_freq, m_n->link_n);
10528 }
10529
10530 static void ironlake_pch_clock_get(struct intel_crtc *crtc,
10531                                    struct intel_crtc_state *pipe_config)
10532 {
10533         struct drm_device *dev = crtc->base.dev;
10534
10535         /* read out port_clock from the DPLL */
10536         i9xx_crtc_clock_get(crtc, pipe_config);
10537
10538         /*
10539          * This value does not include pixel_multiplier.
10540          * We will check that port_clock and adjusted_mode.crtc_clock
10541          * agree once we know their relationship in the encoder's
10542          * get_config() function.
10543          */
10544         pipe_config->base.adjusted_mode.crtc_clock =
10545                 intel_dotclock_calculate(intel_fdi_link_freq(dev) * 10000,
10546                                          &pipe_config->fdi_m_n);
10547 }
10548
10549 /** Returns the currently programmed mode of the given pipe. */
10550 struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
10551                                              struct drm_crtc *crtc)
10552 {
10553         struct drm_i915_private *dev_priv = dev->dev_private;
10554         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10555         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
10556         struct drm_display_mode *mode;
10557         struct intel_crtc_state pipe_config;
10558         int htot = I915_READ(HTOTAL(cpu_transcoder));
10559         int hsync = I915_READ(HSYNC(cpu_transcoder));
10560         int vtot = I915_READ(VTOTAL(cpu_transcoder));
10561         int vsync = I915_READ(VSYNC(cpu_transcoder));
10562         enum pipe pipe = intel_crtc->pipe;
10563
10564         mode = kzalloc(sizeof(*mode), GFP_KERNEL);
10565         if (!mode)
10566                 return NULL;
10567
10568         /*
10569          * Construct a pipe_config sufficient for getting the clock info
10570          * back out of crtc_clock_get.
10571          *
10572          * Note, if LVDS ever uses a non-1 pixel multiplier, we'll need
10573          * to use a real value here instead.
10574          */
10575         pipe_config.cpu_transcoder = (enum transcoder) pipe;
10576         pipe_config.pixel_multiplier = 1;
10577         pipe_config.dpll_hw_state.dpll = I915_READ(DPLL(pipe));
10578         pipe_config.dpll_hw_state.fp0 = I915_READ(FP0(pipe));
10579         pipe_config.dpll_hw_state.fp1 = I915_READ(FP1(pipe));
10580         i9xx_crtc_clock_get(intel_crtc, &pipe_config);
10581
10582         mode->clock = pipe_config.port_clock / pipe_config.pixel_multiplier;
10583         mode->hdisplay = (htot & 0xffff) + 1;
10584         mode->htotal = ((htot & 0xffff0000) >> 16) + 1;
10585         mode->hsync_start = (hsync & 0xffff) + 1;
10586         mode->hsync_end = ((hsync & 0xffff0000) >> 16) + 1;
10587         mode->vdisplay = (vtot & 0xffff) + 1;
10588         mode->vtotal = ((vtot & 0xffff0000) >> 16) + 1;
10589         mode->vsync_start = (vsync & 0xffff) + 1;
10590         mode->vsync_end = ((vsync & 0xffff0000) >> 16) + 1;
10591
10592         drm_mode_set_name(mode);
10593
10594         return mode;
10595 }
10596
10597 void intel_mark_busy(struct drm_device *dev)
10598 {
10599         struct drm_i915_private *dev_priv = dev->dev_private;
10600
10601         if (dev_priv->mm.busy)
10602                 return;
10603
10604         intel_runtime_pm_get(dev_priv);
10605         i915_update_gfx_val(dev_priv);
10606         if (INTEL_INFO(dev)->gen >= 6)
10607                 gen6_rps_busy(dev_priv);
10608         dev_priv->mm.busy = true;
10609 }
10610
10611 void intel_mark_idle(struct drm_device *dev)
10612 {
10613         struct drm_i915_private *dev_priv = dev->dev_private;
10614
10615         if (!dev_priv->mm.busy)
10616                 return;
10617
10618         dev_priv->mm.busy = false;
10619
10620         if (INTEL_INFO(dev)->gen >= 6)
10621                 gen6_rps_idle(dev->dev_private);
10622
10623         intel_runtime_pm_put(dev_priv);
10624 }
10625
10626 static void intel_crtc_destroy(struct drm_crtc *crtc)
10627 {
10628         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10629         struct drm_device *dev = crtc->dev;
10630         struct intel_unpin_work *work;
10631
10632         spin_lock_irq(&dev->event_lock);
10633         work = intel_crtc->unpin_work;
10634         intel_crtc->unpin_work = NULL;
10635         spin_unlock_irq(&dev->event_lock);
10636
10637         if (work) {
10638                 cancel_work_sync(&work->work);
10639                 kfree(work);
10640         }
10641
10642         drm_crtc_cleanup(crtc);
10643
10644         kfree(intel_crtc);
10645 }
10646
10647 static void intel_unpin_work_fn(struct work_struct *__work)
10648 {
10649         struct intel_unpin_work *work =
10650                 container_of(__work, struct intel_unpin_work, work);
10651         struct drm_device *dev = work->crtc->dev;
10652         enum pipe pipe = to_intel_crtc(work->crtc)->pipe;
10653
10654         mutex_lock(&dev->struct_mutex);
10655         intel_unpin_fb_obj(work->old_fb, work->crtc->primary->state);
10656         drm_gem_object_unreference(&work->pending_flip_obj->base);
10657
10658         intel_fbc_update(dev);
10659
10660         if (work->flip_queued_req)
10661                 i915_gem_request_assign(&work->flip_queued_req, NULL);
10662         mutex_unlock(&dev->struct_mutex);
10663
10664         intel_frontbuffer_flip_complete(dev, INTEL_FRONTBUFFER_PRIMARY(pipe));
10665         drm_framebuffer_unreference(work->old_fb);
10666
10667         BUG_ON(atomic_read(&to_intel_crtc(work->crtc)->unpin_work_count) == 0);
10668         atomic_dec(&to_intel_crtc(work->crtc)->unpin_work_count);
10669
10670         kfree(work);
10671 }
10672
10673 static void do_intel_finish_page_flip(struct drm_device *dev,
10674                                       struct drm_crtc *crtc)
10675 {
10676         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10677         struct intel_unpin_work *work;
10678         unsigned long flags;
10679
10680         /* Ignore early vblank irqs */
10681         if (intel_crtc == NULL)
10682                 return;
10683
10684         /*
10685          * This is called both by irq handlers and the reset code (to complete
10686          * lost pageflips) so needs the full irqsave spinlocks.
10687          */
10688         spin_lock_irqsave(&dev->event_lock, flags);
10689         work = intel_crtc->unpin_work;
10690
10691         /* Ensure we don't miss a work->pending update ... */
10692         smp_rmb();
10693
10694         if (work == NULL || atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) {
10695                 spin_unlock_irqrestore(&dev->event_lock, flags);
10696                 return;
10697         }
10698
10699         page_flip_completed(intel_crtc);
10700
10701         spin_unlock_irqrestore(&dev->event_lock, flags);
10702 }
10703
10704 void intel_finish_page_flip(struct drm_device *dev, int pipe)
10705 {
10706         struct drm_i915_private *dev_priv = dev->dev_private;
10707         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
10708
10709         do_intel_finish_page_flip(dev, crtc);
10710 }
10711
10712 void intel_finish_page_flip_plane(struct drm_device *dev, int plane)
10713 {
10714         struct drm_i915_private *dev_priv = dev->dev_private;
10715         struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane];
10716
10717         do_intel_finish_page_flip(dev, crtc);
10718 }
10719
10720 /* Is 'a' after or equal to 'b'? */
10721 static bool g4x_flip_count_after_eq(u32 a, u32 b)
10722 {
10723         return !((a - b) & 0x80000000);
10724 }
10725
10726 static bool page_flip_finished(struct intel_crtc *crtc)
10727 {
10728         struct drm_device *dev = crtc->base.dev;
10729         struct drm_i915_private *dev_priv = dev->dev_private;
10730
10731         if (i915_reset_in_progress(&dev_priv->gpu_error) ||
10732             crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
10733                 return true;
10734
10735         /*
10736          * The relevant registers doen't exist on pre-ctg.
10737          * As the flip done interrupt doesn't trigger for mmio
10738          * flips on gmch platforms, a flip count check isn't
10739          * really needed there. But since ctg has the registers,
10740          * include it in the check anyway.
10741          */
10742         if (INTEL_INFO(dev)->gen < 5 && !IS_G4X(dev))
10743                 return true;
10744
10745         /*
10746          * A DSPSURFLIVE check isn't enough in case the mmio and CS flips
10747          * used the same base address. In that case the mmio flip might
10748          * have completed, but the CS hasn't even executed the flip yet.
10749          *
10750          * A flip count check isn't enough as the CS might have updated
10751          * the base address just after start of vblank, but before we
10752          * managed to process the interrupt. This means we'd complete the
10753          * CS flip too soon.
10754          *
10755          * Combining both checks should get us a good enough result. It may
10756          * still happen that the CS flip has been executed, but has not
10757          * yet actually completed. But in case the base address is the same
10758          * anyway, we don't really care.
10759          */
10760         return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) ==
10761                 crtc->unpin_work->gtt_offset &&
10762                 g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_GM45(crtc->pipe)),
10763                                     crtc->unpin_work->flip_count);
10764 }
10765
10766 void intel_prepare_page_flip(struct drm_device *dev, int plane)
10767 {
10768         struct drm_i915_private *dev_priv = dev->dev_private;
10769         struct intel_crtc *intel_crtc =
10770                 to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]);
10771         unsigned long flags;
10772
10773
10774         /*
10775          * This is called both by irq handlers and the reset code (to complete
10776          * lost pageflips) so needs the full irqsave spinlocks.
10777          *
10778          * NB: An MMIO update of the plane base pointer will also
10779          * generate a page-flip completion irq, i.e. every modeset
10780          * is also accompanied by a spurious intel_prepare_page_flip().
10781          */
10782         spin_lock_irqsave(&dev->event_lock, flags);
10783         if (intel_crtc->unpin_work && page_flip_finished(intel_crtc))
10784                 atomic_inc_not_zero(&intel_crtc->unpin_work->pending);
10785         spin_unlock_irqrestore(&dev->event_lock, flags);
10786 }
10787
10788 static inline void intel_mark_page_flip_active(struct intel_crtc *intel_crtc)
10789 {
10790         /* Ensure that the work item is consistent when activating it ... */
10791         smp_wmb();
10792         atomic_set(&intel_crtc->unpin_work->pending, INTEL_FLIP_PENDING);
10793         /* and that it is marked active as soon as the irq could fire. */
10794         smp_wmb();
10795 }
10796
10797 static int intel_gen2_queue_flip(struct drm_device *dev,
10798                                  struct drm_crtc *crtc,
10799                                  struct drm_framebuffer *fb,
10800                                  struct drm_i915_gem_object *obj,
10801                                  struct drm_i915_gem_request *req,
10802                                  uint32_t flags)
10803 {
10804         struct intel_engine_cs *ring = req->ring;
10805         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10806         u32 flip_mask;
10807         int ret;
10808
10809         ret = intel_ring_begin(req, 6);
10810         if (ret)
10811                 return ret;
10812
10813         /* Can't queue multiple flips, so wait for the previous
10814          * one to finish before executing the next.
10815          */
10816         if (intel_crtc->plane)
10817                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
10818         else
10819                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
10820         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
10821         intel_ring_emit(ring, MI_NOOP);
10822         intel_ring_emit(ring, MI_DISPLAY_FLIP |
10823                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
10824         intel_ring_emit(ring, fb->pitches[0]);
10825         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
10826         intel_ring_emit(ring, 0); /* aux display base address, unused */
10827
10828         intel_mark_page_flip_active(intel_crtc);
10829         return 0;
10830 }
10831
10832 static int intel_gen3_queue_flip(struct drm_device *dev,
10833                                  struct drm_crtc *crtc,
10834                                  struct drm_framebuffer *fb,
10835                                  struct drm_i915_gem_object *obj,
10836                                  struct drm_i915_gem_request *req,
10837                                  uint32_t flags)
10838 {
10839         struct intel_engine_cs *ring = req->ring;
10840         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10841         u32 flip_mask;
10842         int ret;
10843
10844         ret = intel_ring_begin(req, 6);
10845         if (ret)
10846                 return ret;
10847
10848         if (intel_crtc->plane)
10849                 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
10850         else
10851                 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
10852         intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
10853         intel_ring_emit(ring, MI_NOOP);
10854         intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 |
10855                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
10856         intel_ring_emit(ring, fb->pitches[0]);
10857         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
10858         intel_ring_emit(ring, MI_NOOP);
10859
10860         intel_mark_page_flip_active(intel_crtc);
10861         return 0;
10862 }
10863
10864 static int intel_gen4_queue_flip(struct drm_device *dev,
10865                                  struct drm_crtc *crtc,
10866                                  struct drm_framebuffer *fb,
10867                                  struct drm_i915_gem_object *obj,
10868                                  struct drm_i915_gem_request *req,
10869                                  uint32_t flags)
10870 {
10871         struct intel_engine_cs *ring = req->ring;
10872         struct drm_i915_private *dev_priv = dev->dev_private;
10873         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10874         uint32_t pf, pipesrc;
10875         int ret;
10876
10877         ret = intel_ring_begin(req, 4);
10878         if (ret)
10879                 return ret;
10880
10881         /* i965+ uses the linear or tiled offsets from the
10882          * Display Registers (which do not change across a page-flip)
10883          * so we need only reprogram the base address.
10884          */
10885         intel_ring_emit(ring, MI_DISPLAY_FLIP |
10886                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
10887         intel_ring_emit(ring, fb->pitches[0]);
10888         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset |
10889                         obj->tiling_mode);
10890
10891         /* XXX Enabling the panel-fitter across page-flip is so far
10892          * untested on non-native modes, so ignore it for now.
10893          * pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE;
10894          */
10895         pf = 0;
10896         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
10897         intel_ring_emit(ring, pf | pipesrc);
10898
10899         intel_mark_page_flip_active(intel_crtc);
10900         return 0;
10901 }
10902
10903 static int intel_gen6_queue_flip(struct drm_device *dev,
10904                                  struct drm_crtc *crtc,
10905                                  struct drm_framebuffer *fb,
10906                                  struct drm_i915_gem_object *obj,
10907                                  struct drm_i915_gem_request *req,
10908                                  uint32_t flags)
10909 {
10910         struct intel_engine_cs *ring = req->ring;
10911         struct drm_i915_private *dev_priv = dev->dev_private;
10912         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10913         uint32_t pf, pipesrc;
10914         int ret;
10915
10916         ret = intel_ring_begin(req, 4);
10917         if (ret)
10918                 return ret;
10919
10920         intel_ring_emit(ring, MI_DISPLAY_FLIP |
10921                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
10922         intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode);
10923         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
10924
10925         /* Contrary to the suggestions in the documentation,
10926          * "Enable Panel Fitter" does not seem to be required when page
10927          * flipping with a non-native mode, and worse causes a normal
10928          * modeset to fail.
10929          * pf = I915_READ(PF_CTL(intel_crtc->pipe)) & PF_ENABLE;
10930          */
10931         pf = 0;
10932         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
10933         intel_ring_emit(ring, pf | pipesrc);
10934
10935         intel_mark_page_flip_active(intel_crtc);
10936         return 0;
10937 }
10938
10939 static int intel_gen7_queue_flip(struct drm_device *dev,
10940                                  struct drm_crtc *crtc,
10941                                  struct drm_framebuffer *fb,
10942                                  struct drm_i915_gem_object *obj,
10943                                  struct drm_i915_gem_request *req,
10944                                  uint32_t flags)
10945 {
10946         struct intel_engine_cs *ring = req->ring;
10947         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
10948         uint32_t plane_bit = 0;
10949         int len, ret;
10950
10951         switch (intel_crtc->plane) {
10952         case PLANE_A:
10953                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_A;
10954                 break;
10955         case PLANE_B:
10956                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_B;
10957                 break;
10958         case PLANE_C:
10959                 plane_bit = MI_DISPLAY_FLIP_IVB_PLANE_C;
10960                 break;
10961         default:
10962                 WARN_ONCE(1, "unknown plane in flip command\n");
10963                 return -ENODEV;
10964         }
10965
10966         len = 4;
10967         if (ring->id == RCS) {
10968                 len += 6;
10969                 /*
10970                  * On Gen 8, SRM is now taking an extra dword to accommodate
10971                  * 48bits addresses, and we need a NOOP for the batch size to
10972                  * stay even.
10973                  */
10974                 if (IS_GEN8(dev))
10975                         len += 2;
10976         }
10977
10978         /*
10979          * BSpec MI_DISPLAY_FLIP for IVB:
10980          * "The full packet must be contained within the same cache line."
10981          *
10982          * Currently the LRI+SRM+MI_DISPLAY_FLIP all fit within the same
10983          * cacheline, if we ever start emitting more commands before
10984          * the MI_DISPLAY_FLIP we may need to first emit everything else,
10985          * then do the cacheline alignment, and finally emit the
10986          * MI_DISPLAY_FLIP.
10987          */
10988         ret = intel_ring_cacheline_align(req);
10989         if (ret)
10990                 return ret;
10991
10992         ret = intel_ring_begin(req, len);
10993         if (ret)
10994                 return ret;
10995
10996         /* Unmask the flip-done completion message. Note that the bspec says that
10997          * we should do this for both the BCS and RCS, and that we must not unmask
10998          * more than one flip event at any time (or ensure that one flip message
10999          * can be sent by waiting for flip-done prior to queueing new flips).
11000          * Experimentation says that BCS works despite DERRMR masking all
11001          * flip-done completion events and that unmasking all planes at once
11002          * for the RCS also doesn't appear to drop events. Setting the DERRMR
11003          * to zero does lead to lockups within MI_DISPLAY_FLIP.
11004          */
11005         if (ring->id == RCS) {
11006                 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
11007                 intel_ring_emit(ring, DERRMR);
11008                 intel_ring_emit(ring, ~(DERRMR_PIPEA_PRI_FLIP_DONE |
11009                                         DERRMR_PIPEB_PRI_FLIP_DONE |
11010                                         DERRMR_PIPEC_PRI_FLIP_DONE));
11011                 if (IS_GEN8(dev))
11012                         intel_ring_emit(ring, MI_STORE_REGISTER_MEM_GEN8(1) |
11013                                               MI_SRM_LRM_GLOBAL_GTT);
11014                 else
11015                         intel_ring_emit(ring, MI_STORE_REGISTER_MEM(1) |
11016                                               MI_SRM_LRM_GLOBAL_GTT);
11017                 intel_ring_emit(ring, DERRMR);
11018                 intel_ring_emit(ring, ring->scratch.gtt_offset + 256);
11019                 if (IS_GEN8(dev)) {
11020                         intel_ring_emit(ring, 0);
11021                         intel_ring_emit(ring, MI_NOOP);
11022                 }
11023         }
11024
11025         intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | plane_bit);
11026         intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode));
11027         intel_ring_emit(ring, intel_crtc->unpin_work->gtt_offset);
11028         intel_ring_emit(ring, (MI_NOOP));
11029
11030         intel_mark_page_flip_active(intel_crtc);
11031         return 0;
11032 }
11033
11034 static bool use_mmio_flip(struct intel_engine_cs *ring,
11035                           struct drm_i915_gem_object *obj)
11036 {
11037         /*
11038          * This is not being used for older platforms, because
11039          * non-availability of flip done interrupt forces us to use
11040          * CS flips. Older platforms derive flip done using some clever
11041          * tricks involving the flip_pending status bits and vblank irqs.
11042          * So using MMIO flips there would disrupt this mechanism.
11043          */
11044
11045         if (ring == NULL)
11046                 return true;
11047
11048         if (INTEL_INFO(ring->dev)->gen < 5)
11049                 return false;
11050
11051         if (i915.use_mmio_flip < 0)
11052                 return false;
11053         else if (i915.use_mmio_flip > 0)
11054                 return true;
11055         else if (i915.enable_execlists)
11056                 return true;
11057         else
11058                 return ring != i915_gem_request_get_ring(obj->last_write_req);
11059 }
11060
11061 static void skl_do_mmio_flip(struct intel_crtc *intel_crtc)
11062 {
11063         struct drm_device *dev = intel_crtc->base.dev;
11064         struct drm_i915_private *dev_priv = dev->dev_private;
11065         struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
11066         const enum pipe pipe = intel_crtc->pipe;
11067         u32 ctl, stride;
11068
11069         ctl = I915_READ(PLANE_CTL(pipe, 0));
11070         ctl &= ~PLANE_CTL_TILED_MASK;
11071         switch (fb->modifier[0]) {
11072         case DRM_FORMAT_MOD_NONE:
11073                 break;
11074         case I915_FORMAT_MOD_X_TILED:
11075                 ctl |= PLANE_CTL_TILED_X;
11076                 break;
11077         case I915_FORMAT_MOD_Y_TILED:
11078                 ctl |= PLANE_CTL_TILED_Y;
11079                 break;
11080         case I915_FORMAT_MOD_Yf_TILED:
11081                 ctl |= PLANE_CTL_TILED_YF;
11082                 break;
11083         default:
11084                 MISSING_CASE(fb->modifier[0]);
11085         }
11086
11087         /*
11088          * The stride is either expressed as a multiple of 64 bytes chunks for
11089          * linear buffers or in number of tiles for tiled buffers.
11090          */
11091         stride = fb->pitches[0] /
11092                  intel_fb_stride_alignment(dev, fb->modifier[0],
11093                                            fb->pixel_format);
11094
11095         /*
11096          * Both PLANE_CTL and PLANE_STRIDE are not updated on vblank but on
11097          * PLANE_SURF updates, the update is then guaranteed to be atomic.
11098          */
11099         I915_WRITE(PLANE_CTL(pipe, 0), ctl);
11100         I915_WRITE(PLANE_STRIDE(pipe, 0), stride);
11101
11102         I915_WRITE(PLANE_SURF(pipe, 0), intel_crtc->unpin_work->gtt_offset);
11103         POSTING_READ(PLANE_SURF(pipe, 0));
11104 }
11105
11106 static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc)
11107 {
11108         struct drm_device *dev = intel_crtc->base.dev;
11109         struct drm_i915_private *dev_priv = dev->dev_private;
11110         struct intel_framebuffer *intel_fb =
11111                 to_intel_framebuffer(intel_crtc->base.primary->fb);
11112         struct drm_i915_gem_object *obj = intel_fb->obj;
11113         u32 dspcntr;
11114         u32 reg;
11115
11116         reg = DSPCNTR(intel_crtc->plane);
11117         dspcntr = I915_READ(reg);
11118
11119         if (obj->tiling_mode != I915_TILING_NONE)
11120                 dspcntr |= DISPPLANE_TILED;
11121         else
11122                 dspcntr &= ~DISPPLANE_TILED;
11123
11124         I915_WRITE(reg, dspcntr);
11125
11126         I915_WRITE(DSPSURF(intel_crtc->plane),
11127                    intel_crtc->unpin_work->gtt_offset);
11128         POSTING_READ(DSPSURF(intel_crtc->plane));
11129
11130 }
11131
11132 /*
11133  * XXX: This is the temporary way to update the plane registers until we get
11134  * around to using the usual plane update functions for MMIO flips
11135  */
11136 static void intel_do_mmio_flip(struct intel_crtc *intel_crtc)
11137 {
11138         struct drm_device *dev = intel_crtc->base.dev;
11139         bool atomic_update;
11140         u32 start_vbl_count;
11141
11142         intel_mark_page_flip_active(intel_crtc);
11143
11144         atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count);
11145
11146         if (INTEL_INFO(dev)->gen >= 9)
11147                 skl_do_mmio_flip(intel_crtc);
11148         else
11149                 /* use_mmio_flip() retricts MMIO flips to ilk+ */
11150                 ilk_do_mmio_flip(intel_crtc);
11151
11152         if (atomic_update)
11153                 intel_pipe_update_end(intel_crtc, start_vbl_count);
11154 }
11155
11156 static void intel_mmio_flip_work_func(struct work_struct *work)
11157 {
11158         struct intel_mmio_flip *mmio_flip =
11159                 container_of(work, struct intel_mmio_flip, work);
11160
11161         if (mmio_flip->req)
11162                 WARN_ON(__i915_wait_request(mmio_flip->req,
11163                                             mmio_flip->crtc->reset_counter,
11164                                             false, NULL,
11165                                             &mmio_flip->i915->rps.mmioflips));
11166
11167         intel_do_mmio_flip(mmio_flip->crtc);
11168
11169         i915_gem_request_unreference__unlocked(mmio_flip->req);
11170         kfree(mmio_flip);
11171 }
11172
11173 static int intel_queue_mmio_flip(struct drm_device *dev,
11174                                  struct drm_crtc *crtc,
11175                                  struct drm_framebuffer *fb,
11176                                  struct drm_i915_gem_object *obj,
11177                                  struct intel_engine_cs *ring,
11178                                  uint32_t flags)
11179 {
11180         struct intel_mmio_flip *mmio_flip;
11181
11182         mmio_flip = kmalloc(sizeof(*mmio_flip), GFP_KERNEL);
11183         if (mmio_flip == NULL)
11184                 return -ENOMEM;
11185
11186         mmio_flip->i915 = to_i915(dev);
11187         mmio_flip->req = i915_gem_request_reference(obj->last_write_req);
11188         mmio_flip->crtc = to_intel_crtc(crtc);
11189
11190         INIT_WORK(&mmio_flip->work, intel_mmio_flip_work_func);
11191         schedule_work(&mmio_flip->work);
11192
11193         return 0;
11194 }
11195
11196 static int intel_default_queue_flip(struct drm_device *dev,
11197                                     struct drm_crtc *crtc,
11198                                     struct drm_framebuffer *fb,
11199                                     struct drm_i915_gem_object *obj,
11200                                     struct drm_i915_gem_request *req,
11201                                     uint32_t flags)
11202 {
11203         return -ENODEV;
11204 }
11205
11206 static bool __intel_pageflip_stall_check(struct drm_device *dev,
11207                                          struct drm_crtc *crtc)
11208 {
11209         struct drm_i915_private *dev_priv = dev->dev_private;
11210         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11211         struct intel_unpin_work *work = intel_crtc->unpin_work;
11212         u32 addr;
11213
11214         if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE)
11215                 return true;
11216
11217         if (!work->enable_stall_check)
11218                 return false;
11219
11220         if (work->flip_ready_vblank == 0) {
11221                 if (work->flip_queued_req &&
11222                     !i915_gem_request_completed(work->flip_queued_req, true))
11223                         return false;
11224
11225                 work->flip_ready_vblank = drm_crtc_vblank_count(crtc);
11226         }
11227
11228         if (drm_crtc_vblank_count(crtc) - work->flip_ready_vblank < 3)
11229                 return false;
11230
11231         /* Potential stall - if we see that the flip has happened,
11232          * assume a missed interrupt. */
11233         if (INTEL_INFO(dev)->gen >= 4)
11234                 addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane)));
11235         else
11236                 addr = I915_READ(DSPADDR(intel_crtc->plane));
11237
11238         /* There is a potential issue here with a false positive after a flip
11239          * to the same address. We could address this by checking for a
11240          * non-incrementing frame counter.
11241          */
11242         return addr == work->gtt_offset;
11243 }
11244
11245 void intel_check_page_flip(struct drm_device *dev, int pipe)
11246 {
11247         struct drm_i915_private *dev_priv = dev->dev_private;
11248         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
11249         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11250         struct intel_unpin_work *work;
11251
11252         WARN_ON(!in_interrupt());
11253
11254         if (crtc == NULL)
11255                 return;
11256
11257         spin_lock(&dev->event_lock);
11258         work = intel_crtc->unpin_work;
11259         if (work != NULL && __intel_pageflip_stall_check(dev, crtc)) {
11260                 WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n",
11261                          work->flip_queued_vblank, drm_vblank_count(dev, pipe));
11262                 page_flip_completed(intel_crtc);
11263                 work = NULL;
11264         }
11265         if (work != NULL &&
11266             drm_vblank_count(dev, pipe) - work->flip_queued_vblank > 1)
11267                 intel_queue_rps_boost_for_request(dev, work->flip_queued_req);
11268         spin_unlock(&dev->event_lock);
11269 }
11270
11271 static int intel_crtc_page_flip(struct drm_crtc *crtc,
11272                                 struct drm_framebuffer *fb,
11273                                 struct drm_pending_vblank_event *event,
11274                                 uint32_t page_flip_flags)
11275 {
11276         struct drm_device *dev = crtc->dev;
11277         struct drm_i915_private *dev_priv = dev->dev_private;
11278         struct drm_framebuffer *old_fb = crtc->primary->fb;
11279         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
11280         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11281         struct drm_plane *primary = crtc->primary;
11282         enum pipe pipe = intel_crtc->pipe;
11283         struct intel_unpin_work *work;
11284         struct intel_engine_cs *ring;
11285         bool mmio_flip;
11286         struct drm_i915_gem_request *request = NULL;
11287         int ret;
11288
11289         /*
11290          * drm_mode_page_flip_ioctl() should already catch this, but double
11291          * check to be safe.  In the future we may enable pageflipping from
11292          * a disabled primary plane.
11293          */
11294         if (WARN_ON(intel_fb_obj(old_fb) == NULL))
11295                 return -EBUSY;
11296
11297         /* Can't change pixel format via MI display flips. */
11298         if (fb->pixel_format != crtc->primary->fb->pixel_format)
11299                 return -EINVAL;
11300
11301         /*
11302          * TILEOFF/LINOFF registers can't be changed via MI display flips.
11303          * Note that pitch changes could also affect these register.
11304          */
11305         if (INTEL_INFO(dev)->gen > 3 &&
11306             (fb->offsets[0] != crtc->primary->fb->offsets[0] ||
11307              fb->pitches[0] != crtc->primary->fb->pitches[0]))
11308                 return -EINVAL;
11309
11310         if (i915_terminally_wedged(&dev_priv->gpu_error))
11311                 goto out_hang;
11312
11313         work = kzalloc(sizeof(*work), GFP_KERNEL);
11314         if (work == NULL)
11315                 return -ENOMEM;
11316
11317         work->event = event;
11318         work->crtc = crtc;
11319         work->old_fb = old_fb;
11320         INIT_WORK(&work->work, intel_unpin_work_fn);
11321
11322         ret = drm_crtc_vblank_get(crtc);
11323         if (ret)
11324                 goto free_work;
11325
11326         /* We borrow the event spin lock for protecting unpin_work */
11327         spin_lock_irq(&dev->event_lock);
11328         if (intel_crtc->unpin_work) {
11329                 /* Before declaring the flip queue wedged, check if
11330                  * the hardware completed the operation behind our backs.
11331                  */
11332                 if (__intel_pageflip_stall_check(dev, crtc)) {
11333                         DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n");
11334                         page_flip_completed(intel_crtc);
11335                 } else {
11336                         DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
11337                         spin_unlock_irq(&dev->event_lock);
11338
11339                         drm_crtc_vblank_put(crtc);
11340                         kfree(work);
11341                         return -EBUSY;
11342                 }
11343         }
11344         intel_crtc->unpin_work = work;
11345         spin_unlock_irq(&dev->event_lock);
11346
11347         if (atomic_read(&intel_crtc->unpin_work_count) >= 2)
11348                 flush_workqueue(dev_priv->wq);
11349
11350         /* Reference the objects for the scheduled work. */
11351         drm_framebuffer_reference(work->old_fb);
11352         drm_gem_object_reference(&obj->base);
11353
11354         crtc->primary->fb = fb;
11355         update_state_fb(crtc->primary);
11356
11357         work->pending_flip_obj = obj;
11358
11359         ret = i915_mutex_lock_interruptible(dev);
11360         if (ret)
11361                 goto cleanup;
11362
11363         atomic_inc(&intel_crtc->unpin_work_count);
11364         intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
11365
11366         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
11367                 work->flip_count = I915_READ(PIPE_FLIPCOUNT_GM45(pipe)) + 1;
11368
11369         if (IS_VALLEYVIEW(dev)) {
11370                 ring = &dev_priv->ring[BCS];
11371                 if (obj->tiling_mode != intel_fb_obj(work->old_fb)->tiling_mode)
11372                         /* vlv: DISPLAY_FLIP fails to change tiling */
11373                         ring = NULL;
11374         } else if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
11375                 ring = &dev_priv->ring[BCS];
11376         } else if (INTEL_INFO(dev)->gen >= 7) {
11377                 ring = i915_gem_request_get_ring(obj->last_write_req);
11378                 if (ring == NULL || ring->id != RCS)
11379                         ring = &dev_priv->ring[BCS];
11380         } else {
11381                 ring = &dev_priv->ring[RCS];
11382         }
11383
11384         mmio_flip = use_mmio_flip(ring, obj);
11385
11386         /* When using CS flips, we want to emit semaphores between rings.
11387          * However, when using mmio flips we will create a task to do the
11388          * synchronisation, so all we want here is to pin the framebuffer
11389          * into the display plane and skip any waits.
11390          */
11391         ret = intel_pin_and_fence_fb_obj(crtc->primary, fb,
11392                                          crtc->primary->state,
11393                                          mmio_flip ? i915_gem_request_get_ring(obj->last_write_req) : ring, &request);
11394         if (ret)
11395                 goto cleanup_pending;
11396
11397         work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary), obj)
11398                                                   + intel_crtc->dspaddr_offset;
11399
11400         if (mmio_flip) {
11401                 ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring,
11402                                             page_flip_flags);
11403                 if (ret)
11404                         goto cleanup_unpin;
11405
11406                 i915_gem_request_assign(&work->flip_queued_req,
11407                                         obj->last_write_req);
11408         } else {
11409                 if (!request) {
11410                         ret = i915_gem_request_alloc(ring, ring->default_context, &request);
11411                         if (ret)
11412                                 goto cleanup_unpin;
11413                 }
11414
11415                 ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, request,
11416                                                    page_flip_flags);
11417                 if (ret)
11418                         goto cleanup_unpin;
11419
11420                 i915_gem_request_assign(&work->flip_queued_req, request);
11421         }
11422
11423         if (request)
11424                 i915_add_request_no_flush(request);
11425
11426         work->flip_queued_vblank = drm_crtc_vblank_count(crtc);
11427         work->enable_stall_check = true;
11428
11429         i915_gem_track_fb(intel_fb_obj(work->old_fb), obj,
11430                           INTEL_FRONTBUFFER_PRIMARY(pipe));
11431
11432         intel_fbc_disable(dev);
11433         intel_frontbuffer_flip_prepare(dev, INTEL_FRONTBUFFER_PRIMARY(pipe));
11434         mutex_unlock(&dev->struct_mutex);
11435
11436         trace_i915_flip_request(intel_crtc->plane, obj);
11437
11438         return 0;
11439
11440 cleanup_unpin:
11441         intel_unpin_fb_obj(fb, crtc->primary->state);
11442 cleanup_pending:
11443         if (request)
11444                 i915_gem_request_cancel(request);
11445         atomic_dec(&intel_crtc->unpin_work_count);
11446         mutex_unlock(&dev->struct_mutex);
11447 cleanup:
11448         crtc->primary->fb = old_fb;
11449         update_state_fb(crtc->primary);
11450
11451         drm_gem_object_unreference_unlocked(&obj->base);
11452         drm_framebuffer_unreference(work->old_fb);
11453
11454         spin_lock_irq(&dev->event_lock);
11455         intel_crtc->unpin_work = NULL;
11456         spin_unlock_irq(&dev->event_lock);
11457
11458         drm_crtc_vblank_put(crtc);
11459 free_work:
11460         kfree(work);
11461
11462         if (ret == -EIO) {
11463                 struct drm_atomic_state *state;
11464                 struct drm_plane_state *plane_state;
11465
11466 out_hang:
11467                 state = drm_atomic_state_alloc(dev);
11468                 if (!state)
11469                         return -ENOMEM;
11470                 state->acquire_ctx = drm_modeset_legacy_acquire_ctx(crtc);
11471
11472 retry:
11473                 plane_state = drm_atomic_get_plane_state(state, primary);
11474                 ret = PTR_ERR_OR_ZERO(plane_state);
11475                 if (!ret) {
11476                         drm_atomic_set_fb_for_plane(plane_state, fb);
11477
11478                         ret = drm_atomic_set_crtc_for_plane(plane_state, crtc);
11479                         if (!ret)
11480                                 ret = drm_atomic_commit(state);
11481                 }
11482
11483                 if (ret == -EDEADLK) {
11484                         drm_modeset_backoff(state->acquire_ctx);
11485                         drm_atomic_state_clear(state);
11486                         goto retry;
11487                 }
11488
11489                 if (ret)
11490                         drm_atomic_state_free(state);
11491
11492                 if (ret == 0 && event) {
11493                         spin_lock_irq(&dev->event_lock);
11494                         drm_send_vblank_event(dev, pipe, event);
11495                         spin_unlock_irq(&dev->event_lock);
11496                 }
11497         }
11498         return ret;
11499 }
11500
11501
11502 /**
11503  * intel_wm_need_update - Check whether watermarks need updating
11504  * @plane: drm plane
11505  * @state: new plane state
11506  *
11507  * Check current plane state versus the new one to determine whether
11508  * watermarks need to be recalculated.
11509  *
11510  * Returns true or false.
11511  */
11512 static bool intel_wm_need_update(struct drm_plane *plane,
11513                                  struct drm_plane_state *state)
11514 {
11515         /* Update watermarks on tiling changes. */
11516         if (!plane->state->fb || !state->fb ||
11517             plane->state->fb->modifier[0] != state->fb->modifier[0] ||
11518             plane->state->rotation != state->rotation)
11519                 return true;
11520
11521         if (plane->state->crtc_w != state->crtc_w)
11522                 return true;
11523
11524         return false;
11525 }
11526
11527 int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
11528                                     struct drm_plane_state *plane_state)
11529 {
11530         struct drm_crtc *crtc = crtc_state->crtc;
11531         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11532         struct drm_plane *plane = plane_state->plane;
11533         struct drm_device *dev = crtc->dev;
11534         struct drm_i915_private *dev_priv = dev->dev_private;
11535         struct intel_plane_state *old_plane_state =
11536                 to_intel_plane_state(plane->state);
11537         int idx = intel_crtc->base.base.id, ret;
11538         int i = drm_plane_index(plane);
11539         bool mode_changed = needs_modeset(crtc_state);
11540         bool was_crtc_enabled = crtc->state->active;
11541         bool is_crtc_enabled = crtc_state->active;
11542
11543         bool turn_off, turn_on, visible, was_visible;
11544         struct drm_framebuffer *fb = plane_state->fb;
11545
11546         if (crtc_state && INTEL_INFO(dev)->gen >= 9 &&
11547             plane->type != DRM_PLANE_TYPE_CURSOR) {
11548                 ret = skl_update_scaler_plane(
11549                         to_intel_crtc_state(crtc_state),
11550                         to_intel_plane_state(plane_state));
11551                 if (ret)
11552                         return ret;
11553         }
11554
11555         /*
11556          * Disabling a plane is always okay; we just need to update
11557          * fb tracking in a special way since cleanup_fb() won't
11558          * get called by the plane helpers.
11559          */
11560         if (old_plane_state->base.fb && !fb)
11561                 intel_crtc->atomic.disabled_planes |= 1 << i;
11562
11563         was_visible = old_plane_state->visible;
11564         visible = to_intel_plane_state(plane_state)->visible;
11565
11566         if (!was_crtc_enabled && WARN_ON(was_visible))
11567                 was_visible = false;
11568
11569         if (!is_crtc_enabled && WARN_ON(visible))
11570                 visible = false;
11571
11572         if (!was_visible && !visible)
11573                 return 0;
11574
11575         turn_off = was_visible && (!visible || mode_changed);
11576         turn_on = visible && (!was_visible || mode_changed);
11577
11578         DRM_DEBUG_ATOMIC("[CRTC:%i] has [PLANE:%i] with fb %i\n", idx,
11579                          plane->base.id, fb ? fb->base.id : -1);
11580
11581         DRM_DEBUG_ATOMIC("[PLANE:%i] visible %i -> %i, off %i, on %i, ms %i\n",
11582                          plane->base.id, was_visible, visible,
11583                          turn_off, turn_on, mode_changed);
11584
11585         if (intel_wm_need_update(plane, plane_state))
11586                 intel_crtc->atomic.update_wm = true;
11587
11588         switch (plane->type) {
11589         case DRM_PLANE_TYPE_PRIMARY:
11590                 if (visible)
11591                         intel_crtc->atomic.fb_bits |=
11592                             INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe);
11593
11594                 intel_crtc->atomic.wait_for_flips = true;
11595                 intel_crtc->atomic.pre_disable_primary = turn_off;
11596                 intel_crtc->atomic.post_enable_primary = turn_on;
11597
11598                 if (turn_off)
11599                         intel_crtc->atomic.disable_fbc = true;
11600
11601                 /*
11602                  * FBC does not work on some platforms for rotated
11603                  * planes, so disable it when rotation is not 0 and
11604                  * update it when rotation is set back to 0.
11605                  *
11606                  * FIXME: This is redundant with the fbc update done in
11607                  * the primary plane enable function except that that
11608                  * one is done too late. We eventually need to unify
11609                  * this.
11610                  */
11611
11612                 if (visible &&
11613                     INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) &&
11614                     dev_priv->fbc.crtc == intel_crtc &&
11615                     plane_state->rotation != BIT(DRM_ROTATE_0))
11616                         intel_crtc->atomic.disable_fbc = true;
11617
11618                 /*
11619                  * BDW signals flip done immediately if the plane
11620                  * is disabled, even if the plane enable is already
11621                  * armed to occur at the next vblank :(
11622                  */
11623                 if (turn_on && IS_BROADWELL(dev))
11624                         intel_crtc->atomic.wait_vblank = true;
11625
11626                 intel_crtc->atomic.update_fbc |= visible || mode_changed;
11627                 break;
11628         case DRM_PLANE_TYPE_CURSOR:
11629                 if (visible)
11630                         intel_crtc->atomic.fb_bits |=
11631                             INTEL_FRONTBUFFER_CURSOR(intel_crtc->pipe);
11632                 break;
11633         case DRM_PLANE_TYPE_OVERLAY:
11634                 /*
11635                  * 'prepare' is never called when plane is being disabled, so
11636                  * we need to handle frontbuffer tracking as a special case
11637                  */
11638                 if (visible)
11639                         intel_crtc->atomic.fb_bits |=
11640                             INTEL_FRONTBUFFER_SPRITE(intel_crtc->pipe);
11641
11642                 if (turn_off && !mode_changed) {
11643                         intel_crtc->atomic.wait_vblank = true;
11644                         intel_crtc->atomic.update_sprite_watermarks |=
11645                                 1 << i;
11646                 }
11647                 break;
11648         }
11649         return 0;
11650 }
11651
11652 static bool encoders_cloneable(const struct intel_encoder *a,
11653                                const struct intel_encoder *b)
11654 {
11655         /* masks could be asymmetric, so check both ways */
11656         return a == b || (a->cloneable & (1 << b->type) &&
11657                           b->cloneable & (1 << a->type));
11658 }
11659
11660 static bool check_single_encoder_cloning(struct drm_atomic_state *state,
11661                                          struct intel_crtc *crtc,
11662                                          struct intel_encoder *encoder)
11663 {
11664         struct intel_encoder *source_encoder;
11665         struct drm_connector *connector;
11666         struct drm_connector_state *connector_state;
11667         int i;
11668
11669         for_each_connector_in_state(state, connector, connector_state, i) {
11670                 if (connector_state->crtc != &crtc->base)
11671                         continue;
11672
11673                 source_encoder =
11674                         to_intel_encoder(connector_state->best_encoder);
11675                 if (!encoders_cloneable(encoder, source_encoder))
11676                         return false;
11677         }
11678
11679         return true;
11680 }
11681
11682 static bool check_encoder_cloning(struct drm_atomic_state *state,
11683                                   struct intel_crtc *crtc)
11684 {
11685         struct intel_encoder *encoder;
11686         struct drm_connector *connector;
11687         struct drm_connector_state *connector_state;
11688         int i;
11689
11690         for_each_connector_in_state(state, connector, connector_state, i) {
11691                 if (connector_state->crtc != &crtc->base)
11692                         continue;
11693
11694                 encoder = to_intel_encoder(connector_state->best_encoder);
11695                 if (!check_single_encoder_cloning(state, crtc, encoder))
11696                         return false;
11697         }
11698
11699         return true;
11700 }
11701
11702 static void intel_crtc_check_initial_planes(struct drm_crtc *crtc,
11703                                             struct drm_crtc_state *crtc_state)
11704 {
11705         struct intel_crtc_state *pipe_config =
11706                 to_intel_crtc_state(crtc_state);
11707         struct drm_plane *p;
11708         unsigned visible_mask = 0;
11709
11710         drm_for_each_plane_mask(p, crtc->dev, crtc_state->plane_mask) {
11711                 struct drm_plane_state *plane_state =
11712                         drm_atomic_get_existing_plane_state(crtc_state->state, p);
11713
11714                 if (WARN_ON(!plane_state))
11715                         continue;
11716
11717                 if (!plane_state->fb)
11718                         crtc_state->plane_mask &=
11719                                 ~(1 << drm_plane_index(p));
11720                 else if (to_intel_plane_state(plane_state)->visible)
11721                         visible_mask |= 1 << drm_plane_index(p);
11722         }
11723
11724         if (!visible_mask)
11725                 return;
11726
11727         pipe_config->quirks &= ~PIPE_CONFIG_QUIRK_INITIAL_PLANES;
11728 }
11729
11730 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
11731                                    struct drm_crtc_state *crtc_state)
11732 {
11733         struct drm_device *dev = crtc->dev;
11734         struct drm_i915_private *dev_priv = dev->dev_private;
11735         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
11736         struct intel_crtc_state *pipe_config =
11737                 to_intel_crtc_state(crtc_state);
11738         struct drm_atomic_state *state = crtc_state->state;
11739         int ret, idx = crtc->base.id;
11740         bool mode_changed = needs_modeset(crtc_state);
11741
11742         if (mode_changed && !check_encoder_cloning(state, intel_crtc)) {
11743                 DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
11744                 return -EINVAL;
11745         }
11746
11747         I915_STATE_WARN(crtc->state->active != intel_crtc->active,
11748                 "[CRTC:%i] mismatch between state->active(%i) and crtc->active(%i)\n",
11749                 idx, crtc->state->active, intel_crtc->active);
11750
11751         /* plane mask is fixed up after all initial planes are calculated */
11752         if (pipe_config->quirks & PIPE_CONFIG_QUIRK_INITIAL_PLANES)
11753                 intel_crtc_check_initial_planes(crtc, crtc_state);
11754
11755         if (mode_changed)
11756                 intel_crtc->atomic.update_wm = !crtc_state->active;
11757
11758         if (mode_changed && crtc_state->enable &&
11759             dev_priv->display.crtc_compute_clock &&
11760             !WARN_ON(pipe_config->shared_dpll != DPLL_ID_PRIVATE)) {
11761                 ret = dev_priv->display.crtc_compute_clock(intel_crtc,
11762                                                            pipe_config);
11763                 if (ret)
11764                         return ret;
11765         }
11766
11767         return intel_atomic_setup_scalers(dev, intel_crtc, pipe_config);
11768 }
11769
11770 static const struct drm_crtc_helper_funcs intel_helper_funcs = {
11771         .mode_set_base_atomic = intel_pipe_set_base_atomic,
11772         .load_lut = intel_crtc_load_lut,
11773         .atomic_begin = intel_begin_crtc_commit,
11774         .atomic_flush = intel_finish_crtc_commit,
11775         .atomic_check = intel_crtc_atomic_check,
11776 };
11777
11778 /**
11779  * intel_modeset_update_staged_output_state
11780  *
11781  * Updates the staged output configuration state, e.g. after we've read out the
11782  * current hw state.
11783  */
11784 static void intel_modeset_update_staged_output_state(struct drm_device *dev)
11785 {
11786         struct intel_crtc *crtc;
11787         struct intel_encoder *encoder;
11788         struct intel_connector *connector;
11789
11790         for_each_intel_connector(dev, connector) {
11791                 connector->new_encoder =
11792                         to_intel_encoder(connector->base.encoder);
11793         }
11794
11795         for_each_intel_encoder(dev, encoder) {
11796                 encoder->new_crtc =
11797                         to_intel_crtc(encoder->base.crtc);
11798         }
11799
11800         for_each_intel_crtc(dev, crtc) {
11801                 crtc->new_enabled = crtc->base.state->enable;
11802         }
11803 }
11804
11805 /* Transitional helper to copy current connector/encoder state to
11806  * connector->state. This is needed so that code that is partially
11807  * converted to atomic does the right thing.
11808  */
11809 static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
11810 {
11811         struct intel_connector *connector;
11812
11813         for_each_intel_connector(dev, connector) {
11814                 if (connector->base.encoder) {
11815                         connector->base.state->best_encoder =
11816                                 connector->base.encoder;
11817                         connector->base.state->crtc =
11818                                 connector->base.encoder->crtc;
11819                 } else {
11820                         connector->base.state->best_encoder = NULL;
11821                         connector->base.state->crtc = NULL;
11822                 }
11823         }
11824 }
11825
11826 static void
11827 connected_sink_compute_bpp(struct intel_connector *connector,
11828                            struct intel_crtc_state *pipe_config)
11829 {
11830         int bpp = pipe_config->pipe_bpp;
11831
11832         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] checking for sink bpp constrains\n",
11833                 connector->base.base.id,
11834                 connector->base.name);
11835
11836         /* Don't use an invalid EDID bpc value */
11837         if (connector->base.display_info.bpc &&
11838             connector->base.display_info.bpc * 3 < bpp) {
11839                 DRM_DEBUG_KMS("clamping display bpp (was %d) to EDID reported max of %d\n",
11840                               bpp, connector->base.display_info.bpc*3);
11841                 pipe_config->pipe_bpp = connector->base.display_info.bpc*3;
11842         }
11843
11844         /* Clamp bpp to 8 on screens without EDID 1.4 */
11845         if (connector->base.display_info.bpc == 0 && bpp > 24) {
11846                 DRM_DEBUG_KMS("clamping display bpp (was %d) to default limit of 24\n",
11847                               bpp);
11848                 pipe_config->pipe_bpp = 24;
11849         }
11850 }
11851
11852 static int
11853 compute_baseline_pipe_bpp(struct intel_crtc *crtc,
11854                           struct intel_crtc_state *pipe_config)
11855 {
11856         struct drm_device *dev = crtc->base.dev;
11857         struct drm_atomic_state *state;
11858         struct drm_connector *connector;
11859         struct drm_connector_state *connector_state;
11860         int bpp, i;
11861
11862         if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)))
11863                 bpp = 10*3;
11864         else if (INTEL_INFO(dev)->gen >= 5)
11865                 bpp = 12*3;
11866         else
11867                 bpp = 8*3;
11868
11869
11870         pipe_config->pipe_bpp = bpp;
11871
11872         state = pipe_config->base.state;
11873
11874         /* Clamp display bpp to EDID value */
11875         for_each_connector_in_state(state, connector, connector_state, i) {
11876                 if (connector_state->crtc != &crtc->base)
11877                         continue;
11878
11879                 connected_sink_compute_bpp(to_intel_connector(connector),
11880                                            pipe_config);
11881         }
11882
11883         return bpp;
11884 }
11885
11886 static void intel_dump_crtc_timings(const struct drm_display_mode *mode)
11887 {
11888         DRM_DEBUG_KMS("crtc timings: %d %d %d %d %d %d %d %d %d, "
11889                         "type: 0x%x flags: 0x%x\n",
11890                 mode->crtc_clock,
11891                 mode->crtc_hdisplay, mode->crtc_hsync_start,
11892                 mode->crtc_hsync_end, mode->crtc_htotal,
11893                 mode->crtc_vdisplay, mode->crtc_vsync_start,
11894                 mode->crtc_vsync_end, mode->crtc_vtotal, mode->type, mode->flags);
11895 }
11896
11897 static void intel_dump_pipe_config(struct intel_crtc *crtc,
11898                                    struct intel_crtc_state *pipe_config,
11899                                    const char *context)
11900 {
11901         struct drm_device *dev = crtc->base.dev;
11902         struct drm_plane *plane;
11903         struct intel_plane *intel_plane;
11904         struct intel_plane_state *state;
11905         struct drm_framebuffer *fb;
11906
11907         DRM_DEBUG_KMS("[CRTC:%d]%s config %p for pipe %c\n", crtc->base.base.id,
11908                       context, pipe_config, pipe_name(crtc->pipe));
11909
11910         DRM_DEBUG_KMS("cpu_transcoder: %c\n", transcoder_name(pipe_config->cpu_transcoder));
11911         DRM_DEBUG_KMS("pipe bpp: %i, dithering: %i\n",
11912                       pipe_config->pipe_bpp, pipe_config->dither);
11913         DRM_DEBUG_KMS("fdi/pch: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
11914                       pipe_config->has_pch_encoder,
11915                       pipe_config->fdi_lanes,
11916                       pipe_config->fdi_m_n.gmch_m, pipe_config->fdi_m_n.gmch_n,
11917                       pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n,
11918                       pipe_config->fdi_m_n.tu);
11919         DRM_DEBUG_KMS("dp: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
11920                       pipe_config->has_dp_encoder,
11921                       pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n,
11922                       pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n,
11923                       pipe_config->dp_m_n.tu);
11924
11925         DRM_DEBUG_KMS("dp: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
11926                       pipe_config->has_dp_encoder,
11927                       pipe_config->dp_m2_n2.gmch_m,
11928                       pipe_config->dp_m2_n2.gmch_n,
11929                       pipe_config->dp_m2_n2.link_m,
11930                       pipe_config->dp_m2_n2.link_n,
11931                       pipe_config->dp_m2_n2.tu);
11932
11933         DRM_DEBUG_KMS("audio: %i, infoframes: %i\n",
11934                       pipe_config->has_audio,
11935                       pipe_config->has_infoframe);
11936
11937         DRM_DEBUG_KMS("requested mode:\n");
11938         drm_mode_debug_printmodeline(&pipe_config->base.mode);
11939         DRM_DEBUG_KMS("adjusted mode:\n");
11940         drm_mode_debug_printmodeline(&pipe_config->base.adjusted_mode);
11941         intel_dump_crtc_timings(&pipe_config->base.adjusted_mode);
11942         DRM_DEBUG_KMS("port clock: %d\n", pipe_config->port_clock);
11943         DRM_DEBUG_KMS("pipe src size: %dx%d\n",
11944                       pipe_config->pipe_src_w, pipe_config->pipe_src_h);
11945         DRM_DEBUG_KMS("num_scalers: %d, scaler_users: 0x%x, scaler_id: %d\n",
11946                       crtc->num_scalers,
11947                       pipe_config->scaler_state.scaler_users,
11948                       pipe_config->scaler_state.scaler_id);
11949         DRM_DEBUG_KMS("gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
11950                       pipe_config->gmch_pfit.control,
11951                       pipe_config->gmch_pfit.pgm_ratios,
11952                       pipe_config->gmch_pfit.lvds_border_bits);
11953         DRM_DEBUG_KMS("pch pfit: pos: 0x%08x, size: 0x%08x, %s\n",
11954                       pipe_config->pch_pfit.pos,
11955                       pipe_config->pch_pfit.size,
11956                       pipe_config->pch_pfit.enabled ? "enabled" : "disabled");
11957         DRM_DEBUG_KMS("ips: %i\n", pipe_config->ips_enabled);
11958         DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide);
11959
11960         if (IS_BROXTON(dev)) {
11961                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: ebb0: 0x%x, "
11962                               "pll0: 0x%x, pll1: 0x%x, pll2: 0x%x, pll3: 0x%x, "
11963                               "pll6: 0x%x, pll8: 0x%x, pcsdw12: 0x%x\n",
11964                               pipe_config->ddi_pll_sel,
11965                               pipe_config->dpll_hw_state.ebb0,
11966                               pipe_config->dpll_hw_state.pll0,
11967                               pipe_config->dpll_hw_state.pll1,
11968                               pipe_config->dpll_hw_state.pll2,
11969                               pipe_config->dpll_hw_state.pll3,
11970                               pipe_config->dpll_hw_state.pll6,
11971                               pipe_config->dpll_hw_state.pll8,
11972                               pipe_config->dpll_hw_state.pcsdw12);
11973         } else if (IS_SKYLAKE(dev)) {
11974                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: "
11975                               "ctrl1: 0x%x, cfgcr1: 0x%x, cfgcr2: 0x%x\n",
11976                               pipe_config->ddi_pll_sel,
11977                               pipe_config->dpll_hw_state.ctrl1,
11978                               pipe_config->dpll_hw_state.cfgcr1,
11979                               pipe_config->dpll_hw_state.cfgcr2);
11980         } else if (HAS_DDI(dev)) {
11981                 DRM_DEBUG_KMS("ddi_pll_sel: %u; dpll_hw_state: wrpll: 0x%x\n",
11982                               pipe_config->ddi_pll_sel,
11983                               pipe_config->dpll_hw_state.wrpll);
11984         } else {
11985                 DRM_DEBUG_KMS("dpll_hw_state: dpll: 0x%x, dpll_md: 0x%x, "
11986                               "fp0: 0x%x, fp1: 0x%x\n",
11987                               pipe_config->dpll_hw_state.dpll,
11988                               pipe_config->dpll_hw_state.dpll_md,
11989                               pipe_config->dpll_hw_state.fp0,
11990                               pipe_config->dpll_hw_state.fp1);
11991         }
11992
11993         DRM_DEBUG_KMS("planes on this crtc\n");
11994         list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
11995                 intel_plane = to_intel_plane(plane);
11996                 if (intel_plane->pipe != crtc->pipe)
11997                         continue;
11998
11999                 state = to_intel_plane_state(plane->state);
12000                 fb = state->base.fb;
12001                 if (!fb) {
12002                         DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d "
12003                                 "disabled, scaler_id = %d\n",
12004                                 plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD",
12005                                 plane->base.id, intel_plane->pipe,
12006                                 (crtc->base.primary == plane) ? 0 : intel_plane->plane + 1,
12007                                 drm_plane_index(plane), state->scaler_id);
12008                         continue;
12009                 }
12010
12011                 DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d enabled",
12012                         plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD",
12013                         plane->base.id, intel_plane->pipe,
12014                         crtc->base.primary == plane ? 0 : intel_plane->plane + 1,
12015                         drm_plane_index(plane));
12016                 DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = 0x%x",
12017                         fb->base.id, fb->width, fb->height, fb->pixel_format);
12018                 DRM_DEBUG_KMS("\tscaler:%d src (%u, %u) %ux%u dst (%u, %u) %ux%u\n",
12019                         state->scaler_id,
12020                         state->src.x1 >> 16, state->src.y1 >> 16,
12021                         drm_rect_width(&state->src) >> 16,
12022                         drm_rect_height(&state->src) >> 16,
12023                         state->dst.x1, state->dst.y1,
12024                         drm_rect_width(&state->dst), drm_rect_height(&state->dst));
12025         }
12026 }
12027
12028 static bool check_digital_port_conflicts(struct drm_atomic_state *state)
12029 {
12030         struct drm_device *dev = state->dev;
12031         struct intel_encoder *encoder;
12032         struct drm_connector *connector;
12033         struct drm_connector_state *connector_state;
12034         unsigned int used_ports = 0;
12035         int i;
12036
12037         /*
12038          * Walk the connector list instead of the encoder
12039          * list to detect the problem on ddi platforms
12040          * where there's just one encoder per digital port.
12041          */
12042         for_each_connector_in_state(state, connector, connector_state, i) {
12043                 if (!connector_state->best_encoder)
12044                         continue;
12045
12046                 encoder = to_intel_encoder(connector_state->best_encoder);
12047
12048                 WARN_ON(!connector_state->crtc);
12049
12050                 switch (encoder->type) {
12051                         unsigned int port_mask;
12052                 case INTEL_OUTPUT_UNKNOWN:
12053                         if (WARN_ON(!HAS_DDI(dev)))
12054                                 break;
12055                 case INTEL_OUTPUT_DISPLAYPORT:
12056                 case INTEL_OUTPUT_HDMI:
12057                 case INTEL_OUTPUT_EDP:
12058                         port_mask = 1 << enc_to_dig_port(&encoder->base)->port;
12059
12060                         /* the same port mustn't appear more than once */
12061                         if (used_ports & port_mask)
12062                                 return false;
12063
12064                         used_ports |= port_mask;
12065                 default:
12066                         break;
12067                 }
12068         }
12069
12070         return true;
12071 }
12072
12073 static void
12074 clear_intel_crtc_state(struct intel_crtc_state *crtc_state)
12075 {
12076         struct drm_crtc_state tmp_state;
12077         struct intel_crtc_scaler_state scaler_state;
12078         struct intel_dpll_hw_state dpll_hw_state;
12079         enum intel_dpll_id shared_dpll;
12080         uint32_t ddi_pll_sel;
12081
12082         /* FIXME: before the switch to atomic started, a new pipe_config was
12083          * kzalloc'd. Code that depends on any field being zero should be
12084          * fixed, so that the crtc_state can be safely duplicated. For now,
12085          * only fields that are know to not cause problems are preserved. */
12086
12087         tmp_state = crtc_state->base;
12088         scaler_state = crtc_state->scaler_state;
12089         shared_dpll = crtc_state->shared_dpll;
12090         dpll_hw_state = crtc_state->dpll_hw_state;
12091         ddi_pll_sel = crtc_state->ddi_pll_sel;
12092
12093         memset(crtc_state, 0, sizeof *crtc_state);
12094
12095         crtc_state->base = tmp_state;
12096         crtc_state->scaler_state = scaler_state;
12097         crtc_state->shared_dpll = shared_dpll;
12098         crtc_state->dpll_hw_state = dpll_hw_state;
12099         crtc_state->ddi_pll_sel = ddi_pll_sel;
12100 }
12101
12102 static int
12103 intel_modeset_pipe_config(struct drm_crtc *crtc,
12104                           struct intel_crtc_state *pipe_config)
12105 {
12106         struct drm_atomic_state *state = pipe_config->base.state;
12107         struct intel_encoder *encoder;
12108         struct drm_connector *connector;
12109         struct drm_connector_state *connector_state;
12110         int base_bpp, ret = -EINVAL;
12111         int i;
12112         bool retry = true;
12113
12114         clear_intel_crtc_state(pipe_config);
12115
12116         pipe_config->cpu_transcoder =
12117                 (enum transcoder) to_intel_crtc(crtc)->pipe;
12118
12119         /*
12120          * Sanitize sync polarity flags based on requested ones. If neither
12121          * positive or negative polarity is requested, treat this as meaning
12122          * negative polarity.
12123          */
12124         if (!(pipe_config->base.adjusted_mode.flags &
12125               (DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NHSYNC)))
12126                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NHSYNC;
12127
12128         if (!(pipe_config->base.adjusted_mode.flags &
12129               (DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_NVSYNC)))
12130                 pipe_config->base.adjusted_mode.flags |= DRM_MODE_FLAG_NVSYNC;
12131
12132         /* Compute a starting value for pipe_config->pipe_bpp taking the source
12133          * plane pixel format and any sink constraints into account. Returns the
12134          * source plane bpp so that dithering can be selected on mismatches
12135          * after encoders and crtc also have had their say. */
12136         base_bpp = compute_baseline_pipe_bpp(to_intel_crtc(crtc),
12137                                              pipe_config);
12138         if (base_bpp < 0)
12139                 goto fail;
12140
12141         /*
12142          * Determine the real pipe dimensions. Note that stereo modes can
12143          * increase the actual pipe size due to the frame doubling and
12144          * insertion of additional space for blanks between the frame. This
12145          * is stored in the crtc timings. We use the requested mode to do this
12146          * computation to clearly distinguish it from the adjusted mode, which
12147          * can be changed by the connectors in the below retry loop.
12148          */
12149         drm_crtc_get_hv_timing(&pipe_config->base.mode,
12150                                &pipe_config->pipe_src_w,
12151                                &pipe_config->pipe_src_h);
12152
12153 encoder_retry:
12154         /* Ensure the port clock defaults are reset when retrying. */
12155         pipe_config->port_clock = 0;
12156         pipe_config->pixel_multiplier = 1;
12157
12158         /* Fill in default crtc timings, allow encoders to overwrite them. */
12159         drm_mode_set_crtcinfo(&pipe_config->base.adjusted_mode,
12160                               CRTC_STEREO_DOUBLE);
12161
12162         /* Pass our mode to the connectors and the CRTC to give them a chance to
12163          * adjust it according to limitations or connector properties, and also
12164          * a chance to reject the mode entirely.
12165          */
12166         for_each_connector_in_state(state, connector, connector_state, i) {
12167                 if (connector_state->crtc != crtc)
12168                         continue;
12169
12170                 encoder = to_intel_encoder(connector_state->best_encoder);
12171
12172                 if (!(encoder->compute_config(encoder, pipe_config))) {
12173                         DRM_DEBUG_KMS("Encoder config failure\n");
12174                         goto fail;
12175                 }
12176         }
12177
12178         /* Set default port clock if not overwritten by the encoder. Needs to be
12179          * done afterwards in case the encoder adjusts the mode. */
12180         if (!pipe_config->port_clock)
12181                 pipe_config->port_clock = pipe_config->base.adjusted_mode.crtc_clock
12182                         * pipe_config->pixel_multiplier;
12183
12184         ret = intel_crtc_compute_config(to_intel_crtc(crtc), pipe_config);
12185         if (ret < 0) {
12186                 DRM_DEBUG_KMS("CRTC fixup failed\n");
12187                 goto fail;
12188         }
12189
12190         if (ret == RETRY) {
12191                 if (WARN(!retry, "loop in pipe configuration computation\n")) {
12192                         ret = -EINVAL;
12193                         goto fail;
12194                 }
12195
12196                 DRM_DEBUG_KMS("CRTC bw constrained, retrying\n");
12197                 retry = false;
12198                 goto encoder_retry;
12199         }
12200
12201         pipe_config->dither = pipe_config->pipe_bpp != base_bpp;
12202         DRM_DEBUG_KMS("plane bpp: %i, pipe bpp: %i, dithering: %i\n",
12203                       base_bpp, pipe_config->pipe_bpp, pipe_config->dither);
12204
12205         /* Check if we need to force a modeset */
12206         if (pipe_config->has_audio !=
12207             to_intel_crtc_state(crtc->state)->has_audio) {
12208                 pipe_config->base.mode_changed = true;
12209                 ret = drm_atomic_add_affected_planes(state, crtc);
12210         }
12211
12212         /*
12213          * Note we have an issue here with infoframes: current code
12214          * only updates them on the full mode set path per hw
12215          * requirements.  So here we should be checking for any
12216          * required changes and forcing a mode set.
12217          */
12218 fail:
12219         return ret;
12220 }
12221
12222 static bool intel_crtc_in_use(struct drm_crtc *crtc)
12223 {
12224         struct drm_encoder *encoder;
12225         struct drm_device *dev = crtc->dev;
12226
12227         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head)
12228                 if (encoder->crtc == crtc)
12229                         return true;
12230
12231         return false;
12232 }
12233
12234 static void
12235 intel_modeset_update_state(struct drm_atomic_state *state)
12236 {
12237         struct drm_device *dev = state->dev;
12238         struct intel_encoder *intel_encoder;
12239         struct drm_crtc *crtc;
12240         struct drm_crtc_state *crtc_state;
12241         struct drm_connector *connector;
12242
12243         intel_shared_dpll_commit(state);
12244
12245         for_each_intel_encoder(dev, intel_encoder) {
12246                 if (!intel_encoder->base.crtc)
12247                         continue;
12248
12249                 crtc = intel_encoder->base.crtc;
12250                 crtc_state = drm_atomic_get_existing_crtc_state(state, crtc);
12251                 if (!crtc_state || !needs_modeset(crtc->state))
12252                         continue;
12253
12254                 intel_encoder->connectors_active = false;
12255         }
12256
12257         drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
12258         intel_modeset_update_staged_output_state(state->dev);
12259
12260         /* Double check state. */
12261         for_each_crtc(dev, crtc) {
12262                 WARN_ON(crtc->state->enable != intel_crtc_in_use(crtc));
12263
12264                 to_intel_crtc(crtc)->config = to_intel_crtc_state(crtc->state);
12265
12266                 /* Update hwmode for vblank functions */
12267                 if (crtc->state->active)
12268                         crtc->hwmode = crtc->state->adjusted_mode;
12269                 else
12270                         crtc->hwmode.crtc_clock = 0;
12271         }
12272
12273         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
12274                 if (!connector->encoder || !connector->encoder->crtc)
12275                         continue;
12276
12277                 crtc = connector->encoder->crtc;
12278                 crtc_state = drm_atomic_get_existing_crtc_state(state, crtc);
12279                 if (!crtc_state || !needs_modeset(crtc->state))
12280                         continue;
12281
12282                 if (crtc->state->active) {
12283                         struct drm_property *dpms_property =
12284                                 dev->mode_config.dpms_property;
12285
12286                         connector->dpms = DRM_MODE_DPMS_ON;
12287                         drm_object_property_set_value(&connector->base, dpms_property, DRM_MODE_DPMS_ON);
12288
12289                         intel_encoder = to_intel_encoder(connector->encoder);
12290                         intel_encoder->connectors_active = true;
12291                 } else
12292                         connector->dpms = DRM_MODE_DPMS_OFF;
12293         }
12294 }
12295
12296 static bool intel_fuzzy_clock_check(int clock1, int clock2)
12297 {
12298         int diff;
12299
12300         if (clock1 == clock2)
12301                 return true;
12302
12303         if (!clock1 || !clock2)
12304                 return false;
12305
12306         diff = abs(clock1 - clock2);
12307
12308         if (((((diff + clock1 + clock2) * 100)) / (clock1 + clock2)) < 105)
12309                 return true;
12310
12311         return false;
12312 }
12313
12314 #define for_each_intel_crtc_masked(dev, mask, intel_crtc) \
12315         list_for_each_entry((intel_crtc), \
12316                             &(dev)->mode_config.crtc_list, \
12317                             base.head) \
12318                 if (mask & (1 <<(intel_crtc)->pipe))
12319
12320 static bool
12321 intel_pipe_config_compare(struct drm_device *dev,
12322                           struct intel_crtc_state *current_config,
12323                           struct intel_crtc_state *pipe_config)
12324 {
12325 #define PIPE_CONF_CHECK_X(name) \
12326         if (current_config->name != pipe_config->name) { \
12327                 DRM_ERROR("mismatch in " #name " " \
12328                           "(expected 0x%08x, found 0x%08x)\n", \
12329                           current_config->name, \
12330                           pipe_config->name); \
12331                 return false; \
12332         }
12333
12334 #define PIPE_CONF_CHECK_I(name) \
12335         if (current_config->name != pipe_config->name) { \
12336                 DRM_ERROR("mismatch in " #name " " \
12337                           "(expected %i, found %i)\n", \
12338                           current_config->name, \
12339                           pipe_config->name); \
12340                 return false; \
12341         }
12342
12343 /* This is required for BDW+ where there is only one set of registers for
12344  * switching between high and low RR.
12345  * This macro can be used whenever a comparison has to be made between one
12346  * hw state and multiple sw state variables.
12347  */
12348 #define PIPE_CONF_CHECK_I_ALT(name, alt_name) \
12349         if ((current_config->name != pipe_config->name) && \
12350                 (current_config->alt_name != pipe_config->name)) { \
12351                         DRM_ERROR("mismatch in " #name " " \
12352                                   "(expected %i or %i, found %i)\n", \
12353                                   current_config->name, \
12354                                   current_config->alt_name, \
12355                                   pipe_config->name); \
12356                         return false; \
12357         }
12358
12359 #define PIPE_CONF_CHECK_FLAGS(name, mask)       \
12360         if ((current_config->name ^ pipe_config->name) & (mask)) { \
12361                 DRM_ERROR("mismatch in " #name "(" #mask ") "      \
12362                           "(expected %i, found %i)\n", \
12363                           current_config->name & (mask), \
12364                           pipe_config->name & (mask)); \
12365                 return false; \
12366         }
12367
12368 #define PIPE_CONF_CHECK_CLOCK_FUZZY(name) \
12369         if (!intel_fuzzy_clock_check(current_config->name, pipe_config->name)) { \
12370                 DRM_ERROR("mismatch in " #name " " \
12371                           "(expected %i, found %i)\n", \
12372                           current_config->name, \
12373                           pipe_config->name); \
12374                 return false; \
12375         }
12376
12377 #define PIPE_CONF_QUIRK(quirk)  \
12378         ((current_config->quirks | pipe_config->quirks) & (quirk))
12379
12380         PIPE_CONF_CHECK_I(cpu_transcoder);
12381
12382         PIPE_CONF_CHECK_I(has_pch_encoder);
12383         PIPE_CONF_CHECK_I(fdi_lanes);
12384         PIPE_CONF_CHECK_I(fdi_m_n.gmch_m);
12385         PIPE_CONF_CHECK_I(fdi_m_n.gmch_n);
12386         PIPE_CONF_CHECK_I(fdi_m_n.link_m);
12387         PIPE_CONF_CHECK_I(fdi_m_n.link_n);
12388         PIPE_CONF_CHECK_I(fdi_m_n.tu);
12389
12390         PIPE_CONF_CHECK_I(has_dp_encoder);
12391
12392         if (INTEL_INFO(dev)->gen < 8) {
12393                 PIPE_CONF_CHECK_I(dp_m_n.gmch_m);
12394                 PIPE_CONF_CHECK_I(dp_m_n.gmch_n);
12395                 PIPE_CONF_CHECK_I(dp_m_n.link_m);
12396                 PIPE_CONF_CHECK_I(dp_m_n.link_n);
12397                 PIPE_CONF_CHECK_I(dp_m_n.tu);
12398
12399                 if (current_config->has_drrs) {
12400                         PIPE_CONF_CHECK_I(dp_m2_n2.gmch_m);
12401                         PIPE_CONF_CHECK_I(dp_m2_n2.gmch_n);
12402                         PIPE_CONF_CHECK_I(dp_m2_n2.link_m);
12403                         PIPE_CONF_CHECK_I(dp_m2_n2.link_n);
12404                         PIPE_CONF_CHECK_I(dp_m2_n2.tu);
12405                 }
12406         } else {
12407                 PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_m, dp_m2_n2.gmch_m);
12408                 PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_n, dp_m2_n2.gmch_n);
12409                 PIPE_CONF_CHECK_I_ALT(dp_m_n.link_m, dp_m2_n2.link_m);
12410                 PIPE_CONF_CHECK_I_ALT(dp_m_n.link_n, dp_m2_n2.link_n);
12411                 PIPE_CONF_CHECK_I_ALT(dp_m_n.tu, dp_m2_n2.tu);
12412         }
12413
12414         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
12415         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
12416         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_start);
12417         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hblank_end);
12418         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_start);
12419         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hsync_end);
12420
12421         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vdisplay);
12422         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vtotal);
12423         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_start);
12424         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vblank_end);
12425         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_start);
12426         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
12427
12428         PIPE_CONF_CHECK_I(pixel_multiplier);
12429         PIPE_CONF_CHECK_I(has_hdmi_sink);
12430         if ((INTEL_INFO(dev)->gen < 8 && !IS_HASWELL(dev)) ||
12431             IS_VALLEYVIEW(dev))
12432                 PIPE_CONF_CHECK_I(limited_color_range);
12433         PIPE_CONF_CHECK_I(has_infoframe);
12434
12435         PIPE_CONF_CHECK_I(has_audio);
12436
12437         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12438                               DRM_MODE_FLAG_INTERLACE);
12439
12440         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_MODE_SYNC_FLAGS)) {
12441                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12442                                       DRM_MODE_FLAG_PHSYNC);
12443                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12444                                       DRM_MODE_FLAG_NHSYNC);
12445                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12446                                       DRM_MODE_FLAG_PVSYNC);
12447                 PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
12448                                       DRM_MODE_FLAG_NVSYNC);
12449         }
12450
12451         PIPE_CONF_CHECK_I(pipe_src_w);
12452         PIPE_CONF_CHECK_I(pipe_src_h);
12453
12454         /*
12455          * FIXME: BIOS likes to set up a cloned config with lvds+external
12456          * screen. Since we don't yet re-compute the pipe config when moving
12457          * just the lvds port away to another pipe the sw tracking won't match.
12458          *
12459          * Proper atomic modesets with recomputed global state will fix this.
12460          * Until then just don't check gmch state for inherited modes.
12461          */
12462         if (!PIPE_CONF_QUIRK(PIPE_CONFIG_QUIRK_INHERITED_MODE)) {
12463                 PIPE_CONF_CHECK_I(gmch_pfit.control);
12464                 /* pfit ratios are autocomputed by the hw on gen4+ */
12465                 if (INTEL_INFO(dev)->gen < 4)
12466                         PIPE_CONF_CHECK_I(gmch_pfit.pgm_ratios);
12467                 PIPE_CONF_CHECK_I(gmch_pfit.lvds_border_bits);
12468         }
12469
12470         PIPE_CONF_CHECK_I(pch_pfit.enabled);
12471         if (current_config->pch_pfit.enabled) {
12472                 PIPE_CONF_CHECK_I(pch_pfit.pos);
12473                 PIPE_CONF_CHECK_I(pch_pfit.size);
12474         }
12475
12476         PIPE_CONF_CHECK_I(scaler_state.scaler_id);
12477
12478         /* BDW+ don't expose a synchronous way to read the state */
12479         if (IS_HASWELL(dev))
12480                 PIPE_CONF_CHECK_I(ips_enabled);
12481
12482         PIPE_CONF_CHECK_I(double_wide);
12483
12484         PIPE_CONF_CHECK_X(ddi_pll_sel);
12485
12486         PIPE_CONF_CHECK_I(shared_dpll);
12487         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
12488         PIPE_CONF_CHECK_X(dpll_hw_state.dpll_md);
12489         PIPE_CONF_CHECK_X(dpll_hw_state.fp0);
12490         PIPE_CONF_CHECK_X(dpll_hw_state.fp1);
12491         PIPE_CONF_CHECK_X(dpll_hw_state.wrpll);
12492         PIPE_CONF_CHECK_X(dpll_hw_state.ctrl1);
12493         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr1);
12494         PIPE_CONF_CHECK_X(dpll_hw_state.cfgcr2);
12495
12496         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5)
12497                 PIPE_CONF_CHECK_I(pipe_bpp);
12498
12499         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
12500         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
12501
12502 #undef PIPE_CONF_CHECK_X
12503 #undef PIPE_CONF_CHECK_I
12504 #undef PIPE_CONF_CHECK_I_ALT
12505 #undef PIPE_CONF_CHECK_FLAGS
12506 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
12507 #undef PIPE_CONF_QUIRK
12508
12509         return true;
12510 }
12511
12512 static void check_wm_state(struct drm_device *dev)
12513 {
12514         struct drm_i915_private *dev_priv = dev->dev_private;
12515         struct skl_ddb_allocation hw_ddb, *sw_ddb;
12516         struct intel_crtc *intel_crtc;
12517         int plane;
12518
12519         if (INTEL_INFO(dev)->gen < 9)
12520                 return;
12521
12522         skl_ddb_get_hw_state(dev_priv, &hw_ddb);
12523         sw_ddb = &dev_priv->wm.skl_hw.ddb;
12524
12525         for_each_intel_crtc(dev, intel_crtc) {
12526                 struct skl_ddb_entry *hw_entry, *sw_entry;
12527                 const enum pipe pipe = intel_crtc->pipe;
12528
12529                 if (!intel_crtc->active)
12530                         continue;
12531
12532                 /* planes */
12533                 for_each_plane(dev_priv, pipe, plane) {
12534                         hw_entry = &hw_ddb.plane[pipe][plane];
12535                         sw_entry = &sw_ddb->plane[pipe][plane];
12536
12537                         if (skl_ddb_entry_equal(hw_entry, sw_entry))
12538                                 continue;
12539
12540                         DRM_ERROR("mismatch in DDB state pipe %c plane %d "
12541                                   "(expected (%u,%u), found (%u,%u))\n",
12542                                   pipe_name(pipe), plane + 1,
12543                                   sw_entry->start, sw_entry->end,
12544                                   hw_entry->start, hw_entry->end);
12545                 }
12546
12547                 /* cursor */
12548                 hw_entry = &hw_ddb.cursor[pipe];
12549                 sw_entry = &sw_ddb->cursor[pipe];
12550
12551                 if (skl_ddb_entry_equal(hw_entry, sw_entry))
12552                         continue;
12553
12554                 DRM_ERROR("mismatch in DDB state pipe %c cursor "
12555                           "(expected (%u,%u), found (%u,%u))\n",
12556                           pipe_name(pipe),
12557                           sw_entry->start, sw_entry->end,
12558                           hw_entry->start, hw_entry->end);
12559         }
12560 }
12561
12562 static void
12563 check_connector_state(struct drm_device *dev)
12564 {
12565         struct intel_connector *connector;
12566
12567         for_each_intel_connector(dev, connector) {
12568                 /* This also checks the encoder/connector hw state with the
12569                  * ->get_hw_state callbacks. */
12570                 intel_connector_check_state(connector);
12571
12572                 I915_STATE_WARN(&connector->new_encoder->base != connector->base.encoder,
12573                      "connector's staged encoder doesn't match current encoder\n");
12574         }
12575 }
12576
12577 static void
12578 check_encoder_state(struct drm_device *dev)
12579 {
12580         struct intel_encoder *encoder;
12581         struct intel_connector *connector;
12582
12583         for_each_intel_encoder(dev, encoder) {
12584                 bool enabled = false;
12585                 bool active = false;
12586                 enum pipe pipe, tracked_pipe;
12587
12588                 DRM_DEBUG_KMS("[ENCODER:%d:%s]\n",
12589                               encoder->base.base.id,
12590                               encoder->base.name);
12591
12592                 I915_STATE_WARN(&encoder->new_crtc->base != encoder->base.crtc,
12593                      "encoder's stage crtc doesn't match current crtc\n");
12594                 I915_STATE_WARN(encoder->connectors_active && !encoder->base.crtc,
12595                      "encoder's active_connectors set, but no crtc\n");
12596
12597                 for_each_intel_connector(dev, connector) {
12598                         if (connector->base.encoder != &encoder->base)
12599                                 continue;
12600                         enabled = true;
12601                         if (connector->base.dpms != DRM_MODE_DPMS_OFF)
12602                                 active = true;
12603                 }
12604                 /*
12605                  * for MST connectors if we unplug the connector is gone
12606                  * away but the encoder is still connected to a crtc
12607                  * until a modeset happens in response to the hotplug.
12608                  */
12609                 if (!enabled && encoder->base.encoder_type == DRM_MODE_ENCODER_DPMST)
12610                         continue;
12611
12612                 I915_STATE_WARN(!!encoder->base.crtc != enabled,
12613                      "encoder's enabled state mismatch "
12614                      "(expected %i, found %i)\n",
12615                      !!encoder->base.crtc, enabled);
12616                 I915_STATE_WARN(active && !encoder->base.crtc,
12617                      "active encoder with no crtc\n");
12618
12619                 I915_STATE_WARN(encoder->connectors_active != active,
12620                      "encoder's computed active state doesn't match tracked active state "
12621                      "(expected %i, found %i)\n", active, encoder->connectors_active);
12622
12623                 active = encoder->get_hw_state(encoder, &pipe);
12624                 I915_STATE_WARN(active != encoder->connectors_active,
12625                      "encoder's hw state doesn't match sw tracking "
12626                      "(expected %i, found %i)\n",
12627                      encoder->connectors_active, active);
12628
12629                 if (!encoder->base.crtc)
12630                         continue;
12631
12632                 tracked_pipe = to_intel_crtc(encoder->base.crtc)->pipe;
12633                 I915_STATE_WARN(active && pipe != tracked_pipe,
12634                      "active encoder's pipe doesn't match"
12635                      "(expected %i, found %i)\n",
12636                      tracked_pipe, pipe);
12637
12638         }
12639 }
12640
12641 static void
12642 check_crtc_state(struct drm_device *dev)
12643 {
12644         struct drm_i915_private *dev_priv = dev->dev_private;
12645         struct intel_crtc *crtc;
12646         struct intel_encoder *encoder;
12647         struct intel_crtc_state pipe_config;
12648
12649         for_each_intel_crtc(dev, crtc) {
12650                 bool enabled = false;
12651                 bool active = false;
12652
12653                 memset(&pipe_config, 0, sizeof(pipe_config));
12654
12655                 DRM_DEBUG_KMS("[CRTC:%d]\n",
12656                               crtc->base.base.id);
12657
12658                 I915_STATE_WARN(crtc->active && !crtc->base.state->enable,
12659                      "active crtc, but not enabled in sw tracking\n");
12660
12661                 for_each_intel_encoder(dev, encoder) {
12662                         if (encoder->base.crtc != &crtc->base)
12663                                 continue;
12664                         enabled = true;
12665                         if (encoder->connectors_active)
12666                                 active = true;
12667                 }
12668
12669                 I915_STATE_WARN(active != crtc->active,
12670                      "crtc's computed active state doesn't match tracked active state "
12671                      "(expected %i, found %i)\n", active, crtc->active);
12672                 I915_STATE_WARN(enabled != crtc->base.state->enable,
12673                      "crtc's computed enabled state doesn't match tracked enabled state "
12674                      "(expected %i, found %i)\n", enabled,
12675                                 crtc->base.state->enable);
12676
12677                 active = dev_priv->display.get_pipe_config(crtc,
12678                                                            &pipe_config);
12679
12680                 /* hw state is inconsistent with the pipe quirk */
12681                 if ((crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
12682                     (crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
12683                         active = crtc->active;
12684
12685                 for_each_intel_encoder(dev, encoder) {
12686                         enum pipe pipe;
12687                         if (encoder->base.crtc != &crtc->base)
12688                                 continue;
12689                         if (encoder->get_hw_state(encoder, &pipe))
12690                                 encoder->get_config(encoder, &pipe_config);
12691                 }
12692
12693                 I915_STATE_WARN(crtc->active != active,
12694                      "crtc active state doesn't match with hw state "
12695                      "(expected %i, found %i)\n", crtc->active, active);
12696
12697                 I915_STATE_WARN(crtc->active != crtc->base.state->active,
12698                      "transitional active state does not match atomic hw state "
12699                      "(expected %i, found %i)\n", crtc->base.state->active, crtc->active);
12700
12701                 if (active &&
12702                     !intel_pipe_config_compare(dev, crtc->config, &pipe_config)) {
12703                         I915_STATE_WARN(1, "pipe state doesn't match!\n");
12704                         intel_dump_pipe_config(crtc, &pipe_config,
12705                                                "[hw state]");
12706                         intel_dump_pipe_config(crtc, crtc->config,
12707                                                "[sw state]");
12708                 }
12709         }
12710 }
12711
12712 static void
12713 check_shared_dpll_state(struct drm_device *dev)
12714 {
12715         struct drm_i915_private *dev_priv = dev->dev_private;
12716         struct intel_crtc *crtc;
12717         struct intel_dpll_hw_state dpll_hw_state;
12718         int i;
12719
12720         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
12721                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
12722                 int enabled_crtcs = 0, active_crtcs = 0;
12723                 bool active;
12724
12725                 memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
12726
12727                 DRM_DEBUG_KMS("%s\n", pll->name);
12728
12729                 active = pll->get_hw_state(dev_priv, pll, &dpll_hw_state);
12730
12731                 I915_STATE_WARN(pll->active > hweight32(pll->config.crtc_mask),
12732                      "more active pll users than references: %i vs %i\n",
12733                      pll->active, hweight32(pll->config.crtc_mask));
12734                 I915_STATE_WARN(pll->active && !pll->on,
12735                      "pll in active use but not on in sw tracking\n");
12736                 I915_STATE_WARN(pll->on && !pll->active,
12737                      "pll in on but not on in use in sw tracking\n");
12738                 I915_STATE_WARN(pll->on != active,
12739                      "pll on state mismatch (expected %i, found %i)\n",
12740                      pll->on, active);
12741
12742                 for_each_intel_crtc(dev, crtc) {
12743                         if (crtc->base.state->enable && intel_crtc_to_shared_dpll(crtc) == pll)
12744                                 enabled_crtcs++;
12745                         if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll)
12746                                 active_crtcs++;
12747                 }
12748                 I915_STATE_WARN(pll->active != active_crtcs,
12749                      "pll active crtcs mismatch (expected %i, found %i)\n",
12750                      pll->active, active_crtcs);
12751                 I915_STATE_WARN(hweight32(pll->config.crtc_mask) != enabled_crtcs,
12752                      "pll enabled crtcs mismatch (expected %i, found %i)\n",
12753                      hweight32(pll->config.crtc_mask), enabled_crtcs);
12754
12755                 I915_STATE_WARN(pll->on && memcmp(&pll->config.hw_state, &dpll_hw_state,
12756                                        sizeof(dpll_hw_state)),
12757                      "pll hw state mismatch\n");
12758         }
12759 }
12760
12761 void
12762 intel_modeset_check_state(struct drm_device *dev)
12763 {
12764         check_wm_state(dev);
12765         check_connector_state(dev);
12766         check_encoder_state(dev);
12767         check_crtc_state(dev);
12768         check_shared_dpll_state(dev);
12769 }
12770
12771 void ironlake_check_encoder_dotclock(const struct intel_crtc_state *pipe_config,
12772                                      int dotclock)
12773 {
12774         /*
12775          * FDI already provided one idea for the dotclock.
12776          * Yell if the encoder disagrees.
12777          */
12778         WARN(!intel_fuzzy_clock_check(pipe_config->base.adjusted_mode.crtc_clock, dotclock),
12779              "FDI dotclock and encoder dotclock mismatch, fdi: %i, encoder: %i\n",
12780              pipe_config->base.adjusted_mode.crtc_clock, dotclock);
12781 }
12782
12783 static void update_scanline_offset(struct intel_crtc *crtc)
12784 {
12785         struct drm_device *dev = crtc->base.dev;
12786
12787         /*
12788          * The scanline counter increments at the leading edge of hsync.
12789          *
12790          * On most platforms it starts counting from vtotal-1 on the
12791          * first active line. That means the scanline counter value is
12792          * always one less than what we would expect. Ie. just after
12793          * start of vblank, which also occurs at start of hsync (on the
12794          * last active line), the scanline counter will read vblank_start-1.
12795          *
12796          * On gen2 the scanline counter starts counting from 1 instead
12797          * of vtotal-1, so we have to subtract one (or rather add vtotal-1
12798          * to keep the value positive), instead of adding one.
12799          *
12800          * On HSW+ the behaviour of the scanline counter depends on the output
12801          * type. For DP ports it behaves like most other platforms, but on HDMI
12802          * there's an extra 1 line difference. So we need to add two instead of
12803          * one to the value.
12804          */
12805         if (IS_GEN2(dev)) {
12806                 const struct drm_display_mode *mode = &crtc->config->base.adjusted_mode;
12807                 int vtotal;
12808
12809                 vtotal = mode->crtc_vtotal;
12810                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
12811                         vtotal /= 2;
12812
12813                 crtc->scanline_offset = vtotal - 1;
12814         } else if (HAS_DDI(dev) &&
12815                    intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) {
12816                 crtc->scanline_offset = 2;
12817         } else
12818                 crtc->scanline_offset = 1;
12819 }
12820
12821 static void intel_modeset_clear_plls(struct drm_atomic_state *state)
12822 {
12823         struct drm_device *dev = state->dev;
12824         struct drm_i915_private *dev_priv = to_i915(dev);
12825         struct intel_shared_dpll_config *shared_dpll = NULL;
12826         struct intel_crtc *intel_crtc;
12827         struct intel_crtc_state *intel_crtc_state;
12828         struct drm_crtc *crtc;
12829         struct drm_crtc_state *crtc_state;
12830         int i;
12831
12832         if (!dev_priv->display.crtc_compute_clock)
12833                 return;
12834
12835         for_each_crtc_in_state(state, crtc, crtc_state, i) {
12836                 int dpll;
12837
12838                 intel_crtc = to_intel_crtc(crtc);
12839                 intel_crtc_state = to_intel_crtc_state(crtc_state);
12840                 dpll = intel_crtc_state->shared_dpll;
12841
12842                 if (!needs_modeset(crtc_state) || dpll == DPLL_ID_PRIVATE)
12843                         continue;
12844
12845                 intel_crtc_state->shared_dpll = DPLL_ID_PRIVATE;
12846
12847                 if (!shared_dpll)
12848                         shared_dpll = intel_atomic_get_shared_dpll_state(state);
12849
12850                 shared_dpll[dpll].crtc_mask &= ~(1 << intel_crtc->pipe);
12851         }
12852 }
12853
12854 /*
12855  * This implements the workaround described in the "notes" section of the mode
12856  * set sequence documentation. When going from no pipes or single pipe to
12857  * multiple pipes, and planes are enabled after the pipe, we need to wait at
12858  * least 2 vblanks on the first pipe before enabling planes on the second pipe.
12859  */
12860 static int haswell_mode_set_planes_workaround(struct drm_atomic_state *state)
12861 {
12862         struct drm_crtc_state *crtc_state;
12863         struct intel_crtc *intel_crtc;
12864         struct drm_crtc *crtc;
12865         struct intel_crtc_state *first_crtc_state = NULL;
12866         struct intel_crtc_state *other_crtc_state = NULL;
12867         enum pipe first_pipe = INVALID_PIPE, enabled_pipe = INVALID_PIPE;
12868         int i;
12869
12870         /* look at all crtc's that are going to be enabled in during modeset */
12871         for_each_crtc_in_state(state, crtc, crtc_state, i) {
12872                 intel_crtc = to_intel_crtc(crtc);
12873
12874                 if (!crtc_state->active || !needs_modeset(crtc_state))
12875                         continue;
12876
12877                 if (first_crtc_state) {
12878                         other_crtc_state = to_intel_crtc_state(crtc_state);
12879                         break;
12880                 } else {
12881                         first_crtc_state = to_intel_crtc_state(crtc_state);
12882                         first_pipe = intel_crtc->pipe;
12883                 }
12884         }
12885
12886         /* No workaround needed? */
12887         if (!first_crtc_state)
12888                 return 0;
12889
12890         /* w/a possibly needed, check how many crtc's are already enabled. */
12891         for_each_intel_crtc(state->dev, intel_crtc) {
12892                 struct intel_crtc_state *pipe_config;
12893
12894                 pipe_config = intel_atomic_get_crtc_state(state, intel_crtc);
12895                 if (IS_ERR(pipe_config))
12896                         return PTR_ERR(pipe_config);
12897
12898                 pipe_config->hsw_workaround_pipe = INVALID_PIPE;
12899
12900                 if (!pipe_config->base.active ||
12901                     needs_modeset(&pipe_config->base))
12902                         continue;
12903
12904                 /* 2 or more enabled crtcs means no need for w/a */
12905                 if (enabled_pipe != INVALID_PIPE)
12906                         return 0;
12907
12908                 enabled_pipe = intel_crtc->pipe;
12909         }
12910
12911         if (enabled_pipe != INVALID_PIPE)
12912                 first_crtc_state->hsw_workaround_pipe = enabled_pipe;
12913         else if (other_crtc_state)
12914                 other_crtc_state->hsw_workaround_pipe = first_pipe;
12915
12916         return 0;
12917 }
12918
12919 static int intel_modeset_all_pipes(struct drm_atomic_state *state)
12920 {
12921         struct drm_crtc *crtc;
12922         struct drm_crtc_state *crtc_state;
12923         int ret = 0;
12924
12925         /* add all active pipes to the state */
12926         for_each_crtc(state->dev, crtc) {
12927                 crtc_state = drm_atomic_get_crtc_state(state, crtc);
12928                 if (IS_ERR(crtc_state))
12929                         return PTR_ERR(crtc_state);
12930
12931                 if (!crtc_state->active || needs_modeset(crtc_state))
12932                         continue;
12933
12934                 crtc_state->mode_changed = true;
12935
12936                 ret = drm_atomic_add_affected_connectors(state, crtc);
12937                 if (ret)
12938                         break;
12939
12940                 ret = drm_atomic_add_affected_planes(state, crtc);
12941                 if (ret)
12942                         break;
12943         }
12944
12945         return ret;
12946 }
12947
12948
12949 /* Code that should eventually be part of atomic_check() */
12950 static int intel_modeset_checks(struct drm_atomic_state *state)
12951 {
12952         struct drm_device *dev = state->dev;
12953         struct drm_i915_private *dev_priv = dev->dev_private;
12954         int ret;
12955
12956         if (!check_digital_port_conflicts(state)) {
12957                 DRM_DEBUG_KMS("rejecting conflicting digital port configuration\n");
12958                 return -EINVAL;
12959         }
12960
12961         /*
12962          * See if the config requires any additional preparation, e.g.
12963          * to adjust global state with pipes off.  We need to do this
12964          * here so we can get the modeset_pipe updated config for the new
12965          * mode set on this crtc.  For other crtcs we need to use the
12966          * adjusted_mode bits in the crtc directly.
12967          */
12968         if (dev_priv->display.modeset_calc_cdclk) {
12969                 unsigned int cdclk;
12970
12971                 ret = dev_priv->display.modeset_calc_cdclk(state);
12972
12973                 cdclk = to_intel_atomic_state(state)->cdclk;
12974                 if (!ret && cdclk != dev_priv->cdclk_freq)
12975                         ret = intel_modeset_all_pipes(state);
12976
12977                 if (ret < 0)
12978                         return ret;
12979         } else
12980                 to_intel_atomic_state(state)->cdclk = dev_priv->cdclk_freq;
12981
12982         intel_modeset_clear_plls(state);
12983
12984         if (IS_HASWELL(dev))
12985                 return haswell_mode_set_planes_workaround(state);
12986
12987         return 0;
12988 }
12989
12990 static int
12991 intel_modeset_compute_config(struct drm_atomic_state *state)
12992 {
12993         struct drm_crtc *crtc;
12994         struct drm_crtc_state *crtc_state;
12995         int ret, i;
12996         bool any_ms = false;
12997
12998         ret = drm_atomic_helper_check_modeset(state->dev, state);
12999         if (ret)
13000                 return ret;
13001
13002         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13003                 if (!crtc_state->enable) {
13004                         if (needs_modeset(crtc_state))
13005                                 any_ms = true;
13006                         continue;
13007                 }
13008
13009                 if (to_intel_crtc_state(crtc_state)->quirks &
13010                     PIPE_CONFIG_QUIRK_INITIAL_PLANES) {
13011                         ret = drm_atomic_add_affected_planes(state, crtc);
13012                         if (ret)
13013                                 return ret;
13014
13015                         /*
13016                          * We ought to handle i915.fastboot here.
13017                          * If no modeset is required and the primary plane has
13018                          * a fb, update the members of crtc_state as needed,
13019                          * and run the necessary updates during vblank evasion.
13020                          */
13021                 }
13022
13023                 if (!needs_modeset(crtc_state)) {
13024                         ret = drm_atomic_add_affected_connectors(state, crtc);
13025                         if (ret)
13026                                 return ret;
13027                 }
13028
13029                 ret = intel_modeset_pipe_config(crtc,
13030                                         to_intel_crtc_state(crtc_state));
13031                 if (ret)
13032                         return ret;
13033
13034                 if (needs_modeset(crtc_state))
13035                         any_ms = true;
13036
13037                 intel_dump_pipe_config(to_intel_crtc(crtc),
13038                                        to_intel_crtc_state(crtc_state),
13039                                        "[modeset]");
13040         }
13041
13042         if (any_ms) {
13043                 ret = intel_modeset_checks(state);
13044
13045                 if (ret)
13046                         return ret;
13047         } else
13048                 to_intel_atomic_state(state)->cdclk =
13049                         to_i915(state->dev)->cdclk_freq;
13050
13051         return drm_atomic_helper_check_planes(state->dev, state);
13052 }
13053
13054 static int __intel_set_mode(struct drm_atomic_state *state)
13055 {
13056         struct drm_device *dev = state->dev;
13057         struct drm_i915_private *dev_priv = dev->dev_private;
13058         struct drm_crtc *crtc;
13059         struct drm_crtc_state *crtc_state;
13060         int ret = 0;
13061         int i;
13062         bool any_ms = false;
13063
13064         ret = drm_atomic_helper_prepare_planes(dev, state);
13065         if (ret)
13066                 return ret;
13067
13068         drm_atomic_helper_swap_state(dev, state);
13069
13070         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13071                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13072
13073                 if (!needs_modeset(crtc->state))
13074                         continue;
13075
13076                 any_ms = true;
13077                 intel_pre_plane_update(intel_crtc);
13078
13079                 if (crtc_state->active) {
13080                         intel_crtc_disable_planes(crtc, crtc_state->plane_mask);
13081                         dev_priv->display.crtc_disable(crtc);
13082                         intel_crtc->active = false;
13083                         intel_disable_shared_dpll(intel_crtc);
13084                 }
13085         }
13086
13087         /* Only after disabling all output pipelines that will be changed can we
13088          * update the the output configuration. */
13089         intel_modeset_update_state(state);
13090
13091         /* The state has been swaped above, so state actually contains the
13092          * old state now. */
13093         if (any_ms)
13094                 modeset_update_crtc_power_domains(state);
13095
13096         /* Now enable the clocks, plane, pipe, and connectors that we set up. */
13097         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13098                 if (needs_modeset(crtc->state) && crtc->state->active) {
13099                         update_scanline_offset(to_intel_crtc(crtc));
13100                         dev_priv->display.crtc_enable(crtc);
13101                 }
13102
13103                 drm_atomic_helper_commit_planes_on_crtc(crtc_state);
13104         }
13105
13106         /* FIXME: add subpixel order */
13107
13108         drm_atomic_helper_cleanup_planes(dev, state);
13109
13110         drm_atomic_state_free(state);
13111
13112         return 0;
13113 }
13114
13115 static int intel_set_mode_checked(struct drm_atomic_state *state)
13116 {
13117         struct drm_device *dev = state->dev;
13118         int ret;
13119
13120         ret = __intel_set_mode(state);
13121         if (ret == 0)
13122                 intel_modeset_check_state(dev);
13123
13124         return ret;
13125 }
13126
13127 static int intel_set_mode(struct drm_atomic_state *state)
13128 {
13129         int ret;
13130
13131         ret = intel_modeset_compute_config(state);
13132         if (ret)
13133                 return ret;
13134
13135         return intel_set_mode_checked(state);
13136 }
13137
13138 void intel_crtc_restore_mode(struct drm_crtc *crtc)
13139 {
13140         struct drm_device *dev = crtc->dev;
13141         struct drm_atomic_state *state;
13142         struct intel_encoder *encoder;
13143         struct intel_connector *connector;
13144         struct drm_connector_state *connector_state;
13145         struct intel_crtc_state *crtc_state;
13146         int ret;
13147
13148         state = drm_atomic_state_alloc(dev);
13149         if (!state) {
13150                 DRM_DEBUG_KMS("[CRTC:%d] mode restore failed, out of memory",
13151                               crtc->base.id);
13152                 return;
13153         }
13154
13155         state->acquire_ctx = dev->mode_config.acquire_ctx;
13156
13157         /* The force restore path in the HW readout code relies on the staged
13158          * config still keeping the user requested config while the actual
13159          * state has been overwritten by the configuration read from HW. We
13160          * need to copy the staged config to the atomic state, otherwise the
13161          * mode set will just reapply the state the HW is already in. */
13162         for_each_intel_encoder(dev, encoder) {
13163                 if (&encoder->new_crtc->base != crtc)
13164                         continue;
13165
13166                 for_each_intel_connector(dev, connector) {
13167                         if (connector->new_encoder != encoder)
13168                                 continue;
13169
13170                         connector_state = drm_atomic_get_connector_state(state, &connector->base);
13171                         if (IS_ERR(connector_state)) {
13172                                 DRM_DEBUG_KMS("Failed to add [CONNECTOR:%d:%s] to state: %ld\n",
13173                                               connector->base.base.id,
13174                                               connector->base.name,
13175                                               PTR_ERR(connector_state));
13176                                 continue;
13177                         }
13178
13179                         connector_state->crtc = crtc;
13180                         connector_state->best_encoder = &encoder->base;
13181                 }
13182         }
13183
13184         crtc_state = intel_atomic_get_crtc_state(state, to_intel_crtc(crtc));
13185         if (IS_ERR(crtc_state)) {
13186                 DRM_DEBUG_KMS("Failed to add [CRTC:%d] to state: %ld\n",
13187                               crtc->base.id, PTR_ERR(crtc_state));
13188                 drm_atomic_state_free(state);
13189                 return;
13190         }
13191
13192         crtc_state->base.active = crtc_state->base.enable =
13193                 to_intel_crtc(crtc)->new_enabled;
13194
13195         drm_mode_copy(&crtc_state->base.mode, &crtc->mode);
13196
13197         intel_modeset_setup_plane_state(state, crtc, &crtc->mode,
13198                                         crtc->primary->fb, crtc->x, crtc->y);
13199
13200         ret = intel_set_mode(state);
13201         if (ret)
13202                 drm_atomic_state_free(state);
13203 }
13204
13205 #undef for_each_intel_crtc_masked
13206
13207 static bool intel_connector_in_mode_set(struct intel_connector *connector,
13208                                         struct drm_mode_set *set)
13209 {
13210         int ro;
13211
13212         for (ro = 0; ro < set->num_connectors; ro++)
13213                 if (set->connectors[ro] == &connector->base)
13214                         return true;
13215
13216         return false;
13217 }
13218
13219 static int
13220 intel_modeset_stage_output_state(struct drm_device *dev,
13221                                  struct drm_mode_set *set,
13222                                  struct drm_atomic_state *state)
13223 {
13224         struct intel_connector *connector;
13225         struct drm_connector *drm_connector;
13226         struct drm_connector_state *connector_state;
13227         struct drm_crtc *crtc;
13228         struct drm_crtc_state *crtc_state;
13229         int i, ret;
13230
13231         /* The upper layers ensure that we either disable a crtc or have a list
13232          * of connectors. For paranoia, double-check this. */
13233         WARN_ON(!set->fb && (set->num_connectors != 0));
13234         WARN_ON(set->fb && (set->num_connectors == 0));
13235
13236         for_each_intel_connector(dev, connector) {
13237                 bool in_mode_set = intel_connector_in_mode_set(connector, set);
13238
13239                 if (!in_mode_set && connector->base.state->crtc != set->crtc)
13240                         continue;
13241
13242                 connector_state =
13243                         drm_atomic_get_connector_state(state, &connector->base);
13244                 if (IS_ERR(connector_state))
13245                         return PTR_ERR(connector_state);
13246
13247                 if (in_mode_set) {
13248                         int pipe = to_intel_crtc(set->crtc)->pipe;
13249                         connector_state->best_encoder =
13250                                 &intel_find_encoder(connector, pipe)->base;
13251                 }
13252
13253                 if (connector->base.state->crtc != set->crtc)
13254                         continue;
13255
13256                 /* If we disable the crtc, disable all its connectors. Also, if
13257                  * the connector is on the changing crtc but not on the new
13258                  * connector list, disable it. */
13259                 if (!set->fb || !in_mode_set) {
13260                         connector_state->best_encoder = NULL;
13261
13262                         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [NOCRTC]\n",
13263                                 connector->base.base.id,
13264                                 connector->base.name);
13265                 }
13266         }
13267         /* connector->new_encoder is now updated for all connectors. */
13268
13269         for_each_connector_in_state(state, drm_connector, connector_state, i) {
13270                 connector = to_intel_connector(drm_connector);
13271
13272                 if (!connector_state->best_encoder) {
13273                         ret = drm_atomic_set_crtc_for_connector(connector_state,
13274                                                                 NULL);
13275                         if (ret)
13276                                 return ret;
13277
13278                         continue;
13279                 }
13280
13281                 if (intel_connector_in_mode_set(connector, set)) {
13282                         struct drm_crtc *crtc = connector->base.state->crtc;
13283
13284                         /* If this connector was in a previous crtc, add it
13285                          * to the state. We might need to disable it. */
13286                         if (crtc) {
13287                                 crtc_state =
13288                                         drm_atomic_get_crtc_state(state, crtc);
13289                                 if (IS_ERR(crtc_state))
13290                                         return PTR_ERR(crtc_state);
13291                         }
13292
13293                         ret = drm_atomic_set_crtc_for_connector(connector_state,
13294                                                                 set->crtc);
13295                         if (ret)
13296                                 return ret;
13297                 }
13298
13299                 /* Make sure the new CRTC will work with the encoder */
13300                 if (!drm_encoder_crtc_ok(connector_state->best_encoder,
13301                                          connector_state->crtc)) {
13302                         return -EINVAL;
13303                 }
13304
13305                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] to [CRTC:%d]\n",
13306                         connector->base.base.id,
13307                         connector->base.name,
13308                         connector_state->crtc->base.id);
13309
13310                 if (connector_state->best_encoder != &connector->encoder->base)
13311                         connector->encoder =
13312                                 to_intel_encoder(connector_state->best_encoder);
13313         }
13314
13315         for_each_crtc_in_state(state, crtc, crtc_state, i) {
13316                 bool has_connectors;
13317
13318                 ret = drm_atomic_add_affected_connectors(state, crtc);
13319                 if (ret)
13320                         return ret;
13321
13322                 has_connectors = !!drm_atomic_connectors_for_crtc(state, crtc);
13323                 if (has_connectors != crtc_state->enable)
13324                         crtc_state->enable =
13325                         crtc_state->active = has_connectors;
13326         }
13327
13328         ret = intel_modeset_setup_plane_state(state, set->crtc, set->mode,
13329                                               set->fb, set->x, set->y);
13330         if (ret)
13331                 return ret;
13332
13333         crtc_state = drm_atomic_get_crtc_state(state, set->crtc);
13334         if (IS_ERR(crtc_state))
13335                 return PTR_ERR(crtc_state);
13336
13337         ret = drm_atomic_set_mode_for_crtc(crtc_state, set->mode);
13338         if (ret)
13339                 return ret;
13340
13341         if (set->num_connectors)
13342                 crtc_state->active = true;
13343
13344         return 0;
13345 }
13346
13347 static int intel_crtc_set_config(struct drm_mode_set *set)
13348 {
13349         struct drm_device *dev;
13350         struct drm_atomic_state *state = NULL;
13351         int ret;
13352
13353         BUG_ON(!set);
13354         BUG_ON(!set->crtc);
13355         BUG_ON(!set->crtc->helper_private);
13356
13357         /* Enforce sane interface api - has been abused by the fb helper. */
13358         BUG_ON(!set->mode && set->fb);
13359         BUG_ON(set->fb && set->num_connectors == 0);
13360
13361         if (set->fb) {
13362                 DRM_DEBUG_KMS("[CRTC:%d] [FB:%d] #connectors=%d (x y) (%i %i)\n",
13363                                 set->crtc->base.id, set->fb->base.id,
13364                                 (int)set->num_connectors, set->x, set->y);
13365         } else {
13366                 DRM_DEBUG_KMS("[CRTC:%d] [NOFB]\n", set->crtc->base.id);
13367         }
13368
13369         dev = set->crtc->dev;
13370
13371         state = drm_atomic_state_alloc(dev);
13372         if (!state)
13373                 return -ENOMEM;
13374
13375         state->acquire_ctx = dev->mode_config.acquire_ctx;
13376
13377         ret = intel_modeset_stage_output_state(dev, set, state);
13378         if (ret)
13379                 goto out;
13380
13381         ret = intel_modeset_compute_config(state);
13382         if (ret)
13383                 goto out;
13384
13385         intel_update_pipe_size(to_intel_crtc(set->crtc));
13386
13387         ret = intel_set_mode_checked(state);
13388         if (ret) {
13389                 DRM_DEBUG_KMS("failed to set mode on [CRTC:%d], err = %d\n",
13390                               set->crtc->base.id, ret);
13391         }
13392
13393 out:
13394         if (ret)
13395                 drm_atomic_state_free(state);
13396         return ret;
13397 }
13398
13399 static const struct drm_crtc_funcs intel_crtc_funcs = {
13400         .gamma_set = intel_crtc_gamma_set,
13401         .set_config = intel_crtc_set_config,
13402         .destroy = intel_crtc_destroy,
13403         .page_flip = intel_crtc_page_flip,
13404         .atomic_duplicate_state = intel_crtc_duplicate_state,
13405         .atomic_destroy_state = intel_crtc_destroy_state,
13406 };
13407
13408 static bool ibx_pch_dpll_get_hw_state(struct drm_i915_private *dev_priv,
13409                                       struct intel_shared_dpll *pll,
13410                                       struct intel_dpll_hw_state *hw_state)
13411 {
13412         uint32_t val;
13413
13414         if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_PLLS))
13415                 return false;
13416
13417         val = I915_READ(PCH_DPLL(pll->id));
13418         hw_state->dpll = val;
13419         hw_state->fp0 = I915_READ(PCH_FP0(pll->id));
13420         hw_state->fp1 = I915_READ(PCH_FP1(pll->id));
13421
13422         return val & DPLL_VCO_ENABLE;
13423 }
13424
13425 static void ibx_pch_dpll_mode_set(struct drm_i915_private *dev_priv,
13426                                   struct intel_shared_dpll *pll)
13427 {
13428         I915_WRITE(PCH_FP0(pll->id), pll->config.hw_state.fp0);
13429         I915_WRITE(PCH_FP1(pll->id), pll->config.hw_state.fp1);
13430 }
13431
13432 static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv,
13433                                 struct intel_shared_dpll *pll)
13434 {
13435         /* PCH refclock must be enabled first */
13436         ibx_assert_pch_refclk_enabled(dev_priv);
13437
13438         I915_WRITE(PCH_DPLL(pll->id), pll->config.hw_state.dpll);
13439
13440         /* Wait for the clocks to stabilize. */
13441         POSTING_READ(PCH_DPLL(pll->id));
13442         udelay(150);
13443
13444         /* The pixel multiplier can only be updated once the
13445          * DPLL is enabled and the clocks are stable.
13446          *
13447          * So write it again.
13448          */
13449         I915_WRITE(PCH_DPLL(pll->id), pll->config.hw_state.dpll);
13450         POSTING_READ(PCH_DPLL(pll->id));
13451         udelay(200);
13452 }
13453
13454 static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv,
13455                                  struct intel_shared_dpll *pll)
13456 {
13457         struct drm_device *dev = dev_priv->dev;
13458         struct intel_crtc *crtc;
13459
13460         /* Make sure no transcoder isn't still depending on us. */
13461         for_each_intel_crtc(dev, crtc) {
13462                 if (intel_crtc_to_shared_dpll(crtc) == pll)
13463                         assert_pch_transcoder_disabled(dev_priv, crtc->pipe);
13464         }
13465
13466         I915_WRITE(PCH_DPLL(pll->id), 0);
13467         POSTING_READ(PCH_DPLL(pll->id));
13468         udelay(200);
13469 }
13470
13471 static char *ibx_pch_dpll_names[] = {
13472         "PCH DPLL A",
13473         "PCH DPLL B",
13474 };
13475
13476 static void ibx_pch_dpll_init(struct drm_device *dev)
13477 {
13478         struct drm_i915_private *dev_priv = dev->dev_private;
13479         int i;
13480
13481         dev_priv->num_shared_dpll = 2;
13482
13483         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
13484                 dev_priv->shared_dplls[i].id = i;
13485                 dev_priv->shared_dplls[i].name = ibx_pch_dpll_names[i];
13486                 dev_priv->shared_dplls[i].mode_set = ibx_pch_dpll_mode_set;
13487                 dev_priv->shared_dplls[i].enable = ibx_pch_dpll_enable;
13488                 dev_priv->shared_dplls[i].disable = ibx_pch_dpll_disable;
13489                 dev_priv->shared_dplls[i].get_hw_state =
13490                         ibx_pch_dpll_get_hw_state;
13491         }
13492 }
13493
13494 static void intel_shared_dpll_init(struct drm_device *dev)
13495 {
13496         struct drm_i915_private *dev_priv = dev->dev_private;
13497
13498         intel_update_cdclk(dev);
13499
13500         if (HAS_DDI(dev))
13501                 intel_ddi_pll_init(dev);
13502         else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
13503                 ibx_pch_dpll_init(dev);
13504         else
13505                 dev_priv->num_shared_dpll = 0;
13506
13507         BUG_ON(dev_priv->num_shared_dpll > I915_NUM_PLLS);
13508 }
13509
13510 /**
13511  * intel_prepare_plane_fb - Prepare fb for usage on plane
13512  * @plane: drm plane to prepare for
13513  * @fb: framebuffer to prepare for presentation
13514  *
13515  * Prepares a framebuffer for usage on a display plane.  Generally this
13516  * involves pinning the underlying object and updating the frontbuffer tracking
13517  * bits.  Some older platforms need special physical address handling for
13518  * cursor planes.
13519  *
13520  * Returns 0 on success, negative error code on failure.
13521  */
13522 int
13523 intel_prepare_plane_fb(struct drm_plane *plane,
13524                        struct drm_framebuffer *fb,
13525                        const struct drm_plane_state *new_state)
13526 {
13527         struct drm_device *dev = plane->dev;
13528         struct intel_plane *intel_plane = to_intel_plane(plane);
13529         enum pipe pipe = intel_plane->pipe;
13530         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13531         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->fb);
13532         unsigned frontbuffer_bits = 0;
13533         int ret = 0;
13534
13535         if (!obj)
13536                 return 0;
13537
13538         switch (plane->type) {
13539         case DRM_PLANE_TYPE_PRIMARY:
13540                 frontbuffer_bits = INTEL_FRONTBUFFER_PRIMARY(pipe);
13541                 break;
13542         case DRM_PLANE_TYPE_CURSOR:
13543                 frontbuffer_bits = INTEL_FRONTBUFFER_CURSOR(pipe);
13544                 break;
13545         case DRM_PLANE_TYPE_OVERLAY:
13546                 frontbuffer_bits = INTEL_FRONTBUFFER_SPRITE(pipe);
13547                 break;
13548         }
13549
13550         mutex_lock(&dev->struct_mutex);
13551
13552         if (plane->type == DRM_PLANE_TYPE_CURSOR &&
13553             INTEL_INFO(dev)->cursor_needs_physical) {
13554                 int align = IS_I830(dev) ? 16 * 1024 : 256;
13555                 ret = i915_gem_object_attach_phys(obj, align);
13556                 if (ret)
13557                         DRM_DEBUG_KMS("failed to attach phys object\n");
13558         } else {
13559                 ret = intel_pin_and_fence_fb_obj(plane, fb, new_state, NULL, NULL);
13560         }
13561
13562         if (ret == 0)
13563                 i915_gem_track_fb(old_obj, obj, frontbuffer_bits);
13564
13565         mutex_unlock(&dev->struct_mutex);
13566
13567         return ret;
13568 }
13569
13570 /**
13571  * intel_cleanup_plane_fb - Cleans up an fb after plane use
13572  * @plane: drm plane to clean up for
13573  * @fb: old framebuffer that was on plane
13574  *
13575  * Cleans up a framebuffer that has just been removed from a plane.
13576  */
13577 void
13578 intel_cleanup_plane_fb(struct drm_plane *plane,
13579                        struct drm_framebuffer *fb,
13580                        const struct drm_plane_state *old_state)
13581 {
13582         struct drm_device *dev = plane->dev;
13583         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13584
13585         if (WARN_ON(!obj))
13586                 return;
13587
13588         if (plane->type != DRM_PLANE_TYPE_CURSOR ||
13589             !INTEL_INFO(dev)->cursor_needs_physical) {
13590                 mutex_lock(&dev->struct_mutex);
13591                 intel_unpin_fb_obj(fb, old_state);
13592                 mutex_unlock(&dev->struct_mutex);
13593         }
13594 }
13595
13596 int
13597 skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state)
13598 {
13599         int max_scale;
13600         struct drm_device *dev;
13601         struct drm_i915_private *dev_priv;
13602         int crtc_clock, cdclk;
13603
13604         if (!intel_crtc || !crtc_state)
13605                 return DRM_PLANE_HELPER_NO_SCALING;
13606
13607         dev = intel_crtc->base.dev;
13608         dev_priv = dev->dev_private;
13609         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
13610         cdclk = to_intel_atomic_state(crtc_state->base.state)->cdclk;
13611
13612         if (!crtc_clock || !cdclk)
13613                 return DRM_PLANE_HELPER_NO_SCALING;
13614
13615         /*
13616          * skl max scale is lower of:
13617          *    close to 3 but not 3, -1 is for that purpose
13618          *            or
13619          *    cdclk/crtc_clock
13620          */
13621         max_scale = min((1 << 16) * 3 - 1, (1 << 8) * ((cdclk << 8) / crtc_clock));
13622
13623         return max_scale;
13624 }
13625
13626 static int
13627 intel_check_primary_plane(struct drm_plane *plane,
13628                           struct intel_crtc_state *crtc_state,
13629                           struct intel_plane_state *state)
13630 {
13631         struct drm_crtc *crtc = state->base.crtc;
13632         struct drm_framebuffer *fb = state->base.fb;
13633         int min_scale = DRM_PLANE_HELPER_NO_SCALING;
13634         int max_scale = DRM_PLANE_HELPER_NO_SCALING;
13635         bool can_position = false;
13636
13637         /* use scaler when colorkey is not required */
13638         if (INTEL_INFO(plane->dev)->gen >= 9 &&
13639             state->ckey.flags == I915_SET_COLORKEY_NONE) {
13640                 min_scale = 1;
13641                 max_scale = skl_max_scale(to_intel_crtc(crtc), crtc_state);
13642                 can_position = true;
13643         }
13644
13645         return drm_plane_helper_check_update(plane, crtc, fb, &state->src,
13646                                              &state->dst, &state->clip,
13647                                              min_scale, max_scale,
13648                                              can_position, true,
13649                                              &state->visible);
13650 }
13651
13652 static void
13653 intel_commit_primary_plane(struct drm_plane *plane,
13654                            struct intel_plane_state *state)
13655 {
13656         struct drm_crtc *crtc = state->base.crtc;
13657         struct drm_framebuffer *fb = state->base.fb;
13658         struct drm_device *dev = plane->dev;
13659         struct drm_i915_private *dev_priv = dev->dev_private;
13660         struct intel_crtc *intel_crtc;
13661         struct drm_rect *src = &state->src;
13662
13663         crtc = crtc ? crtc : plane->crtc;
13664         intel_crtc = to_intel_crtc(crtc);
13665
13666         plane->fb = fb;
13667         crtc->x = src->x1 >> 16;
13668         crtc->y = src->y1 >> 16;
13669
13670         if (!crtc->state->active)
13671                 return;
13672
13673         if (state->visible)
13674                 /* FIXME: kill this fastboot hack */
13675                 intel_update_pipe_size(intel_crtc);
13676
13677         dev_priv->display.update_primary_plane(crtc, fb, crtc->x, crtc->y);
13678 }
13679
13680 static void
13681 intel_disable_primary_plane(struct drm_plane *plane,
13682                             struct drm_crtc *crtc)
13683 {
13684         struct drm_device *dev = plane->dev;
13685         struct drm_i915_private *dev_priv = dev->dev_private;
13686
13687         dev_priv->display.update_primary_plane(crtc, NULL, 0, 0);
13688 }
13689
13690 static void intel_begin_crtc_commit(struct drm_crtc *crtc)
13691 {
13692         struct drm_device *dev = crtc->dev;
13693         struct drm_i915_private *dev_priv = dev->dev_private;
13694         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13695
13696         if (!needs_modeset(crtc->state))
13697                 intel_pre_plane_update(intel_crtc);
13698
13699         if (intel_crtc->atomic.update_wm)
13700                 intel_update_watermarks(crtc);
13701
13702         intel_runtime_pm_get(dev_priv);
13703
13704         /* Perform vblank evasion around commit operation */
13705         if (crtc->state->active)
13706                 intel_crtc->atomic.evade =
13707                         intel_pipe_update_start(intel_crtc,
13708                                                 &intel_crtc->atomic.start_vbl_count);
13709
13710         if (!needs_modeset(crtc->state) && INTEL_INFO(dev)->gen >= 9)
13711                 skl_detach_scalers(intel_crtc);
13712 }
13713
13714 static void intel_finish_crtc_commit(struct drm_crtc *crtc)
13715 {
13716         struct drm_device *dev = crtc->dev;
13717         struct drm_i915_private *dev_priv = dev->dev_private;
13718         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
13719
13720         if (intel_crtc->atomic.evade)
13721                 intel_pipe_update_end(intel_crtc,
13722                                       intel_crtc->atomic.start_vbl_count);
13723
13724         intel_runtime_pm_put(dev_priv);
13725
13726         intel_post_plane_update(intel_crtc);
13727 }
13728
13729 /**
13730  * intel_plane_destroy - destroy a plane
13731  * @plane: plane to destroy
13732  *
13733  * Common destruction function for all types of planes (primary, cursor,
13734  * sprite).
13735  */
13736 void intel_plane_destroy(struct drm_plane *plane)
13737 {
13738         struct intel_plane *intel_plane = to_intel_plane(plane);
13739         drm_plane_cleanup(plane);
13740         kfree(intel_plane);
13741 }
13742
13743 const struct drm_plane_funcs intel_plane_funcs = {
13744         .update_plane = drm_atomic_helper_update_plane,
13745         .disable_plane = drm_atomic_helper_disable_plane,
13746         .destroy = intel_plane_destroy,
13747         .set_property = drm_atomic_helper_plane_set_property,
13748         .atomic_get_property = intel_plane_atomic_get_property,
13749         .atomic_set_property = intel_plane_atomic_set_property,
13750         .atomic_duplicate_state = intel_plane_duplicate_state,
13751         .atomic_destroy_state = intel_plane_destroy_state,
13752
13753 };
13754
13755 static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
13756                                                     int pipe)
13757 {
13758         struct intel_plane *primary;
13759         struct intel_plane_state *state;
13760         const uint32_t *intel_primary_formats;
13761         int num_formats;
13762
13763         primary = kzalloc(sizeof(*primary), GFP_KERNEL);
13764         if (primary == NULL)
13765                 return NULL;
13766
13767         state = intel_create_plane_state(&primary->base);
13768         if (!state) {
13769                 kfree(primary);
13770                 return NULL;
13771         }
13772         primary->base.state = &state->base;
13773
13774         primary->can_scale = false;
13775         primary->max_downscale = 1;
13776         if (INTEL_INFO(dev)->gen >= 9) {
13777                 primary->can_scale = true;
13778                 state->scaler_id = -1;
13779         }
13780         primary->pipe = pipe;
13781         primary->plane = pipe;
13782         primary->check_plane = intel_check_primary_plane;
13783         primary->commit_plane = intel_commit_primary_plane;
13784         primary->disable_plane = intel_disable_primary_plane;
13785         if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4)
13786                 primary->plane = !pipe;
13787
13788         if (INTEL_INFO(dev)->gen >= 9) {
13789                 intel_primary_formats = skl_primary_formats;
13790                 num_formats = ARRAY_SIZE(skl_primary_formats);
13791         } else if (INTEL_INFO(dev)->gen >= 4) {
13792                 intel_primary_formats = i965_primary_formats;
13793                 num_formats = ARRAY_SIZE(i965_primary_formats);
13794         } else {
13795                 intel_primary_formats = i8xx_primary_formats;
13796                 num_formats = ARRAY_SIZE(i8xx_primary_formats);
13797         }
13798
13799         drm_universal_plane_init(dev, &primary->base, 0,
13800                                  &intel_plane_funcs,
13801                                  intel_primary_formats, num_formats,
13802                                  DRM_PLANE_TYPE_PRIMARY);
13803
13804         if (INTEL_INFO(dev)->gen >= 4)
13805                 intel_create_rotation_property(dev, primary);
13806
13807         drm_plane_helper_add(&primary->base, &intel_plane_helper_funcs);
13808
13809         return &primary->base;
13810 }
13811
13812 void intel_create_rotation_property(struct drm_device *dev, struct intel_plane *plane)
13813 {
13814         if (!dev->mode_config.rotation_property) {
13815                 unsigned long flags = BIT(DRM_ROTATE_0) |
13816                         BIT(DRM_ROTATE_180);
13817
13818                 if (INTEL_INFO(dev)->gen >= 9)
13819                         flags |= BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270);
13820
13821                 dev->mode_config.rotation_property =
13822                         drm_mode_create_rotation_property(dev, flags);
13823         }
13824         if (dev->mode_config.rotation_property)
13825                 drm_object_attach_property(&plane->base.base,
13826                                 dev->mode_config.rotation_property,
13827                                 plane->base.state->rotation);
13828 }
13829
13830 static int
13831 intel_check_cursor_plane(struct drm_plane *plane,
13832                          struct intel_crtc_state *crtc_state,
13833                          struct intel_plane_state *state)
13834 {
13835         struct drm_crtc *crtc = crtc_state->base.crtc;
13836         struct drm_framebuffer *fb = state->base.fb;
13837         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
13838         unsigned stride;
13839         int ret;
13840
13841         ret = drm_plane_helper_check_update(plane, crtc, fb, &state->src,
13842                                             &state->dst, &state->clip,
13843                                             DRM_PLANE_HELPER_NO_SCALING,
13844                                             DRM_PLANE_HELPER_NO_SCALING,
13845                                             true, true, &state->visible);
13846         if (ret)
13847                 return ret;
13848
13849         /* if we want to turn off the cursor ignore width and height */
13850         if (!obj)
13851                 return 0;
13852
13853         /* Check for which cursor types we support */
13854         if (!cursor_size_ok(plane->dev, state->base.crtc_w, state->base.crtc_h)) {
13855                 DRM_DEBUG("Cursor dimension %dx%d not supported\n",
13856                           state->base.crtc_w, state->base.crtc_h);
13857                 return -EINVAL;
13858         }
13859
13860         stride = roundup_pow_of_two(state->base.crtc_w) * 4;
13861         if (obj->base.size < stride * state->base.crtc_h) {
13862                 DRM_DEBUG_KMS("buffer is too small\n");
13863                 return -ENOMEM;
13864         }
13865
13866         if (fb->modifier[0] != DRM_FORMAT_MOD_NONE) {
13867                 DRM_DEBUG_KMS("cursor cannot be tiled\n");
13868                 return -EINVAL;
13869         }
13870
13871         return 0;
13872 }
13873
13874 static void
13875 intel_disable_cursor_plane(struct drm_plane *plane,
13876                            struct drm_crtc *crtc)
13877 {
13878         intel_crtc_update_cursor(crtc, false);
13879 }
13880
13881 static void
13882 intel_commit_cursor_plane(struct drm_plane *plane,
13883                           struct intel_plane_state *state)
13884 {
13885         struct drm_crtc *crtc = state->base.crtc;
13886         struct drm_device *dev = plane->dev;
13887         struct intel_crtc *intel_crtc;
13888         struct drm_i915_gem_object *obj = intel_fb_obj(state->base.fb);
13889         uint32_t addr;
13890
13891         crtc = crtc ? crtc : plane->crtc;
13892         intel_crtc = to_intel_crtc(crtc);
13893
13894         plane->fb = state->base.fb;
13895         crtc->cursor_x = state->base.crtc_x;
13896         crtc->cursor_y = state->base.crtc_y;
13897
13898         if (intel_crtc->cursor_bo == obj)
13899                 goto update;
13900
13901         if (!obj)
13902                 addr = 0;
13903         else if (!INTEL_INFO(dev)->cursor_needs_physical)
13904                 addr = i915_gem_obj_ggtt_offset(obj);
13905         else
13906                 addr = obj->phys_handle->busaddr;
13907
13908         intel_crtc->cursor_addr = addr;
13909         intel_crtc->cursor_bo = obj;
13910
13911 update:
13912         if (crtc->state->active)
13913                 intel_crtc_update_cursor(crtc, state->visible);
13914 }
13915
13916 static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
13917                                                    int pipe)
13918 {
13919         struct intel_plane *cursor;
13920         struct intel_plane_state *state;
13921
13922         cursor = kzalloc(sizeof(*cursor), GFP_KERNEL);
13923         if (cursor == NULL)
13924                 return NULL;
13925
13926         state = intel_create_plane_state(&cursor->base);
13927         if (!state) {
13928                 kfree(cursor);
13929                 return NULL;
13930         }
13931         cursor->base.state = &state->base;
13932
13933         cursor->can_scale = false;
13934         cursor->max_downscale = 1;
13935         cursor->pipe = pipe;
13936         cursor->plane = pipe;
13937         cursor->check_plane = intel_check_cursor_plane;
13938         cursor->commit_plane = intel_commit_cursor_plane;
13939         cursor->disable_plane = intel_disable_cursor_plane;
13940
13941         drm_universal_plane_init(dev, &cursor->base, 0,
13942                                  &intel_plane_funcs,
13943                                  intel_cursor_formats,
13944                                  ARRAY_SIZE(intel_cursor_formats),
13945                                  DRM_PLANE_TYPE_CURSOR);
13946
13947         if (INTEL_INFO(dev)->gen >= 4) {
13948                 if (!dev->mode_config.rotation_property)
13949                         dev->mode_config.rotation_property =
13950                                 drm_mode_create_rotation_property(dev,
13951                                                         BIT(DRM_ROTATE_0) |
13952                                                         BIT(DRM_ROTATE_180));
13953                 if (dev->mode_config.rotation_property)
13954                         drm_object_attach_property(&cursor->base.base,
13955                                 dev->mode_config.rotation_property,
13956                                 state->base.rotation);
13957         }
13958
13959         if (INTEL_INFO(dev)->gen >=9)
13960                 state->scaler_id = -1;
13961
13962         drm_plane_helper_add(&cursor->base, &intel_plane_helper_funcs);
13963
13964         return &cursor->base;
13965 }
13966
13967 static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_crtc,
13968         struct intel_crtc_state *crtc_state)
13969 {
13970         int i;
13971         struct intel_scaler *intel_scaler;
13972         struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
13973
13974         for (i = 0; i < intel_crtc->num_scalers; i++) {
13975                 intel_scaler = &scaler_state->scalers[i];
13976                 intel_scaler->in_use = 0;
13977                 intel_scaler->mode = PS_SCALER_MODE_DYN;
13978         }
13979
13980         scaler_state->scaler_id = -1;
13981 }
13982
13983 static void intel_crtc_init(struct drm_device *dev, int pipe)
13984 {
13985         struct drm_i915_private *dev_priv = dev->dev_private;
13986         struct intel_crtc *intel_crtc;
13987         struct intel_crtc_state *crtc_state = NULL;
13988         struct drm_plane *primary = NULL;
13989         struct drm_plane *cursor = NULL;
13990         int i, ret;
13991
13992         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
13993         if (intel_crtc == NULL)
13994                 return;
13995
13996         crtc_state = kzalloc(sizeof(*crtc_state), GFP_KERNEL);
13997         if (!crtc_state)
13998                 goto fail;
13999         intel_crtc->config = crtc_state;
14000         intel_crtc->base.state = &crtc_state->base;
14001         crtc_state->base.crtc = &intel_crtc->base;
14002
14003         /* initialize shared scalers */
14004         if (INTEL_INFO(dev)->gen >= 9) {
14005                 if (pipe == PIPE_C)
14006                         intel_crtc->num_scalers = 1;
14007                 else
14008                         intel_crtc->num_scalers = SKL_NUM_SCALERS;
14009
14010                 skl_init_scalers(dev, intel_crtc, crtc_state);
14011         }
14012
14013         primary = intel_primary_plane_create(dev, pipe);
14014         if (!primary)
14015                 goto fail;
14016
14017         cursor = intel_cursor_plane_create(dev, pipe);
14018         if (!cursor)
14019                 goto fail;
14020
14021         ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary,
14022                                         cursor, &intel_crtc_funcs);
14023         if (ret)
14024                 goto fail;
14025
14026         drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256);
14027         for (i = 0; i < 256; i++) {
14028                 intel_crtc->lut_r[i] = i;
14029                 intel_crtc->lut_g[i] = i;
14030                 intel_crtc->lut_b[i] = i;
14031         }
14032
14033         /*
14034          * On gen2/3 only plane A can do fbc, but the panel fitter and lvds port
14035          * is hooked to pipe B. Hence we want plane A feeding pipe B.
14036          */
14037         intel_crtc->pipe = pipe;
14038         intel_crtc->plane = pipe;
14039         if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4) {
14040                 DRM_DEBUG_KMS("swapping pipes & planes for FBC\n");
14041                 intel_crtc->plane = !pipe;
14042         }
14043
14044         intel_crtc->cursor_base = ~0;
14045         intel_crtc->cursor_cntl = ~0;
14046         intel_crtc->cursor_size = ~0;
14047
14048         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
14049                dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
14050         dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base;
14051         dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base;
14052
14053         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
14054
14055         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
14056         return;
14057
14058 fail:
14059         if (primary)
14060                 drm_plane_cleanup(primary);
14061         if (cursor)
14062                 drm_plane_cleanup(cursor);
14063         kfree(crtc_state);
14064         kfree(intel_crtc);
14065 }
14066
14067 enum pipe intel_get_pipe_from_connector(struct intel_connector *connector)
14068 {
14069         struct drm_encoder *encoder = connector->base.encoder;
14070         struct drm_device *dev = connector->base.dev;
14071
14072         WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
14073
14074         if (!encoder || WARN_ON(!encoder->crtc))
14075                 return INVALID_PIPE;
14076
14077         return to_intel_crtc(encoder->crtc)->pipe;
14078 }
14079
14080 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
14081                                 struct drm_file *file)
14082 {
14083         struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
14084         struct drm_crtc *drmmode_crtc;
14085         struct intel_crtc *crtc;
14086
14087         drmmode_crtc = drm_crtc_find(dev, pipe_from_crtc_id->crtc_id);
14088
14089         if (!drmmode_crtc) {
14090                 DRM_ERROR("no such CRTC id\n");
14091                 return -ENOENT;
14092         }
14093
14094         crtc = to_intel_crtc(drmmode_crtc);
14095         pipe_from_crtc_id->pipe = crtc->pipe;
14096
14097         return 0;
14098 }
14099
14100 static int intel_encoder_clones(struct intel_encoder *encoder)
14101 {
14102         struct drm_device *dev = encoder->base.dev;
14103         struct intel_encoder *source_encoder;
14104         int index_mask = 0;
14105         int entry = 0;
14106
14107         for_each_intel_encoder(dev, source_encoder) {
14108                 if (encoders_cloneable(encoder, source_encoder))
14109                         index_mask |= (1 << entry);
14110
14111                 entry++;
14112         }
14113
14114         return index_mask;
14115 }
14116
14117 static bool has_edp_a(struct drm_device *dev)
14118 {
14119         struct drm_i915_private *dev_priv = dev->dev_private;
14120
14121         if (!IS_MOBILE(dev))
14122                 return false;
14123
14124         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
14125                 return false;
14126
14127         if (IS_GEN5(dev) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
14128                 return false;
14129
14130         return true;
14131 }
14132
14133 static bool intel_crt_present(struct drm_device *dev)
14134 {
14135         struct drm_i915_private *dev_priv = dev->dev_private;
14136
14137         if (INTEL_INFO(dev)->gen >= 9)
14138                 return false;
14139
14140         if (IS_HSW_ULT(dev) || IS_BDW_ULT(dev))
14141                 return false;
14142
14143         if (IS_CHERRYVIEW(dev))
14144                 return false;
14145
14146         if (IS_VALLEYVIEW(dev) && !dev_priv->vbt.int_crt_support)
14147                 return false;
14148
14149         return true;
14150 }
14151
14152 static void intel_setup_outputs(struct drm_device *dev)
14153 {
14154         struct drm_i915_private *dev_priv = dev->dev_private;
14155         struct intel_encoder *encoder;
14156         bool dpd_is_edp = false;
14157
14158         intel_lvds_init(dev);
14159
14160         if (intel_crt_present(dev))
14161                 intel_crt_init(dev);
14162
14163         if (IS_BROXTON(dev)) {
14164                 /*
14165                  * FIXME: Broxton doesn't support port detection via the
14166                  * DDI_BUF_CTL_A or SFUSE_STRAP registers, find another way to
14167                  * detect the ports.
14168                  */
14169                 intel_ddi_init(dev, PORT_A);
14170                 intel_ddi_init(dev, PORT_B);
14171                 intel_ddi_init(dev, PORT_C);
14172         } else if (HAS_DDI(dev)) {
14173                 int found;
14174
14175                 /*
14176                  * Haswell uses DDI functions to detect digital outputs.
14177                  * On SKL pre-D0 the strap isn't connected, so we assume
14178                  * it's there.
14179                  */
14180                 found = I915_READ(DDI_BUF_CTL_A) & DDI_INIT_DISPLAY_DETECTED;
14181                 /* WaIgnoreDDIAStrap: skl */
14182                 if (found ||
14183                     (IS_SKYLAKE(dev) && INTEL_REVID(dev) < SKL_REVID_D0))
14184                         intel_ddi_init(dev, PORT_A);
14185
14186                 /* DDI B, C and D detection is indicated by the SFUSE_STRAP
14187                  * register */
14188                 found = I915_READ(SFUSE_STRAP);
14189
14190                 if (found & SFUSE_STRAP_DDIB_DETECTED)
14191                         intel_ddi_init(dev, PORT_B);
14192                 if (found & SFUSE_STRAP_DDIC_DETECTED)
14193                         intel_ddi_init(dev, PORT_C);
14194                 if (found & SFUSE_STRAP_DDID_DETECTED)
14195                         intel_ddi_init(dev, PORT_D);
14196         } else if (HAS_PCH_SPLIT(dev)) {
14197                 int found;
14198                 dpd_is_edp = intel_dp_is_edp(dev, PORT_D);
14199
14200                 if (has_edp_a(dev))
14201                         intel_dp_init(dev, DP_A, PORT_A);
14202
14203                 if (I915_READ(PCH_HDMIB) & SDVO_DETECTED) {
14204                         /* PCH SDVOB multiplex with HDMIB */
14205                         found = intel_sdvo_init(dev, PCH_SDVOB, true);
14206                         if (!found)
14207                                 intel_hdmi_init(dev, PCH_HDMIB, PORT_B);
14208                         if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
14209                                 intel_dp_init(dev, PCH_DP_B, PORT_B);
14210                 }
14211
14212                 if (I915_READ(PCH_HDMIC) & SDVO_DETECTED)
14213                         intel_hdmi_init(dev, PCH_HDMIC, PORT_C);
14214
14215                 if (!dpd_is_edp && I915_READ(PCH_HDMID) & SDVO_DETECTED)
14216                         intel_hdmi_init(dev, PCH_HDMID, PORT_D);
14217
14218                 if (I915_READ(PCH_DP_C) & DP_DETECTED)
14219                         intel_dp_init(dev, PCH_DP_C, PORT_C);
14220
14221                 if (I915_READ(PCH_DP_D) & DP_DETECTED)
14222                         intel_dp_init(dev, PCH_DP_D, PORT_D);
14223         } else if (IS_VALLEYVIEW(dev)) {
14224                 /*
14225                  * The DP_DETECTED bit is the latched state of the DDC
14226                  * SDA pin at boot. However since eDP doesn't require DDC
14227                  * (no way to plug in a DP->HDMI dongle) the DDC pins for
14228                  * eDP ports may have been muxed to an alternate function.
14229                  * Thus we can't rely on the DP_DETECTED bit alone to detect
14230                  * eDP ports. Consult the VBT as well as DP_DETECTED to
14231                  * detect eDP ports.
14232                  */
14233                 if (I915_READ(VLV_DISPLAY_BASE + GEN4_HDMIB) & SDVO_DETECTED &&
14234                     !intel_dp_is_edp(dev, PORT_B))
14235                         intel_hdmi_init(dev, VLV_DISPLAY_BASE + GEN4_HDMIB,
14236                                         PORT_B);
14237                 if (I915_READ(VLV_DISPLAY_BASE + DP_B) & DP_DETECTED ||
14238                     intel_dp_is_edp(dev, PORT_B))
14239                         intel_dp_init(dev, VLV_DISPLAY_BASE + DP_B, PORT_B);
14240
14241                 if (I915_READ(VLV_DISPLAY_BASE + GEN4_HDMIC) & SDVO_DETECTED &&
14242                     !intel_dp_is_edp(dev, PORT_C))
14243                         intel_hdmi_init(dev, VLV_DISPLAY_BASE + GEN4_HDMIC,
14244                                         PORT_C);
14245                 if (I915_READ(VLV_DISPLAY_BASE + DP_C) & DP_DETECTED ||
14246                     intel_dp_is_edp(dev, PORT_C))
14247                         intel_dp_init(dev, VLV_DISPLAY_BASE + DP_C, PORT_C);
14248
14249                 if (IS_CHERRYVIEW(dev)) {
14250                         if (I915_READ(VLV_DISPLAY_BASE + CHV_HDMID) & SDVO_DETECTED)
14251                                 intel_hdmi_init(dev, VLV_DISPLAY_BASE + CHV_HDMID,
14252                                                 PORT_D);
14253                         /* eDP not supported on port D, so don't check VBT */
14254                         if (I915_READ(VLV_DISPLAY_BASE + DP_D) & DP_DETECTED)
14255                                 intel_dp_init(dev, VLV_DISPLAY_BASE + DP_D, PORT_D);
14256                 }
14257
14258                 intel_dsi_init(dev);
14259         } else if (SUPPORTS_DIGITAL_OUTPUTS(dev)) {
14260                 bool found = false;
14261
14262                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14263                         DRM_DEBUG_KMS("probing SDVOB\n");
14264                         found = intel_sdvo_init(dev, GEN3_SDVOB, true);
14265                         if (!found && SUPPORTS_INTEGRATED_HDMI(dev)) {
14266                                 DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
14267                                 intel_hdmi_init(dev, GEN4_HDMIB, PORT_B);
14268                         }
14269
14270                         if (!found && SUPPORTS_INTEGRATED_DP(dev))
14271                                 intel_dp_init(dev, DP_B, PORT_B);
14272                 }
14273
14274                 /* Before G4X SDVOC doesn't have its own detect register */
14275
14276                 if (I915_READ(GEN3_SDVOB) & SDVO_DETECTED) {
14277                         DRM_DEBUG_KMS("probing SDVOC\n");
14278                         found = intel_sdvo_init(dev, GEN3_SDVOC, false);
14279                 }
14280
14281                 if (!found && (I915_READ(GEN3_SDVOC) & SDVO_DETECTED)) {
14282
14283                         if (SUPPORTS_INTEGRATED_HDMI(dev)) {
14284                                 DRM_DEBUG_KMS("probing HDMI on SDVOC\n");
14285                                 intel_hdmi_init(dev, GEN4_HDMIC, PORT_C);
14286                         }
14287                         if (SUPPORTS_INTEGRATED_DP(dev))
14288                                 intel_dp_init(dev, DP_C, PORT_C);
14289                 }
14290
14291                 if (SUPPORTS_INTEGRATED_DP(dev) &&
14292                     (I915_READ(DP_D) & DP_DETECTED))
14293                         intel_dp_init(dev, DP_D, PORT_D);
14294         } else if (IS_GEN2(dev))
14295                 intel_dvo_init(dev);
14296
14297         if (SUPPORTS_TV(dev))
14298                 intel_tv_init(dev);
14299
14300         intel_psr_init(dev);
14301
14302         for_each_intel_encoder(dev, encoder) {
14303                 encoder->base.possible_crtcs = encoder->crtc_mask;
14304                 encoder->base.possible_clones =
14305                         intel_encoder_clones(encoder);
14306         }
14307
14308         intel_init_pch_refclk(dev);
14309
14310         drm_helper_move_panel_connectors_to_head(dev);
14311 }
14312
14313 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
14314 {
14315         struct drm_device *dev = fb->dev;
14316         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14317
14318         drm_framebuffer_cleanup(fb);
14319         mutex_lock(&dev->struct_mutex);
14320         WARN_ON(!intel_fb->obj->framebuffer_references--);
14321         drm_gem_object_unreference(&intel_fb->obj->base);
14322         mutex_unlock(&dev->struct_mutex);
14323         kfree(intel_fb);
14324 }
14325
14326 static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb,
14327                                                 struct drm_file *file,
14328                                                 unsigned int *handle)
14329 {
14330         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
14331         struct drm_i915_gem_object *obj = intel_fb->obj;
14332
14333         return drm_gem_handle_create(file, &obj->base, handle);
14334 }
14335
14336 static const struct drm_framebuffer_funcs intel_fb_funcs = {
14337         .destroy = intel_user_framebuffer_destroy,
14338         .create_handle = intel_user_framebuffer_create_handle,
14339 };
14340
14341 static
14342 u32 intel_fb_pitch_limit(struct drm_device *dev, uint64_t fb_modifier,
14343                          uint32_t pixel_format)
14344 {
14345         u32 gen = INTEL_INFO(dev)->gen;
14346
14347         if (gen >= 9) {
14348                 /* "The stride in bytes must not exceed the of the size of 8K
14349                  *  pixels and 32K bytes."
14350                  */
14351                  return min(8192*drm_format_plane_cpp(pixel_format, 0), 32768);
14352         } else if (gen >= 5 && !IS_VALLEYVIEW(dev)) {
14353                 return 32*1024;
14354         } else if (gen >= 4) {
14355                 if (fb_modifier == I915_FORMAT_MOD_X_TILED)
14356                         return 16*1024;
14357                 else
14358                         return 32*1024;
14359         } else if (gen >= 3) {
14360                 if (fb_modifier == I915_FORMAT_MOD_X_TILED)
14361                         return 8*1024;
14362                 else
14363                         return 16*1024;
14364         } else {
14365                 /* XXX DSPC is limited to 4k tiled */
14366                 return 8*1024;
14367         }
14368 }
14369
14370 static int intel_framebuffer_init(struct drm_device *dev,
14371                                   struct intel_framebuffer *intel_fb,
14372                                   struct drm_mode_fb_cmd2 *mode_cmd,
14373                                   struct drm_i915_gem_object *obj)
14374 {
14375         unsigned int aligned_height;
14376         int ret;
14377         u32 pitch_limit, stride_alignment;
14378
14379         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
14380
14381         if (mode_cmd->flags & DRM_MODE_FB_MODIFIERS) {
14382                 /* Enforce that fb modifier and tiling mode match, but only for
14383                  * X-tiled. This is needed for FBC. */
14384                 if (!!(obj->tiling_mode == I915_TILING_X) !=
14385                     !!(mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED)) {
14386                         DRM_DEBUG("tiling_mode doesn't match fb modifier\n");
14387                         return -EINVAL;
14388                 }
14389         } else {
14390                 if (obj->tiling_mode == I915_TILING_X)
14391                         mode_cmd->modifier[0] = I915_FORMAT_MOD_X_TILED;
14392                 else if (obj->tiling_mode == I915_TILING_Y) {
14393                         DRM_DEBUG("No Y tiling for legacy addfb\n");
14394                         return -EINVAL;
14395                 }
14396         }
14397
14398         /* Passed in modifier sanity checking. */
14399         switch (mode_cmd->modifier[0]) {
14400         case I915_FORMAT_MOD_Y_TILED:
14401         case I915_FORMAT_MOD_Yf_TILED:
14402                 if (INTEL_INFO(dev)->gen < 9) {
14403                         DRM_DEBUG("Unsupported tiling 0x%llx!\n",
14404                                   mode_cmd->modifier[0]);
14405                         return -EINVAL;
14406                 }
14407         case DRM_FORMAT_MOD_NONE:
14408         case I915_FORMAT_MOD_X_TILED:
14409                 break;
14410         default:
14411                 DRM_DEBUG("Unsupported fb modifier 0x%llx!\n",
14412                           mode_cmd->modifier[0]);
14413                 return -EINVAL;
14414         }
14415
14416         stride_alignment = intel_fb_stride_alignment(dev, mode_cmd->modifier[0],
14417                                                      mode_cmd->pixel_format);
14418         if (mode_cmd->pitches[0] & (stride_alignment - 1)) {
14419                 DRM_DEBUG("pitch (%d) must be at least %u byte aligned\n",
14420                           mode_cmd->pitches[0], stride_alignment);
14421                 return -EINVAL;
14422         }
14423
14424         pitch_limit = intel_fb_pitch_limit(dev, mode_cmd->modifier[0],
14425                                            mode_cmd->pixel_format);
14426         if (mode_cmd->pitches[0] > pitch_limit) {
14427                 DRM_DEBUG("%s pitch (%u) must be at less than %d\n",
14428                           mode_cmd->modifier[0] != DRM_FORMAT_MOD_NONE ?
14429                           "tiled" : "linear",
14430                           mode_cmd->pitches[0], pitch_limit);
14431                 return -EINVAL;
14432         }
14433
14434         if (mode_cmd->modifier[0] == I915_FORMAT_MOD_X_TILED &&
14435             mode_cmd->pitches[0] != obj->stride) {
14436                 DRM_DEBUG("pitch (%d) must match tiling stride (%d)\n",
14437                           mode_cmd->pitches[0], obj->stride);
14438                 return -EINVAL;
14439         }
14440
14441         /* Reject formats not supported by any plane early. */
14442         switch (mode_cmd->pixel_format) {
14443         case DRM_FORMAT_C8:
14444         case DRM_FORMAT_RGB565:
14445         case DRM_FORMAT_XRGB8888:
14446         case DRM_FORMAT_ARGB8888:
14447                 break;
14448         case DRM_FORMAT_XRGB1555:
14449                 if (INTEL_INFO(dev)->gen > 3) {
14450                         DRM_DEBUG("unsupported pixel format: %s\n",
14451                                   drm_get_format_name(mode_cmd->pixel_format));
14452                         return -EINVAL;
14453                 }
14454                 break;
14455         case DRM_FORMAT_ABGR8888:
14456                 if (!IS_VALLEYVIEW(dev) && INTEL_INFO(dev)->gen < 9) {
14457                         DRM_DEBUG("unsupported pixel format: %s\n",
14458                                   drm_get_format_name(mode_cmd->pixel_format));
14459                         return -EINVAL;
14460                 }
14461                 break;
14462         case DRM_FORMAT_XBGR8888:
14463         case DRM_FORMAT_XRGB2101010:
14464         case DRM_FORMAT_XBGR2101010:
14465                 if (INTEL_INFO(dev)->gen < 4) {
14466                         DRM_DEBUG("unsupported pixel format: %s\n",
14467                                   drm_get_format_name(mode_cmd->pixel_format));
14468                         return -EINVAL;
14469                 }
14470                 break;
14471         case DRM_FORMAT_ABGR2101010:
14472                 if (!IS_VALLEYVIEW(dev)) {
14473                         DRM_DEBUG("unsupported pixel format: %s\n",
14474                                   drm_get_format_name(mode_cmd->pixel_format));
14475                         return -EINVAL;
14476                 }
14477                 break;
14478         case DRM_FORMAT_YUYV:
14479         case DRM_FORMAT_UYVY:
14480         case DRM_FORMAT_YVYU:
14481         case DRM_FORMAT_VYUY:
14482                 if (INTEL_INFO(dev)->gen < 5) {
14483                         DRM_DEBUG("unsupported pixel format: %s\n",
14484                                   drm_get_format_name(mode_cmd->pixel_format));
14485                         return -EINVAL;
14486                 }
14487                 break;
14488         default:
14489                 DRM_DEBUG("unsupported pixel format: %s\n",
14490                           drm_get_format_name(mode_cmd->pixel_format));
14491                 return -EINVAL;
14492         }
14493
14494         /* FIXME need to adjust LINOFF/TILEOFF accordingly. */
14495         if (mode_cmd->offsets[0] != 0)
14496                 return -EINVAL;
14497
14498         aligned_height = intel_fb_align_height(dev, mode_cmd->height,
14499                                                mode_cmd->pixel_format,
14500                                                mode_cmd->modifier[0]);
14501         /* FIXME drm helper for size checks (especially planar formats)? */
14502         if (obj->base.size < aligned_height * mode_cmd->pitches[0])
14503                 return -EINVAL;
14504
14505         drm_helper_mode_fill_fb_struct(&intel_fb->base, mode_cmd);
14506         intel_fb->obj = obj;
14507         intel_fb->obj->framebuffer_references++;
14508
14509         ret = drm_framebuffer_init(dev, &intel_fb->base, &intel_fb_funcs);
14510         if (ret) {
14511                 DRM_ERROR("framebuffer init failed %d\n", ret);
14512                 return ret;
14513         }
14514
14515         return 0;
14516 }
14517
14518 static struct drm_framebuffer *
14519 intel_user_framebuffer_create(struct drm_device *dev,
14520                               struct drm_file *filp,
14521                               struct drm_mode_fb_cmd2 *mode_cmd)
14522 {
14523         struct drm_i915_gem_object *obj;
14524
14525         obj = to_intel_bo(drm_gem_object_lookup(dev, filp,
14526                                                 mode_cmd->handles[0]));
14527         if (&obj->base == NULL)
14528                 return ERR_PTR(-ENOENT);
14529
14530         return intel_framebuffer_create(dev, mode_cmd, obj);
14531 }
14532
14533 #ifndef CONFIG_DRM_I915_FBDEV
14534 static inline void intel_fbdev_output_poll_changed(struct drm_device *dev)
14535 {
14536 }
14537 #endif
14538
14539 static const struct drm_mode_config_funcs intel_mode_funcs = {
14540         .fb_create = intel_user_framebuffer_create,
14541         .output_poll_changed = intel_fbdev_output_poll_changed,
14542         .atomic_check = intel_atomic_check,
14543         .atomic_commit = intel_atomic_commit,
14544         .atomic_state_alloc = intel_atomic_state_alloc,
14545         .atomic_state_clear = intel_atomic_state_clear,
14546 };
14547
14548 /* Set up chip specific display functions */
14549 static void intel_init_display(struct drm_device *dev)
14550 {
14551         struct drm_i915_private *dev_priv = dev->dev_private;
14552
14553         if (HAS_PCH_SPLIT(dev) || IS_G4X(dev))
14554                 dev_priv->display.find_dpll = g4x_find_best_dpll;
14555         else if (IS_CHERRYVIEW(dev))
14556                 dev_priv->display.find_dpll = chv_find_best_dpll;
14557         else if (IS_VALLEYVIEW(dev))
14558                 dev_priv->display.find_dpll = vlv_find_best_dpll;
14559         else if (IS_PINEVIEW(dev))
14560                 dev_priv->display.find_dpll = pnv_find_best_dpll;
14561         else
14562                 dev_priv->display.find_dpll = i9xx_find_best_dpll;
14563
14564         if (INTEL_INFO(dev)->gen >= 9) {
14565                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14566                 dev_priv->display.get_initial_plane_config =
14567                         skylake_get_initial_plane_config;
14568                 dev_priv->display.crtc_compute_clock =
14569                         haswell_crtc_compute_clock;
14570                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14571                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14572                 dev_priv->display.update_primary_plane =
14573                         skylake_update_primary_plane;
14574         } else if (HAS_DDI(dev)) {
14575                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
14576                 dev_priv->display.get_initial_plane_config =
14577                         ironlake_get_initial_plane_config;
14578                 dev_priv->display.crtc_compute_clock =
14579                         haswell_crtc_compute_clock;
14580                 dev_priv->display.crtc_enable = haswell_crtc_enable;
14581                 dev_priv->display.crtc_disable = haswell_crtc_disable;
14582                 dev_priv->display.update_primary_plane =
14583                         ironlake_update_primary_plane;
14584         } else if (HAS_PCH_SPLIT(dev)) {
14585                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
14586                 dev_priv->display.get_initial_plane_config =
14587                         ironlake_get_initial_plane_config;
14588                 dev_priv->display.crtc_compute_clock =
14589                         ironlake_crtc_compute_clock;
14590                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
14591                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
14592                 dev_priv->display.update_primary_plane =
14593                         ironlake_update_primary_plane;
14594         } else if (IS_VALLEYVIEW(dev)) {
14595                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14596                 dev_priv->display.get_initial_plane_config =
14597                         i9xx_get_initial_plane_config;
14598                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14599                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
14600                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14601                 dev_priv->display.update_primary_plane =
14602                         i9xx_update_primary_plane;
14603         } else {
14604                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
14605                 dev_priv->display.get_initial_plane_config =
14606                         i9xx_get_initial_plane_config;
14607                 dev_priv->display.crtc_compute_clock = i9xx_crtc_compute_clock;
14608                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
14609                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
14610                 dev_priv->display.update_primary_plane =
14611                         i9xx_update_primary_plane;
14612         }
14613
14614         /* Returns the core display clock speed */
14615         if (IS_SKYLAKE(dev))
14616                 dev_priv->display.get_display_clock_speed =
14617                         skylake_get_display_clock_speed;
14618         else if (IS_BROADWELL(dev))
14619                 dev_priv->display.get_display_clock_speed =
14620                         broadwell_get_display_clock_speed;
14621         else if (IS_HASWELL(dev))
14622                 dev_priv->display.get_display_clock_speed =
14623                         haswell_get_display_clock_speed;
14624         else if (IS_VALLEYVIEW(dev))
14625                 dev_priv->display.get_display_clock_speed =
14626                         valleyview_get_display_clock_speed;
14627         else if (IS_GEN5(dev))
14628                 dev_priv->display.get_display_clock_speed =
14629                         ilk_get_display_clock_speed;
14630         else if (IS_I945G(dev) || IS_BROADWATER(dev) ||
14631                  IS_GEN6(dev) || IS_IVYBRIDGE(dev))
14632                 dev_priv->display.get_display_clock_speed =
14633                         i945_get_display_clock_speed;
14634         else if (IS_GM45(dev))
14635                 dev_priv->display.get_display_clock_speed =
14636                         gm45_get_display_clock_speed;
14637         else if (IS_CRESTLINE(dev))
14638                 dev_priv->display.get_display_clock_speed =
14639                         i965gm_get_display_clock_speed;
14640         else if (IS_PINEVIEW(dev))
14641                 dev_priv->display.get_display_clock_speed =
14642                         pnv_get_display_clock_speed;
14643         else if (IS_G33(dev) || IS_G4X(dev))
14644                 dev_priv->display.get_display_clock_speed =
14645                         g33_get_display_clock_speed;
14646         else if (IS_I915G(dev))
14647                 dev_priv->display.get_display_clock_speed =
14648                         i915_get_display_clock_speed;
14649         else if (IS_I945GM(dev) || IS_845G(dev))
14650                 dev_priv->display.get_display_clock_speed =
14651                         i9xx_misc_get_display_clock_speed;
14652         else if (IS_PINEVIEW(dev))
14653                 dev_priv->display.get_display_clock_speed =
14654                         pnv_get_display_clock_speed;
14655         else if (IS_I915GM(dev))
14656                 dev_priv->display.get_display_clock_speed =
14657                         i915gm_get_display_clock_speed;
14658         else if (IS_I865G(dev))
14659                 dev_priv->display.get_display_clock_speed =
14660                         i865_get_display_clock_speed;
14661         else if (IS_I85X(dev))
14662                 dev_priv->display.get_display_clock_speed =
14663                         i85x_get_display_clock_speed;
14664         else { /* 830 */
14665                 WARN(!IS_I830(dev), "Unknown platform. Assuming 133 MHz CDCLK\n");
14666                 dev_priv->display.get_display_clock_speed =
14667                         i830_get_display_clock_speed;
14668         }
14669
14670         if (IS_GEN5(dev)) {
14671                 dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
14672         } else if (IS_GEN6(dev)) {
14673                 dev_priv->display.fdi_link_train = gen6_fdi_link_train;
14674         } else if (IS_IVYBRIDGE(dev)) {
14675                 /* FIXME: detect B0+ stepping and use auto training */
14676                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
14677         } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
14678                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
14679                 if (IS_BROADWELL(dev)) {
14680                         dev_priv->display.modeset_commit_cdclk =
14681                                 broadwell_modeset_commit_cdclk;
14682                         dev_priv->display.modeset_calc_cdclk =
14683                                 broadwell_modeset_calc_cdclk;
14684                 }
14685         } else if (IS_VALLEYVIEW(dev)) {
14686                 dev_priv->display.modeset_commit_cdclk =
14687                         valleyview_modeset_commit_cdclk;
14688                 dev_priv->display.modeset_calc_cdclk =
14689                         valleyview_modeset_calc_cdclk;
14690         } else if (IS_BROXTON(dev)) {
14691                 dev_priv->display.modeset_commit_cdclk =
14692                         broxton_modeset_commit_cdclk;
14693                 dev_priv->display.modeset_calc_cdclk =
14694                         broxton_modeset_calc_cdclk;
14695         }
14696
14697         switch (INTEL_INFO(dev)->gen) {
14698         case 2:
14699                 dev_priv->display.queue_flip = intel_gen2_queue_flip;
14700                 break;
14701
14702         case 3:
14703                 dev_priv->display.queue_flip = intel_gen3_queue_flip;
14704                 break;
14705
14706         case 4:
14707         case 5:
14708                 dev_priv->display.queue_flip = intel_gen4_queue_flip;
14709                 break;
14710
14711         case 6:
14712                 dev_priv->display.queue_flip = intel_gen6_queue_flip;
14713                 break;
14714         case 7:
14715         case 8: /* FIXME(BDW): Check that the gen8 RCS flip works. */
14716                 dev_priv->display.queue_flip = intel_gen7_queue_flip;
14717                 break;
14718         case 9:
14719                 /* Drop through - unsupported since execlist only. */
14720         default:
14721                 /* Default just returns -ENODEV to indicate unsupported */
14722                 dev_priv->display.queue_flip = intel_default_queue_flip;
14723         }
14724
14725         intel_panel_init_backlight_funcs(dev);
14726
14727         mutex_init(&dev_priv->pps_mutex);
14728 }
14729
14730 /*
14731  * Some BIOSes insist on assuming the GPU's pipe A is enabled at suspend,
14732  * resume, or other times.  This quirk makes sure that's the case for
14733  * affected systems.
14734  */
14735 static void quirk_pipea_force(struct drm_device *dev)
14736 {
14737         struct drm_i915_private *dev_priv = dev->dev_private;
14738
14739         dev_priv->quirks |= QUIRK_PIPEA_FORCE;
14740         DRM_INFO("applying pipe a force quirk\n");
14741 }
14742
14743 static void quirk_pipeb_force(struct drm_device *dev)
14744 {
14745         struct drm_i915_private *dev_priv = dev->dev_private;
14746
14747         dev_priv->quirks |= QUIRK_PIPEB_FORCE;
14748         DRM_INFO("applying pipe b force quirk\n");
14749 }
14750
14751 /*
14752  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
14753  */
14754 static void quirk_ssc_force_disable(struct drm_device *dev)
14755 {
14756         struct drm_i915_private *dev_priv = dev->dev_private;
14757         dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
14758         DRM_INFO("applying lvds SSC disable quirk\n");
14759 }
14760
14761 /*
14762  * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
14763  * brightness value
14764  */
14765 static void quirk_invert_brightness(struct drm_device *dev)
14766 {
14767         struct drm_i915_private *dev_priv = dev->dev_private;
14768         dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
14769         DRM_INFO("applying inverted panel brightness quirk\n");
14770 }
14771
14772 /* Some VBT's incorrectly indicate no backlight is present */
14773 static void quirk_backlight_present(struct drm_device *dev)
14774 {
14775         struct drm_i915_private *dev_priv = dev->dev_private;
14776         dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT;
14777         DRM_INFO("applying backlight present quirk\n");
14778 }
14779
14780 struct intel_quirk {
14781         int device;
14782         int subsystem_vendor;
14783         int subsystem_device;
14784         void (*hook)(struct drm_device *dev);
14785 };
14786
14787 /* For systems that don't have a meaningful PCI subdevice/subvendor ID */
14788 struct intel_dmi_quirk {
14789         void (*hook)(struct drm_device *dev);
14790         const struct dmi_system_id (*dmi_id_list)[];
14791 };
14792
14793 static int intel_dmi_reverse_brightness(const struct dmi_system_id *id)
14794 {
14795         DRM_INFO("Backlight polarity reversed on %s\n", id->ident);
14796         return 1;
14797 }
14798
14799 static const struct intel_dmi_quirk intel_dmi_quirks[] = {
14800         {
14801                 .dmi_id_list = &(const struct dmi_system_id[]) {
14802                         {
14803                                 .callback = intel_dmi_reverse_brightness,
14804                                 .ident = "NCR Corporation",
14805                                 .matches = {DMI_MATCH(DMI_SYS_VENDOR, "NCR Corporation"),
14806                                             DMI_MATCH(DMI_PRODUCT_NAME, ""),
14807                                 },
14808                         },
14809                         { }  /* terminating entry */
14810                 },
14811                 .hook = quirk_invert_brightness,
14812         },
14813 };
14814
14815 static struct intel_quirk intel_quirks[] = {
14816         /* Toshiba Protege R-205, S-209 needs pipe A force quirk */
14817         { 0x2592, 0x1179, 0x0001, quirk_pipea_force },
14818
14819         /* ThinkPad T60 needs pipe A force quirk (bug #16494) */
14820         { 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
14821
14822         /* 830 needs to leave pipe A & dpll A up */
14823         { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
14824
14825         /* 830 needs to leave pipe B & dpll B up */
14826         { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipeb_force },
14827
14828         /* Lenovo U160 cannot use SSC on LVDS */
14829         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
14830
14831         /* Sony Vaio Y cannot use SSC on LVDS */
14832         { 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
14833
14834         /* Acer Aspire 5734Z must invert backlight brightness */
14835         { 0x2a42, 0x1025, 0x0459, quirk_invert_brightness },
14836
14837         /* Acer/eMachines G725 */
14838         { 0x2a42, 0x1025, 0x0210, quirk_invert_brightness },
14839
14840         /* Acer/eMachines e725 */
14841         { 0x2a42, 0x1025, 0x0212, quirk_invert_brightness },
14842
14843         /* Acer/Packard Bell NCL20 */
14844         { 0x2a42, 0x1025, 0x034b, quirk_invert_brightness },
14845
14846         /* Acer Aspire 4736Z */
14847         { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness },
14848
14849         /* Acer Aspire 5336 */
14850         { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness },
14851
14852         /* Acer C720 and C720P Chromebooks (Celeron 2955U) have backlights */
14853         { 0x0a06, 0x1025, 0x0a11, quirk_backlight_present },
14854
14855         /* Acer C720 Chromebook (Core i3 4005U) */
14856         { 0x0a16, 0x1025, 0x0a11, quirk_backlight_present },
14857
14858         /* Apple Macbook 2,1 (Core 2 T7400) */
14859         { 0x27a2, 0x8086, 0x7270, quirk_backlight_present },
14860
14861         /* Toshiba CB35 Chromebook (Celeron 2955U) */
14862         { 0x0a06, 0x1179, 0x0a88, quirk_backlight_present },
14863
14864         /* HP Chromebook 14 (Celeron 2955U) */
14865         { 0x0a06, 0x103c, 0x21ed, quirk_backlight_present },
14866
14867         /* Dell Chromebook 11 */
14868         { 0x0a06, 0x1028, 0x0a35, quirk_backlight_present },
14869 };
14870
14871 static void intel_init_quirks(struct drm_device *dev)
14872 {
14873         struct pci_dev *d = dev->pdev;
14874         int i;
14875
14876         for (i = 0; i < ARRAY_SIZE(intel_quirks); i++) {
14877                 struct intel_quirk *q = &intel_quirks[i];
14878
14879                 if (d->device == q->device &&
14880                     (d->subsystem_vendor == q->subsystem_vendor ||
14881                      q->subsystem_vendor == PCI_ANY_ID) &&
14882                     (d->subsystem_device == q->subsystem_device ||
14883                      q->subsystem_device == PCI_ANY_ID))
14884                         q->hook(dev);
14885         }
14886         for (i = 0; i < ARRAY_SIZE(intel_dmi_quirks); i++) {
14887                 if (dmi_check_system(*intel_dmi_quirks[i].dmi_id_list) != 0)
14888                         intel_dmi_quirks[i].hook(dev);
14889         }
14890 }
14891
14892 /* Disable the VGA plane that we never use */
14893 static void i915_disable_vga(struct drm_device *dev)
14894 {
14895         struct drm_i915_private *dev_priv = dev->dev_private;
14896         u8 sr1;
14897         u32 vga_reg = i915_vgacntrl_reg(dev);
14898
14899         /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
14900         vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
14901         outb(SR01, VGA_SR_INDEX);
14902         sr1 = inb(VGA_SR_DATA);
14903         outb(sr1 | 1<<5, VGA_SR_DATA);
14904         vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
14905         udelay(300);
14906
14907         I915_WRITE(vga_reg, VGA_DISP_DISABLE);
14908         POSTING_READ(vga_reg);
14909 }
14910
14911 void intel_modeset_init_hw(struct drm_device *dev)
14912 {
14913         intel_update_cdclk(dev);
14914         intel_prepare_ddi(dev);
14915         intel_init_clock_gating(dev);
14916         intel_enable_gt_powersave(dev);
14917 }
14918
14919 void intel_modeset_init(struct drm_device *dev)
14920 {
14921         struct drm_i915_private *dev_priv = dev->dev_private;
14922         int sprite, ret;
14923         enum pipe pipe;
14924         struct intel_crtc *crtc;
14925
14926         drm_mode_config_init(dev);
14927
14928         dev->mode_config.min_width = 0;
14929         dev->mode_config.min_height = 0;
14930
14931         dev->mode_config.preferred_depth = 24;
14932         dev->mode_config.prefer_shadow = 1;
14933
14934         dev->mode_config.allow_fb_modifiers = true;
14935
14936         dev->mode_config.funcs = &intel_mode_funcs;
14937
14938         intel_init_quirks(dev);
14939
14940         intel_init_pm(dev);
14941
14942         if (INTEL_INFO(dev)->num_pipes == 0)
14943                 return;
14944
14945         intel_init_display(dev);
14946         intel_init_audio(dev);
14947
14948         if (IS_GEN2(dev)) {
14949                 dev->mode_config.max_width = 2048;
14950                 dev->mode_config.max_height = 2048;
14951         } else if (IS_GEN3(dev)) {
14952                 dev->mode_config.max_width = 4096;
14953                 dev->mode_config.max_height = 4096;
14954         } else {
14955                 dev->mode_config.max_width = 8192;
14956                 dev->mode_config.max_height = 8192;
14957         }
14958
14959         if (IS_845G(dev) || IS_I865G(dev)) {
14960                 dev->mode_config.cursor_width = IS_845G(dev) ? 64 : 512;
14961                 dev->mode_config.cursor_height = 1023;
14962         } else if (IS_GEN2(dev)) {
14963                 dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH;
14964                 dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT;
14965         } else {
14966                 dev->mode_config.cursor_width = MAX_CURSOR_WIDTH;
14967                 dev->mode_config.cursor_height = MAX_CURSOR_HEIGHT;
14968         }
14969
14970         dev->mode_config.fb_base = dev_priv->gtt.mappable_base;
14971
14972         DRM_DEBUG_KMS("%d display pipe%s available.\n",
14973                       INTEL_INFO(dev)->num_pipes,
14974                       INTEL_INFO(dev)->num_pipes > 1 ? "s" : "");
14975
14976         for_each_pipe(dev_priv, pipe) {
14977                 intel_crtc_init(dev, pipe);
14978                 for_each_sprite(dev_priv, pipe, sprite) {
14979                         ret = intel_plane_init(dev, pipe, sprite);
14980                         if (ret)
14981                                 DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n",
14982                                               pipe_name(pipe), sprite_name(pipe, sprite), ret);
14983                 }
14984         }
14985
14986         intel_init_dpio(dev);
14987
14988         intel_shared_dpll_init(dev);
14989
14990         /* Just disable it once at startup */
14991         i915_disable_vga(dev);
14992         intel_setup_outputs(dev);
14993
14994         /* Just in case the BIOS is doing something questionable. */
14995         intel_fbc_disable(dev);
14996
14997         drm_modeset_lock_all(dev);
14998         intel_modeset_setup_hw_state(dev, false);
14999         drm_modeset_unlock_all(dev);
15000
15001         for_each_intel_crtc(dev, crtc) {
15002                 if (!crtc->active)
15003                         continue;
15004
15005                 /*
15006                  * Note that reserving the BIOS fb up front prevents us
15007                  * from stuffing other stolen allocations like the ring
15008                  * on top.  This prevents some ugliness at boot time, and
15009                  * can even allow for smooth boot transitions if the BIOS
15010                  * fb is large enough for the active pipe configuration.
15011                  */
15012                 if (dev_priv->display.get_initial_plane_config) {
15013                         dev_priv->display.get_initial_plane_config(crtc,
15014                                                            &crtc->plane_config);
15015                         /*
15016                          * If the fb is shared between multiple heads, we'll
15017                          * just get the first one.
15018                          */
15019                         intel_find_initial_plane_obj(crtc, &crtc->plane_config);
15020                 }
15021         }
15022 }
15023
15024 static void intel_enable_pipe_a(struct drm_device *dev)
15025 {
15026         struct intel_connector *connector;
15027         struct drm_connector *crt = NULL;
15028         struct intel_load_detect_pipe load_detect_temp;
15029         struct drm_modeset_acquire_ctx *ctx = dev->mode_config.acquire_ctx;
15030
15031         /* We can't just switch on the pipe A, we need to set things up with a
15032          * proper mode and output configuration. As a gross hack, enable pipe A
15033          * by enabling the load detect pipe once. */
15034         for_each_intel_connector(dev, connector) {
15035                 if (connector->encoder->type == INTEL_OUTPUT_ANALOG) {
15036                         crt = &connector->base;
15037                         break;
15038                 }
15039         }
15040
15041         if (!crt)
15042                 return;
15043
15044         if (intel_get_load_detect_pipe(crt, NULL, &load_detect_temp, ctx))
15045                 intel_release_load_detect_pipe(crt, &load_detect_temp, ctx);
15046 }
15047
15048 static bool
15049 intel_check_plane_mapping(struct intel_crtc *crtc)
15050 {
15051         struct drm_device *dev = crtc->base.dev;
15052         struct drm_i915_private *dev_priv = dev->dev_private;
15053         u32 reg, val;
15054
15055         if (INTEL_INFO(dev)->num_pipes == 1)
15056                 return true;
15057
15058         reg = DSPCNTR(!crtc->plane);
15059         val = I915_READ(reg);
15060
15061         if ((val & DISPLAY_PLANE_ENABLE) &&
15062             (!!(val & DISPPLANE_SEL_PIPE_MASK) == crtc->pipe))
15063                 return false;
15064
15065         return true;
15066 }
15067
15068 static void intel_sanitize_crtc(struct intel_crtc *crtc)
15069 {
15070         struct drm_device *dev = crtc->base.dev;
15071         struct drm_i915_private *dev_priv = dev->dev_private;
15072         struct intel_encoder *encoder;
15073         u32 reg;
15074         bool enable;
15075
15076         /* Clear any frame start delays used for debugging left by the BIOS */
15077         reg = PIPECONF(crtc->config->cpu_transcoder);
15078         I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
15079
15080         /* restore vblank interrupts to correct state */
15081         drm_crtc_vblank_reset(&crtc->base);
15082         if (crtc->active) {
15083                 update_scanline_offset(crtc);
15084                 drm_crtc_vblank_on(&crtc->base);
15085         }
15086
15087         /* We need to sanitize the plane -> pipe mapping first because this will
15088          * disable the crtc (and hence change the state) if it is wrong. Note
15089          * that gen4+ has a fixed plane -> pipe mapping.  */
15090         if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) {
15091                 bool plane;
15092
15093                 DRM_DEBUG_KMS("[CRTC:%d] wrong plane connection detected!\n",
15094                               crtc->base.base.id);
15095
15096                 /* Pipe has the wrong plane attached and the plane is active.
15097                  * Temporarily change the plane mapping and disable everything
15098                  * ...  */
15099                 plane = crtc->plane;
15100                 to_intel_plane_state(crtc->base.primary->state)->visible = true;
15101                 crtc->plane = !plane;
15102                 intel_crtc_disable_noatomic(&crtc->base);
15103                 crtc->plane = plane;
15104         }
15105
15106         if (dev_priv->quirks & QUIRK_PIPEA_FORCE &&
15107             crtc->pipe == PIPE_A && !crtc->active) {
15108                 /* BIOS forgot to enable pipe A, this mostly happens after
15109                  * resume. Force-enable the pipe to fix this, the update_dpms
15110                  * call below we restore the pipe to the right state, but leave
15111                  * the required bits on. */
15112                 intel_enable_pipe_a(dev);
15113         }
15114
15115         /* Adjust the state of the output pipe according to whether we
15116          * have active connectors/encoders. */
15117         enable = false;
15118         for_each_encoder_on_crtc(dev, &crtc->base, encoder)
15119                 enable |= encoder->connectors_active;
15120
15121         if (!enable)
15122                 intel_crtc_disable_noatomic(&crtc->base);
15123
15124         if (crtc->active != crtc->base.state->active) {
15125
15126                 /* This can happen either due to bugs in the get_hw_state
15127                  * functions or because of calls to intel_crtc_disable_noatomic,
15128                  * or because the pipe is force-enabled due to the
15129                  * pipe A quirk. */
15130                 DRM_DEBUG_KMS("[CRTC:%d] hw state adjusted, was %s, now %s\n",
15131                               crtc->base.base.id,
15132                               crtc->base.state->enable ? "enabled" : "disabled",
15133                               crtc->active ? "enabled" : "disabled");
15134
15135                 crtc->base.state->enable = crtc->active;
15136                 crtc->base.state->active = crtc->active;
15137                 crtc->base.enabled = crtc->active;
15138
15139                 /* Because we only establish the connector -> encoder ->
15140                  * crtc links if something is active, this means the
15141                  * crtc is now deactivated. Break the links. connector
15142                  * -> encoder links are only establish when things are
15143                  *  actually up, hence no need to break them. */
15144                 WARN_ON(crtc->active);
15145
15146                 for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
15147                         WARN_ON(encoder->connectors_active);
15148                         encoder->base.crtc = NULL;
15149                 }
15150         }
15151
15152         if (crtc->active || HAS_GMCH_DISPLAY(dev)) {
15153                 /*
15154                  * We start out with underrun reporting disabled to avoid races.
15155                  * For correct bookkeeping mark this on active crtcs.
15156                  *
15157                  * Also on gmch platforms we dont have any hardware bits to
15158                  * disable the underrun reporting. Which means we need to start
15159                  * out with underrun reporting disabled also on inactive pipes,
15160                  * since otherwise we'll complain about the garbage we read when
15161                  * e.g. coming up after runtime pm.
15162                  *
15163                  * No protection against concurrent access is required - at
15164                  * worst a fifo underrun happens which also sets this to false.
15165                  */
15166                 crtc->cpu_fifo_underrun_disabled = true;
15167                 crtc->pch_fifo_underrun_disabled = true;
15168         }
15169 }
15170
15171 static void intel_sanitize_encoder(struct intel_encoder *encoder)
15172 {
15173         struct intel_connector *connector;
15174         struct drm_device *dev = encoder->base.dev;
15175
15176         /* We need to check both for a crtc link (meaning that the
15177          * encoder is active and trying to read from a pipe) and the
15178          * pipe itself being active. */
15179         bool has_active_crtc = encoder->base.crtc &&
15180                 to_intel_crtc(encoder->base.crtc)->active;
15181
15182         if (encoder->connectors_active && !has_active_crtc) {
15183                 DRM_DEBUG_KMS("[ENCODER:%d:%s] has active connectors but no active pipe!\n",
15184                               encoder->base.base.id,
15185                               encoder->base.name);
15186
15187                 /* Connector is active, but has no active pipe. This is
15188                  * fallout from our resume register restoring. Disable
15189                  * the encoder manually again. */
15190                 if (encoder->base.crtc) {
15191                         DRM_DEBUG_KMS("[ENCODER:%d:%s] manually disabled\n",
15192                                       encoder->base.base.id,
15193                                       encoder->base.name);
15194                         encoder->disable(encoder);
15195                         if (encoder->post_disable)
15196                                 encoder->post_disable(encoder);
15197                 }
15198                 encoder->base.crtc = NULL;
15199                 encoder->connectors_active = false;
15200
15201                 /* Inconsistent output/port/pipe state happens presumably due to
15202                  * a bug in one of the get_hw_state functions. Or someplace else
15203                  * in our code, like the register restore mess on resume. Clamp
15204                  * things to off as a safer default. */
15205                 for_each_intel_connector(dev, connector) {
15206                         if (connector->encoder != encoder)
15207                                 continue;
15208                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15209                         connector->base.encoder = NULL;
15210                 }
15211         }
15212         /* Enabled encoders without active connectors will be fixed in
15213          * the crtc fixup. */
15214 }
15215
15216 void i915_redisable_vga_power_on(struct drm_device *dev)
15217 {
15218         struct drm_i915_private *dev_priv = dev->dev_private;
15219         u32 vga_reg = i915_vgacntrl_reg(dev);
15220
15221         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
15222                 DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
15223                 i915_disable_vga(dev);
15224         }
15225 }
15226
15227 void i915_redisable_vga(struct drm_device *dev)
15228 {
15229         struct drm_i915_private *dev_priv = dev->dev_private;
15230
15231         /* This function can be called both from intel_modeset_setup_hw_state or
15232          * at a very early point in our resume sequence, where the power well
15233          * structures are not yet restored. Since this function is at a very
15234          * paranoid "someone might have enabled VGA while we were not looking"
15235          * level, just check if the power well is enabled instead of trying to
15236          * follow the "don't touch the power well if we don't need it" policy
15237          * the rest of the driver uses. */
15238         if (!intel_display_power_is_enabled(dev_priv, POWER_DOMAIN_VGA))
15239                 return;
15240
15241         i915_redisable_vga_power_on(dev);
15242 }
15243
15244 static bool primary_get_hw_state(struct intel_crtc *crtc)
15245 {
15246         struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
15247
15248         return !!(I915_READ(DSPCNTR(crtc->plane)) & DISPLAY_PLANE_ENABLE);
15249 }
15250
15251 static void readout_plane_state(struct intel_crtc *crtc,
15252                                 struct intel_crtc_state *crtc_state)
15253 {
15254         struct intel_plane *p;
15255         struct drm_plane_state *drm_plane_state;
15256         bool active = crtc_state->base.active;
15257
15258         if (active) {
15259                 crtc_state->quirks |= PIPE_CONFIG_QUIRK_INITIAL_PLANES;
15260
15261                 /* apply to previous sw state too */
15262                 to_intel_crtc_state(crtc->base.state)->quirks |=
15263                         PIPE_CONFIG_QUIRK_INITIAL_PLANES;
15264         }
15265
15266         for_each_intel_plane(crtc->base.dev, p) {
15267                 bool visible = active;
15268
15269                 if (crtc->pipe != p->pipe)
15270                         continue;
15271
15272                 drm_plane_state = p->base.state;
15273                 if (active && p->base.type == DRM_PLANE_TYPE_PRIMARY) {
15274                         visible = primary_get_hw_state(crtc);
15275                         to_intel_plane_state(drm_plane_state)->visible = visible;
15276                 } else {
15277                         /*
15278                          * unknown state, assume it's off to force a transition
15279                          * to on when calculating state changes.
15280                          */
15281                         to_intel_plane_state(drm_plane_state)->visible = false;
15282                 }
15283
15284                 if (visible) {
15285                         crtc_state->base.plane_mask |=
15286                                 1 << drm_plane_index(&p->base);
15287                 } else if (crtc_state->base.state) {
15288                         /* Make this unconditional for atomic hw readout. */
15289                         crtc_state->base.plane_mask &=
15290                                 ~(1 << drm_plane_index(&p->base));
15291                 }
15292         }
15293 }
15294
15295 static void intel_modeset_readout_hw_state(struct drm_device *dev)
15296 {
15297         struct drm_i915_private *dev_priv = dev->dev_private;
15298         enum pipe pipe;
15299         struct intel_crtc *crtc;
15300         struct intel_encoder *encoder;
15301         struct intel_connector *connector;
15302         int i;
15303
15304         for_each_intel_crtc(dev, crtc) {
15305                 memset(crtc->config, 0, sizeof(*crtc->config));
15306                 crtc->config->base.crtc = &crtc->base;
15307
15308                 crtc->config->quirks |= PIPE_CONFIG_QUIRK_INHERITED_MODE;
15309
15310                 crtc->active = dev_priv->display.get_pipe_config(crtc,
15311                                                                  crtc->config);
15312
15313                 crtc->base.state->enable = crtc->active;
15314                 crtc->base.state->active = crtc->active;
15315                 crtc->base.enabled = crtc->active;
15316                 crtc->base.hwmode = crtc->config->base.adjusted_mode;
15317
15318                 readout_plane_state(crtc, to_intel_crtc_state(crtc->base.state));
15319
15320                 DRM_DEBUG_KMS("[CRTC:%d] hw state readout: %s\n",
15321                               crtc->base.base.id,
15322                               crtc->active ? "enabled" : "disabled");
15323         }
15324
15325         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15326                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15327
15328                 pll->on = pll->get_hw_state(dev_priv, pll,
15329                                             &pll->config.hw_state);
15330                 pll->active = 0;
15331                 pll->config.crtc_mask = 0;
15332                 for_each_intel_crtc(dev, crtc) {
15333                         if (crtc->active && intel_crtc_to_shared_dpll(crtc) == pll) {
15334                                 pll->active++;
15335                                 pll->config.crtc_mask |= 1 << crtc->pipe;
15336                         }
15337                 }
15338
15339                 DRM_DEBUG_KMS("%s hw state readout: crtc_mask 0x%08x, on %i\n",
15340                               pll->name, pll->config.crtc_mask, pll->on);
15341
15342                 if (pll->config.crtc_mask)
15343                         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
15344         }
15345
15346         for_each_intel_encoder(dev, encoder) {
15347                 pipe = 0;
15348
15349                 if (encoder->get_hw_state(encoder, &pipe)) {
15350                         crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
15351                         encoder->base.crtc = &crtc->base;
15352                         encoder->get_config(encoder, crtc->config);
15353                 } else {
15354                         encoder->base.crtc = NULL;
15355                 }
15356
15357                 encoder->connectors_active = false;
15358                 DRM_DEBUG_KMS("[ENCODER:%d:%s] hw state readout: %s, pipe %c\n",
15359                               encoder->base.base.id,
15360                               encoder->base.name,
15361                               encoder->base.crtc ? "enabled" : "disabled",
15362                               pipe_name(pipe));
15363         }
15364
15365         for_each_intel_connector(dev, connector) {
15366                 if (connector->get_hw_state(connector)) {
15367                         connector->base.dpms = DRM_MODE_DPMS_ON;
15368                         connector->encoder->connectors_active = true;
15369                         connector->base.encoder = &connector->encoder->base;
15370                 } else {
15371                         connector->base.dpms = DRM_MODE_DPMS_OFF;
15372                         connector->base.encoder = NULL;
15373                 }
15374                 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] hw state readout: %s\n",
15375                               connector->base.base.id,
15376                               connector->base.name,
15377                               connector->base.encoder ? "enabled" : "disabled");
15378         }
15379 }
15380
15381 /* Scan out the current hw modeset state, sanitizes it and maps it into the drm
15382  * and i915 state tracking structures. */
15383 void intel_modeset_setup_hw_state(struct drm_device *dev,
15384                                   bool force_restore)
15385 {
15386         struct drm_i915_private *dev_priv = dev->dev_private;
15387         enum pipe pipe;
15388         struct intel_crtc *crtc;
15389         struct intel_encoder *encoder;
15390         int i;
15391
15392         intel_modeset_readout_hw_state(dev);
15393
15394         /*
15395          * Now that we have the config, copy it to each CRTC struct
15396          * Note that this could go away if we move to using crtc_config
15397          * checking everywhere.
15398          */
15399         for_each_intel_crtc(dev, crtc) {
15400                 if (crtc->active && i915.fastboot) {
15401                         intel_mode_from_pipe_config(&crtc->base.mode,
15402                                                     crtc->config);
15403                         DRM_DEBUG_KMS("[CRTC:%d] found active mode: ",
15404                                       crtc->base.base.id);
15405                         drm_mode_debug_printmodeline(&crtc->base.mode);
15406                 }
15407         }
15408
15409         /* HW state is read out, now we need to sanitize this mess. */
15410         for_each_intel_encoder(dev, encoder) {
15411                 intel_sanitize_encoder(encoder);
15412         }
15413
15414         for_each_pipe(dev_priv, pipe) {
15415                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
15416                 intel_sanitize_crtc(crtc);
15417                 intel_dump_pipe_config(crtc, crtc->config,
15418                                        "[setup_hw_state]");
15419         }
15420
15421         intel_modeset_update_connector_atomic_state(dev);
15422
15423         for (i = 0; i < dev_priv->num_shared_dpll; i++) {
15424                 struct intel_shared_dpll *pll = &dev_priv->shared_dplls[i];
15425
15426                 if (!pll->on || pll->active)
15427                         continue;
15428
15429                 DRM_DEBUG_KMS("%s enabled but not in use, disabling\n", pll->name);
15430
15431                 pll->disable(dev_priv, pll);
15432                 pll->on = false;
15433         }
15434
15435         if (IS_GEN9(dev))
15436                 skl_wm_get_hw_state(dev);
15437         else if (HAS_PCH_SPLIT(dev))
15438                 ilk_wm_get_hw_state(dev);
15439
15440         if (force_restore) {
15441                 i915_redisable_vga(dev);
15442
15443                 /*
15444                  * We need to use raw interfaces for restoring state to avoid
15445                  * checking (bogus) intermediate states.
15446                  */
15447                 for_each_pipe(dev_priv, pipe) {
15448                         struct drm_crtc *crtc =
15449                                 dev_priv->pipe_to_crtc_mapping[pipe];
15450
15451                         intel_crtc_restore_mode(crtc);
15452                 }
15453         } else {
15454                 intel_modeset_update_staged_output_state(dev);
15455         }
15456
15457         intel_modeset_check_state(dev);
15458 }
15459
15460 void intel_modeset_gem_init(struct drm_device *dev)
15461 {
15462         struct drm_i915_private *dev_priv = dev->dev_private;
15463         struct drm_crtc *c;
15464         struct drm_i915_gem_object *obj;
15465         int ret;
15466
15467         mutex_lock(&dev->struct_mutex);
15468         intel_init_gt_powersave(dev);
15469         mutex_unlock(&dev->struct_mutex);
15470
15471         /*
15472          * There may be no VBT; and if the BIOS enabled SSC we can
15473          * just keep using it to avoid unnecessary flicker.  Whereas if the
15474          * BIOS isn't using it, don't assume it will work even if the VBT
15475          * indicates as much.
15476          */
15477         if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
15478                 dev_priv->vbt.lvds_use_ssc = !!(I915_READ(PCH_DREF_CONTROL) &
15479                                                 DREF_SSC1_ENABLE);
15480
15481         intel_modeset_init_hw(dev);
15482
15483         intel_setup_overlay(dev);
15484
15485         /*
15486          * Make sure any fbs we allocated at startup are properly
15487          * pinned & fenced.  When we do the allocation it's too early
15488          * for this.
15489          */
15490         for_each_crtc(dev, c) {
15491                 obj = intel_fb_obj(c->primary->fb);
15492                 if (obj == NULL)
15493                         continue;
15494
15495                 mutex_lock(&dev->struct_mutex);
15496                 ret = intel_pin_and_fence_fb_obj(c->primary,
15497                                                  c->primary->fb,
15498                                                  c->primary->state,
15499                                                  NULL, NULL);
15500                 mutex_unlock(&dev->struct_mutex);
15501                 if (ret) {
15502                         DRM_ERROR("failed to pin boot fb on pipe %d\n",
15503                                   to_intel_crtc(c)->pipe);
15504                         drm_framebuffer_unreference(c->primary->fb);
15505                         c->primary->fb = NULL;
15506                         c->primary->crtc = c->primary->state->crtc = NULL;
15507                         update_state_fb(c->primary);
15508                         c->state->plane_mask &= ~(1 << drm_plane_index(c->primary));
15509                 }
15510         }
15511
15512         intel_backlight_register(dev);
15513 }
15514
15515 void intel_connector_unregister(struct intel_connector *intel_connector)
15516 {
15517         struct drm_connector *connector = &intel_connector->base;
15518
15519         intel_panel_destroy_backlight(connector);
15520         drm_connector_unregister(connector);
15521 }
15522
15523 void intel_modeset_cleanup(struct drm_device *dev)
15524 {
15525         struct drm_i915_private *dev_priv = dev->dev_private;
15526         struct drm_connector *connector;
15527
15528         intel_disable_gt_powersave(dev);
15529
15530         intel_backlight_unregister(dev);
15531
15532         /*
15533          * Interrupts and polling as the first thing to avoid creating havoc.
15534          * Too much stuff here (turning of connectors, ...) would
15535          * experience fancy races otherwise.
15536          */
15537         intel_irq_uninstall(dev_priv);
15538
15539         /*
15540          * Due to the hpd irq storm handling the hotplug work can re-arm the
15541          * poll handlers. Hence disable polling after hpd handling is shut down.
15542          */
15543         drm_kms_helper_poll_fini(dev);
15544
15545         mutex_lock(&dev->struct_mutex);
15546
15547         intel_unregister_dsm_handler();
15548
15549         intel_fbc_disable(dev);
15550
15551         mutex_unlock(&dev->struct_mutex);
15552
15553         /* flush any delayed tasks or pending work */
15554         flush_scheduled_work();
15555
15556         /* destroy the backlight and sysfs files before encoders/connectors */
15557         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
15558                 struct intel_connector *intel_connector;
15559
15560                 intel_connector = to_intel_connector(connector);
15561                 intel_connector->unregister(intel_connector);
15562         }
15563
15564         drm_mode_config_cleanup(dev);
15565
15566         intel_cleanup_overlay(dev);
15567
15568         mutex_lock(&dev->struct_mutex);
15569         intel_cleanup_gt_powersave(dev);
15570         mutex_unlock(&dev->struct_mutex);
15571 }
15572
15573 /*
15574  * Return which encoder is currently attached for connector.
15575  */
15576 struct drm_encoder *intel_best_encoder(struct drm_connector *connector)
15577 {
15578         return &intel_attached_encoder(connector)->base;
15579 }
15580
15581 void intel_connector_attach_encoder(struct intel_connector *connector,
15582                                     struct intel_encoder *encoder)
15583 {
15584         connector->encoder = encoder;
15585         drm_mode_connector_attach_encoder(&connector->base,
15586                                           &encoder->base);
15587 }
15588
15589 /*
15590  * set vga decode state - true == enable VGA decode
15591  */
15592 int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
15593 {
15594         struct drm_i915_private *dev_priv = dev->dev_private;
15595         unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
15596         u16 gmch_ctrl;
15597
15598         if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
15599                 DRM_ERROR("failed to read control word\n");
15600                 return -EIO;
15601         }
15602
15603         if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
15604                 return 0;
15605
15606         if (state)
15607                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
15608         else
15609                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
15610
15611         if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
15612                 DRM_ERROR("failed to write control word\n");
15613                 return -EIO;
15614         }
15615
15616         return 0;
15617 }
15618
15619 struct intel_display_error_state {
15620
15621         u32 power_well_driver;
15622
15623         int num_transcoders;
15624
15625         struct intel_cursor_error_state {
15626                 u32 control;
15627                 u32 position;
15628                 u32 base;
15629                 u32 size;
15630         } cursor[I915_MAX_PIPES];
15631
15632         struct intel_pipe_error_state {
15633                 bool power_domain_on;
15634                 u32 source;
15635                 u32 stat;
15636         } pipe[I915_MAX_PIPES];
15637
15638         struct intel_plane_error_state {
15639                 u32 control;
15640                 u32 stride;
15641                 u32 size;
15642                 u32 pos;
15643                 u32 addr;
15644                 u32 surface;
15645                 u32 tile_offset;
15646         } plane[I915_MAX_PIPES];
15647
15648         struct intel_transcoder_error_state {
15649                 bool power_domain_on;
15650                 enum transcoder cpu_transcoder;
15651
15652                 u32 conf;
15653
15654                 u32 htotal;
15655                 u32 hblank;
15656                 u32 hsync;
15657                 u32 vtotal;
15658                 u32 vblank;
15659                 u32 vsync;
15660         } transcoder[4];
15661 };
15662
15663 struct intel_display_error_state *
15664 intel_display_capture_error_state(struct drm_device *dev)
15665 {
15666         struct drm_i915_private *dev_priv = dev->dev_private;
15667         struct intel_display_error_state *error;
15668         int transcoders[] = {
15669                 TRANSCODER_A,
15670                 TRANSCODER_B,
15671                 TRANSCODER_C,
15672                 TRANSCODER_EDP,
15673         };
15674         int i;
15675
15676         if (INTEL_INFO(dev)->num_pipes == 0)
15677                 return NULL;
15678
15679         error = kzalloc(sizeof(*error), GFP_ATOMIC);
15680         if (error == NULL)
15681                 return NULL;
15682
15683         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
15684                 error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER);
15685
15686         for_each_pipe(dev_priv, i) {
15687                 error->pipe[i].power_domain_on =
15688                         __intel_display_power_is_enabled(dev_priv,
15689                                                          POWER_DOMAIN_PIPE(i));
15690                 if (!error->pipe[i].power_domain_on)
15691                         continue;
15692
15693                 error->cursor[i].control = I915_READ(CURCNTR(i));
15694                 error->cursor[i].position = I915_READ(CURPOS(i));
15695                 error->cursor[i].base = I915_READ(CURBASE(i));
15696
15697                 error->plane[i].control = I915_READ(DSPCNTR(i));
15698                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
15699                 if (INTEL_INFO(dev)->gen <= 3) {
15700                         error->plane[i].size = I915_READ(DSPSIZE(i));
15701                         error->plane[i].pos = I915_READ(DSPPOS(i));
15702                 }
15703                 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
15704                         error->plane[i].addr = I915_READ(DSPADDR(i));
15705                 if (INTEL_INFO(dev)->gen >= 4) {
15706                         error->plane[i].surface = I915_READ(DSPSURF(i));
15707                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
15708                 }
15709
15710                 error->pipe[i].source = I915_READ(PIPESRC(i));
15711
15712                 if (HAS_GMCH_DISPLAY(dev))
15713                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
15714         }
15715
15716         error->num_transcoders = INTEL_INFO(dev)->num_pipes;
15717         if (HAS_DDI(dev_priv->dev))
15718                 error->num_transcoders++; /* Account for eDP. */
15719
15720         for (i = 0; i < error->num_transcoders; i++) {
15721                 enum transcoder cpu_transcoder = transcoders[i];
15722
15723                 error->transcoder[i].power_domain_on =
15724                         __intel_display_power_is_enabled(dev_priv,
15725                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
15726                 if (!error->transcoder[i].power_domain_on)
15727                         continue;
15728
15729                 error->transcoder[i].cpu_transcoder = cpu_transcoder;
15730
15731                 error->transcoder[i].conf = I915_READ(PIPECONF(cpu_transcoder));
15732                 error->transcoder[i].htotal = I915_READ(HTOTAL(cpu_transcoder));
15733                 error->transcoder[i].hblank = I915_READ(HBLANK(cpu_transcoder));
15734                 error->transcoder[i].hsync = I915_READ(HSYNC(cpu_transcoder));
15735                 error->transcoder[i].vtotal = I915_READ(VTOTAL(cpu_transcoder));
15736                 error->transcoder[i].vblank = I915_READ(VBLANK(cpu_transcoder));
15737                 error->transcoder[i].vsync = I915_READ(VSYNC(cpu_transcoder));
15738         }
15739
15740         return error;
15741 }
15742
15743 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
15744
15745 void
15746 intel_display_print_error_state(struct drm_i915_error_state_buf *m,
15747                                 struct drm_device *dev,
15748                                 struct intel_display_error_state *error)
15749 {
15750         struct drm_i915_private *dev_priv = dev->dev_private;
15751         int i;
15752
15753         if (!error)
15754                 return;
15755
15756         err_printf(m, "Num Pipes: %d\n", INTEL_INFO(dev)->num_pipes);
15757         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
15758                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
15759                            error->power_well_driver);
15760         for_each_pipe(dev_priv, i) {
15761                 err_printf(m, "Pipe [%d]:\n", i);
15762                 err_printf(m, "  Power: %s\n",
15763                            error->pipe[i].power_domain_on ? "on" : "off");
15764                 err_printf(m, "  SRC: %08x\n", error->pipe[i].source);
15765                 err_printf(m, "  STAT: %08x\n", error->pipe[i].stat);
15766
15767                 err_printf(m, "Plane [%d]:\n", i);
15768                 err_printf(m, "  CNTR: %08x\n", error->plane[i].control);
15769                 err_printf(m, "  STRIDE: %08x\n", error->plane[i].stride);
15770                 if (INTEL_INFO(dev)->gen <= 3) {
15771                         err_printf(m, "  SIZE: %08x\n", error->plane[i].size);
15772                         err_printf(m, "  POS: %08x\n", error->plane[i].pos);
15773                 }
15774                 if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
15775                         err_printf(m, "  ADDR: %08x\n", error->plane[i].addr);
15776                 if (INTEL_INFO(dev)->gen >= 4) {
15777                         err_printf(m, "  SURF: %08x\n", error->plane[i].surface);
15778                         err_printf(m, "  TILEOFF: %08x\n", error->plane[i].tile_offset);
15779                 }
15780
15781                 err_printf(m, "Cursor [%d]:\n", i);
15782                 err_printf(m, "  CNTR: %08x\n", error->cursor[i].control);
15783                 err_printf(m, "  POS: %08x\n", error->cursor[i].position);
15784                 err_printf(m, "  BASE: %08x\n", error->cursor[i].base);
15785         }
15786
15787         for (i = 0; i < error->num_transcoders; i++) {
15788                 err_printf(m, "CPU transcoder: %c\n",
15789                            transcoder_name(error->transcoder[i].cpu_transcoder));
15790                 err_printf(m, "  Power: %s\n",
15791                            error->transcoder[i].power_domain_on ? "on" : "off");
15792                 err_printf(m, "  CONF: %08x\n", error->transcoder[i].conf);
15793                 err_printf(m, "  HTOTAL: %08x\n", error->transcoder[i].htotal);
15794                 err_printf(m, "  HBLANK: %08x\n", error->transcoder[i].hblank);
15795                 err_printf(m, "  HSYNC: %08x\n", error->transcoder[i].hsync);
15796                 err_printf(m, "  VTOTAL: %08x\n", error->transcoder[i].vtotal);
15797                 err_printf(m, "  VBLANK: %08x\n", error->transcoder[i].vblank);
15798                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
15799         }
15800 }
15801
15802 void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file)
15803 {
15804         struct intel_crtc *crtc;
15805
15806         for_each_intel_crtc(dev, crtc) {
15807                 struct intel_unpin_work *work;
15808
15809                 spin_lock_irq(&dev->event_lock);
15810
15811                 work = crtc->unpin_work;
15812
15813                 if (work && work->event &&
15814                     work->event->base.file_priv == file) {
15815                         kfree(work->event);
15816                         work->event = NULL;
15817                 }
15818
15819                 spin_unlock_irq(&dev->event_lock);
15820         }
15821 }